Merge pull request running-elephant#1737 from running-elephant/pre-release

Pre release - 1.0.0-rc.0

Author: scottsut

.github/workflows/dev-ut-stage.js.yml

@@ -0,0 +1,35 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [14.x, 16.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+        cache-dependency-path: '**/package-lock.json'
+    - run: npm ci
+      working-directory: ./frontend
+    - run: npm run build --if-present
+      working-directory: ./frontend
+    - run: npm run test:ci
+      working-directory: ./frontend

config/jdbc-driver-ext.yml

@@ -1,10 +1,18 @@
 #EXAPLE:
 #  db-type: "example"
 #  name: "example"
-#  driver-class: "com.example.jdbc.Driver"
 #  literal-quote: "'"
 #  identifier-quote: "`"
-#  url-prefix: jdbc:example://
+#  driver-class: "com.example.jdbc.Driver" # optional
+#  url-prefix: "jdbc:example://" # optional
+#  sql-dialect: "datart.data.provider.calcite.dialect.ExampleSqlDialectSupport"  # optional
+#  identifier-end-quote: "`"  # optional
+#  literal-end-quote: "'"  # optional
+#  identifier-escaped-quote: "`"  # optional
+#  adapter-class: 'datart.data.provider.jdbc.adapters.ExampleDataProviderAdapter'  # optional
+#  quote-identifiers: true  # optional
+#  support-sql-limit: true  # optional
+#
 
 IMPALA:
   db-type: "impala"

core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-beta.4</version>
+        <version>1.0.0-rc.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

core/src/main/java/datart/core/data/provider/Dataframe.java

@@ -23,8 +23,8 @@
 import lombok.Data;
 
 import java.io.Serializable;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
+import java.util.stream.Collectors;
 
 
 @Data
@@ -62,4 +62,51 @@ public static Dataframe empty() {
         return dataframe;
     }
 
+    // 按照指定的列定义，将数据集按照表名称进行分割，以还原原始表结构
+    public Dataframes splitByTable(Map<String, Column> newSchema) {
+        Map<Integer, String> tableColumnIndex = new HashMap<>();
+        for (int i = 0; i < columns.size(); i++) {
+            Column column = columns.get(i);
+            Column schemaColumn = newSchema.get(column.columnKey());
+            tableColumnIndex.put(i, schemaColumn.tableName());
+        }
+        Map<String, List<List<Object>>> tableRows = newSchema
+                .values()
+                .stream()
+                .map(Column::tableName)
+                .distinct()
+                .collect(Collectors.toMap(k -> k, v -> new ArrayList()));
+        for (List<Object> row : rows) {
+            int i = 0;
+            Map<String, List<Object>> tableRowMap = new HashMap<>();
+            for (Object item : row) {
+                String tableName = tableColumnIndex.get(i);
+                tableRowMap.computeIfAbsent(tableName, v -> new ArrayList<>()).add(item);
+                i++;
+            }
+            for (String key : tableRowMap.keySet()) {
+                tableRows.get(key).add(tableRowMap.get(key));
+            }
+        }
+        Map<String, List<Column>> tableColumns = new HashMap<>();
+        for (int i = 0; i < columns.size(); i++) {
+            Column column = columns.get(i);
+            Column newColumn = newSchema.get(column.columnKey());
+            String tableName = newColumn.tableName();
+            newColumn.setName(newColumn.columnName());
+            tableColumns.computeIfAbsent(tableName, v -> new ArrayList<>())
+                    .add(newColumn);
+        }
+        Dataframe[] dataframes = tableColumns.keySet().stream()
+                .map(tableName -> {
+                    Dataframe df = new Dataframe();
+                    df.setName(tableName);
+                    df.setColumns(tableColumns.get(tableName));
+                    df.setRows(tableRows.get(tableName));
+                    return df;
+                }).toArray(Dataframe[]::new);
+        return Dataframes.of(id, dataframes);
+    }
+
+
 }

core/src/main/java/datart/core/data/provider/Dataframes.java

@@ -30,7 +30,7 @@ public class Dataframes implements Serializable {
 
     private final String key;
 
-    List<Dataframe> dataframes;
+    private final List<Dataframe> dataframes;
 
     private Dataframes(String key) {
         this.key = "DB" + key;
@@ -54,4 +54,8 @@ public static Dataframes of(String key, Dataframe... dataframes) {
     public void add(Dataframe df) {
         dataframes.add(df);
     }
+
+    public int size() {
+        return dataframes.size();
+    }
 }

core/src/main/java/datart/core/mappers/ext/ShareMapperExt.java

@@ -28,7 +28,7 @@
 public interface ShareMapperExt extends ShareMapper {
 
     @Select({"<script>",
-            "SELECT * FROM `share` where viz_id = #{vizId} ORDER BY create_time",
+            "SELECT * FROM `share` where viz_id = #{vizId} AND create_by &lt;&gt; 'SCHEDULER' ORDER BY create_time",
             "</script>"})
     List<Share> selectByViz(String vizId);
 

data-providers/data-provider-base/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-data-provider</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-beta.4</version>
+        <version>1.0.0-rc.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

data-providers/data-provider-base/src/main/java/datart/data/provider/ProviderManager.java

@@ -195,7 +195,7 @@ private void excludeColumns(Dataframe data, Set<SelectColumn> include) {
             if (include
                     .stream()
                     .noneMatch(selectColumn ->
-                            column.getName().equals(selectColumn.getColumnKey()) || column.getName().equals(selectColumn.getAlias()))) {
+                            column.columnKey().equals(selectColumn.getColumnKey()) || column.columnKey().equals(selectColumn.getAlias()))) {
                 excludeIndex.add(i);
             }
         }

data-providers/data-provider-base/src/main/java/datart/data/provider/calcite/SqlVariableVisitor.java

@@ -17,8 +17,10 @@
  */
 package datart.data.provider.calcite;
 
+import datart.core.base.exception.Exceptions;
 import datart.core.data.provider.ScriptVariable;
 import datart.data.provider.jdbc.SimpleVariablePlaceholder;
+import datart.data.provider.script.SqlStringUtils;
 import datart.data.provider.script.VariablePlaceholder;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.calcite.sql.*;
@@ -121,16 +123,10 @@ private void createVariablePlaceholders(SqlCall logicExpressionCall, Set<SqlIden
         }
 
         logicExpressionCall = SpecialSqlCallConverter.convert(logicExpressionCall);
-        int startIndex = logicExpressionCall.getParserPosition().getColumnNum();
-        int endIndex = logicExpressionCall.getParserPosition().getEndColumnNum();
-        // 处理calcite不把左右括号算进index，导致的index错位问题
-        if (startIndex > 1 && srcSql.charAt(startIndex - 2) == '(') {
-            startIndex = startIndex - 1;
-        }
-        if (endIndex < srcSql.length() && srcSql.charAt(endIndex) == ')') {
-            endIndex = endIndex + 1;
-        }
-        String originalSqlFragment = srcSql.substring(startIndex - 1, endIndex).trim();
+        int startIndex = logicExpressionCall.getParserPosition().getColumnNum() - 1;
+        int endIndex = logicExpressionCall.getParserPosition().getEndColumnNum() - 1;
+
+        String originalSqlFragment = fixMissedParentheses(srcSql, startIndex, endIndex);
 
         List<ScriptVariable> variables = new LinkedList<>();
         for (SqlIdentifier identifier : variableIdentifier) {
@@ -145,4 +141,50 @@ private void createVariablePlaceholders(SqlCall logicExpressionCall, Set<SqlIden
         variablePlaceholders.add(new VariablePlaceholder(variables, sqlDialect, logicExpressionCall, originalSqlFragment));
     }
 
+    // 处理calcite某些情况下不把左右括号算进index，导致的index错位问题
+    private String fixMissedParentheses(String srcSql, int startIndex, int endIndex) {
+
+        String originalSqlFragment = srcSql.substring(startIndex, endIndex + 1).trim();
+
+        char[] missedParentheses = SqlStringUtils.findMissedParentheses(originalSqlFragment);
+        if (missedParentheses.length != 0) {
+            int left = 0;
+            int right = 0;
+            for (char parenthesis : missedParentheses) {
+                if (parenthesis == '(') {
+                    left++;
+                } else {
+                    right++;
+                }
+            }
+            while (left != 0) {
+                startIndex--;
+                if (startIndex < 0) {
+                    Exceptions.msg("There are mismatched parentheses nearby " + originalSqlFragment);
+                }
+                if (srcSql.charAt(startIndex) == ' ') {
+                    continue;
+                }
+                if (srcSql.charAt(startIndex) == '(') {
+                    left--;
+                }
+            }
+            while (right != 0) {
+                endIndex++;
+                if (endIndex >= srcSql.length()) {
+                    Exceptions.msg("There are mismatched parentheses nearby " + originalSqlFragment);
+                }
+                if (srcSql.charAt(endIndex) == ' ') {
+                    continue;
+                }
+                if (srcSql.charAt(endIndex) == ')') {
+                    right--;
+                } else {
+                    Exceptions.msg("There are mismatched parentheses nearby " + originalSqlFragment);
+                }
+            }
+        }
+        return srcSql.substring(startIndex, endIndex + 1).trim();
+    }
+
 }

data-providers/data-provider-base/src/main/java/datart/data/provider/calcite/StructScriptProcessor.java

@@ -52,6 +52,9 @@ public QueryScriptProcessResult process(QueryScript queryScript) {
             SqlNode conditionNode = null;
             if (!CollectionUtils.isEmpty(tableJoin.getConditions())) {
                 for (JoinCondition joinCondition : tableJoin.getConditions()) {
+                    if (!joinCondition.isValid()) {
+                        continue;
+                    }
                     SqlBasicCall condition = new SqlBasicCall(SqlStdOperatorTable.EQUALS
                             , new SqlNode[]{SqlNodeUtils.createSqlIdentifier(joinCondition.getLeft())
                             , SqlNodeUtils.createSqlIdentifier(joinCondition.getRight())}