Skip to content

Commit 5b62a8b

Browse files
ivandaschivandasch
authored
IGNITE-17190 Move ignite query statistics to core, fix SQL ANALYZE command (apache#10175)
1 parent 2361ff3 commit 5b62a8b

File tree

90 files changed

+1374
-1034
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

90 files changed

+1374
-1034
lines changed

‎modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/metadata/IgniteMdSelectivity.java‎

Lines changed: 12 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,10 @@
1818
package org.apache.ignite.internal.processors.query.calcite.metadata;
1919

2020
import java.math.BigDecimal;
21-
import java.math.BigInteger;
2221
import java.math.MathContext;
2322
import java.util.HashMap;
2423
import java.util.List;
2524
import java.util.Map;
26-
2725
import org.apache.calcite.plan.RelOptUtil;
2826
import org.apache.calcite.plan.volcano.RelSubset;
2927
import org.apache.calcite.rel.RelNode;
@@ -60,7 +58,6 @@
6058
import org.apache.ignite.internal.processors.query.calcite.schema.IgniteTable;
6159
import org.apache.ignite.internal.processors.query.calcite.util.RexUtils;
6260
import org.apache.ignite.internal.processors.query.stat.ColumnStatistics;
63-
import org.h2.value.Value;
6461
import org.jetbrains.annotations.Nullable;
6562

6663
/** */
@@ -167,74 +164,6 @@ private BigDecimal toComparableValue(RexLiteral val) {
167164
return null;
168165
}
169166

170-
/**
171-
* Convert specified value into comparable type: BigDecimal,
172-
*
173-
* @param val Value to convert to comparable form.
174-
* @return Comparable form of value.
175-
*/
176-
private BigDecimal toComparableValue(Value val) {
177-
if (val == null)
178-
return null;
179-
180-
switch (val.getType()) {
181-
case Value.NULL:
182-
throw new IllegalArgumentException("Can't compare null values");
183-
184-
case Value.BOOLEAN:
185-
return (val.getBoolean()) ? BigDecimal.ONE : BigDecimal.ZERO;
186-
187-
case Value.BYTE:
188-
return new BigDecimal(val.getByte());
189-
190-
case Value.SHORT:
191-
return new BigDecimal(val.getShort());
192-
193-
case Value.INT:
194-
return new BigDecimal(val.getInt());
195-
196-
case Value.LONG:
197-
return new BigDecimal(val.getLong());
198-
199-
case Value.DECIMAL:
200-
return val.getBigDecimal();
201-
202-
case Value.DOUBLE:
203-
return BigDecimal.valueOf(val.getDouble());
204-
205-
case Value.FLOAT:
206-
return BigDecimal.valueOf(val.getFloat());
207-
208-
case Value.DATE:
209-
return BigDecimal.valueOf(val.getDate().getTime());
210-
211-
case Value.TIME:
212-
return BigDecimal.valueOf(val.getTime().getTime());
213-
214-
case Value.TIMESTAMP:
215-
return BigDecimal.valueOf(val.getTimestamp().getTime());
216-
217-
case Value.BYTES:
218-
BigInteger bigInteger = new BigInteger(1, val.getBytes());
219-
return new BigDecimal(bigInteger);
220-
221-
case Value.STRING:
222-
case Value.STRING_FIXED:
223-
case Value.STRING_IGNORECASE:
224-
case Value.ARRAY:
225-
case Value.JAVA_OBJECT:
226-
case Value.GEOMETRY:
227-
return null;
228-
229-
case Value.UUID:
230-
BigInteger bigInt = new BigInteger(1, val.getBytes());
231-
return new BigDecimal(bigInt);
232-
233-
default:
234-
throw new IllegalStateException("Unsupported H2 type: " + val.getType());
235-
}
236-
}
237-
238167
/**
239168
* Predicate based selectivity for table. Estimate condition on each column taking in comparison it's statistics.
240169
*
@@ -395,16 +324,20 @@ private double estimateRefSelectivity(ProjectableFilterableTableScan rel, RelMet
395324
ColumnStatistics colStat = getColumnStatistics(mq, rel, ref);
396325
double res = 0.33;
397326

398-
if (colStat == null) {
327+
if (colStat == null || colStat.max() == null || colStat.min() == null) {
399328
// true, false and null with equivalent probability
400329
return res;
401330
}
402331

403-
if (colStat.max() == null || colStat.max().getType() != Value.BOOLEAN)
332+
// Check whether it can be considered as BOOL.
333+
boolean isBool = (colStat.max().compareTo(BigDecimal.ONE) == 0 || colStat.max().compareTo(BigDecimal.ZERO) == 0)
334+
&& (colStat.min().compareTo(BigDecimal.ONE) == 0 || colStat.min().compareTo(BigDecimal.ZERO) == 0);
335+
336+
if (!isBool)
404337
return res;
405338

406-
Boolean min = colStat.min().getBoolean();
407-
Boolean max = colStat.max().getBoolean();
339+
Boolean min = colStat.min().compareTo(BigDecimal.ONE) == 0;
340+
Boolean max = colStat.max().compareTo(BigDecimal.ONE) == 0;
408341

409342
if (!max)
410343
return 0;
@@ -450,8 +383,8 @@ private double estimateSelectivity(ColumnStatistics colStat, BigDecimal val, Rex
450383

451384
SqlOperator op = ((RexCall)pred).op;
452385

453-
BigDecimal min = toComparableValue(colStat.min());
454-
BigDecimal max = toComparableValue(colStat.max());
386+
BigDecimal min = colStat.min();
387+
BigDecimal max = colStat.max();
455388
BigDecimal total = (min == null || max == null) ? null : max.subtract(min).abs();
456389

457390
if (total == null)
@@ -537,13 +470,13 @@ private double estimateEqualsSelectivity(ColumnStatistics colStat, RexCall pred)
537470
return guessSelectivity(pred);
538471

539472
if (colStat.min() != null) {
540-
BigDecimal minComparable = toComparableValue(colStat.min());
473+
BigDecimal minComparable = colStat.min();
541474
if (minComparable != null && minComparable.compareTo(comparableVal) > 0)
542475
return 0.;
543476
}
544477

545478
if (colStat.max() != null) {
546-
BigDecimal maxComparable = toComparableValue(colStat.max());
479+
BigDecimal maxComparable = colStat.max();
547480
if (maxComparable != null && maxComparable.compareTo(comparableVal) < 0)
548481
return 0.;
549482
}

‎modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/QueryPlanCacheImpl.java‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@
1818
package org.apache.ignite.internal.processors.query.calcite.prepare;
1919

2020
import java.lang.reflect.Method;
21+
import java.util.List;
2122
import java.util.Map;
2223
import java.util.function.Supplier;
2324
import org.apache.ignite.internal.GridKernalContext;
2425
import org.apache.ignite.internal.cache.query.index.Index;
2526
import org.apache.ignite.internal.processors.cache.GridCacheContextInfo;
2627
import org.apache.ignite.internal.processors.query.GridQueryIndexDescriptor;
2728
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
29+
import org.apache.ignite.internal.processors.query.QueryField;
2830
import org.apache.ignite.internal.processors.query.calcite.util.AbstractService;
2931
import org.apache.ignite.internal.processors.query.schema.SchemaChangeListener;
3032
import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
@@ -100,7 +102,11 @@ public void subscriptionProcessor(GridInternalSubscriptionProcessor subscription
100102
}
101103

102104
/** {@inheritDoc} */
103-
@Override public void onSqlTypeDropped(String schemaName, GridQueryTypeDescriptor typeDescriptor) {
105+
@Override public void onSqlTypeDropped(
106+
String schemaName,
107+
GridQueryTypeDescriptor typeDescriptor,
108+
boolean destroy
109+
) {
104110
clear();
105111
}
106112

@@ -140,8 +146,22 @@ public void subscriptionProcessor(GridInternalSubscriptionProcessor subscription
140146
}
141147

142148
/** {@inheritDoc} */
143-
@Override public void onSqlTypeUpdated(String schemaName, GridQueryTypeDescriptor typeDesc,
144-
GridCacheContextInfo<?, ?> cacheInfo) {
149+
@Override public void onColumnsAdded(
150+
String schemaName,
151+
GridQueryTypeDescriptor typeDesc,
152+
GridCacheContextInfo<?, ?> cacheInfo,
153+
List<QueryField> cols
154+
) {
155+
clear();
156+
}
157+
158+
/** {@inheritDoc} */
159+
@Override public void onColumnsDropped(
160+
String schemaName,
161+
GridQueryTypeDescriptor typeDesc,
162+
GridCacheContextInfo<?, ?> cacheInfo,
163+
List<String> cols
164+
) {
145165
clear();
146166
}
147167

‎modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.ignite.internal.processors.cache.GridCacheContextInfo;
3838
import org.apache.ignite.internal.processors.query.GridQueryIndexDescriptor;
3939
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
40+
import org.apache.ignite.internal.processors.query.QueryField;
4041
import org.apache.ignite.internal.processors.query.calcite.exec.exp.IgniteScalarFunction;
4142
import org.apache.ignite.internal.processors.query.calcite.trait.TraitUtils;
4243
import org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeFactory;
@@ -188,10 +189,21 @@ public void subscriptionProcessor(GridInternalSubscriptionProcessor subscription
188189
}
189190

190191
/** {@inheritDoc} */
191-
@Override public void onSqlTypeUpdated(
192+
@Override public void onColumnsAdded(
192193
String schemaName,
193194
GridQueryTypeDescriptor typeDesc,
194-
GridCacheContextInfo<?, ?> cacheInfo
195+
GridCacheContextInfo<?, ?> cacheInfo,
196+
List<QueryField> cols
197+
) {
198+
onSqlTypeCreated(schemaName, typeDesc, cacheInfo);
199+
}
200+
201+
/** {@inheritDoc} */
202+
@Override public void onColumnsDropped(
203+
String schemaName,
204+
GridQueryTypeDescriptor typeDesc,
205+
GridCacheContextInfo<?, ?> cacheInfo,
206+
List<String> cols
195207
) {
196208
onSqlTypeCreated(schemaName, typeDesc, cacheInfo);
197209
}
@@ -206,7 +218,8 @@ private static Object affinityIdentity(CacheConfiguration<?, ?> ccfg) {
206218
/** {@inheritDoc} */
207219
@Override public synchronized void onSqlTypeDropped(
208220
String schemaName,
209-
GridQueryTypeDescriptor typeDesc
221+
GridQueryTypeDescriptor typeDesc,
222+
boolean destroy
210223
) {
211224
IgniteSchema schema = igniteSchemas.computeIfAbsent(schemaName, IgniteSchema::new);
212225

‎modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/StatisticsPlannerTest.java‎

Lines changed: 17 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -37,20 +37,11 @@
3737
import org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeSystem;
3838
import org.apache.ignite.internal.processors.query.stat.ColumnStatistics;
3939
import org.apache.ignite.internal.processors.query.stat.ObjectStatisticsImpl;
40-
import org.h2.value.ValueBoolean;
41-
import org.h2.value.ValueByte;
42-
import org.h2.value.ValueDate;
43-
import org.h2.value.ValueDouble;
44-
import org.h2.value.ValueFloat;
45-
import org.h2.value.ValueInt;
46-
import org.h2.value.ValueLong;
47-
import org.h2.value.ValueShort;
48-
import org.h2.value.ValueString;
49-
import org.h2.value.ValueTime;
50-
import org.h2.value.ValueTimestamp;
5140
import org.junit.Before;
5241
import org.junit.Test;
5342

43+
import static org.apache.ignite.internal.processors.query.stat.StatisticsUtils.toDecimal;
44+
5445
/**
5546
* Statistic related simple tests.
5647
*/
@@ -150,40 +141,35 @@ public class StatisticsPlannerTest extends AbstractPlannerTest {
150141
tbl4.addIndex("TBL4_SHORT_LONG", 6, 7);
151142

152143
HashMap<String, ColumnStatistics> colStat1 = new HashMap<>();
153-
colStat1.put("T1C1INT", new ColumnStatistics(ValueInt.get(1), ValueInt.get(1000),
154-
0, 1000, t1rc, 4, null, 1, 0));
144+
colStat1.put("T1C1INT", new ColumnStatistics(toDecimal(1), toDecimal(1000), 0, 1000, t1rc, 4, null, 1, 0));
155145

156-
colStat1.put("T1C2STR", new ColumnStatistics(ValueString.get("A1"), ValueString.get("Z9"),
157-
100, 20, t1rc, 2, null, 1, 0));
146+
colStat1.put("T1C2STR", new ColumnStatistics(null, null, 100, 20, t1rc, 2, null, 1, 0));
158147

159-
colStat1.put("T1C3DBL", new ColumnStatistics(ValueDouble.get(0.01), ValueDouble.get(0.99),
160-
10, 1000, t1rc, 8, null, 1, 0));
148+
colStat1.put("T1C3DBL", new ColumnStatistics(toDecimal(0.01), toDecimal(0.99), 10, 1000, t1rc, 8, null, 1, 0));
161149

162-
colStat1.put("T1C4BYTE", new ColumnStatistics(ValueByte.get((byte)0), ValueByte.get((byte)255),
163-
10, 1000, t1rc, 8, null, 1, 0));
150+
colStat1.put("T1C4BYTE", new ColumnStatistics(toDecimal((byte)0), toDecimal((byte)255), 10, 1000, t1rc, 8, null,
151+
1, 0));
164152

165-
colStat1.put("T1C5BOOLEAN", new ColumnStatistics(ValueBoolean.get(false), ValueBoolean.get(true),
166-
0, 2, t1rc, 1, null, 1, 0));
153+
colStat1.put("T1C5BOOLEAN", new ColumnStatistics(toDecimal(false), toDecimal(true), 0, 2, t1rc, 1, null, 1, 0));
167154

168-
colStat1.put("T1C6CHARACTER", new ColumnStatistics(ValueString.get("A"), ValueString.get("Z"),
169-
10, 10, t1rc, 1, null, 1, 0));
155+
colStat1.put("T1C6CHARACTER", new ColumnStatistics(null, null, 10, 10, t1rc, 1, null, 1, 0));
170156

171-
colStat1.put("T1C7SHORT", new ColumnStatistics(ValueShort.get((short)1), ValueShort.get((short)5000),
157+
colStat1.put("T1C7SHORT", new ColumnStatistics(toDecimal((short)1), toDecimal((short)5000),
172158
110, 500, t1rc, 2, null, 1, 0));
173159

174-
colStat1.put("T1C8LONG", new ColumnStatistics(ValueLong.get(1L), ValueLong.get(100000L),
160+
colStat1.put("T1C8LONG", new ColumnStatistics(toDecimal(1L), toDecimal(100000L),
175161
10, 100000, t1rc, 8, null, 1, 0));
176162

177-
colStat1.put("T1C9FLOAT", new ColumnStatistics(ValueFloat.get((float)0.1), ValueFloat.get((float)0.9),
163+
colStat1.put("T1C9FLOAT", new ColumnStatistics(toDecimal((float)0.1), toDecimal((float)0.9),
178164
10, 1000, t1rc, 8, null, 1, 0));
179165

180-
colStat1.put("T1C10DATE", new ColumnStatistics(ValueDate.get(MIN_DATE), ValueDate.get(MAX_DATE),
166+
colStat1.put("T1C10DATE", new ColumnStatistics(toDecimal(MIN_DATE), toDecimal(MAX_DATE),
181167
20, 1000, t1rc, 8, null, 1, 0));
182168

183-
colStat1.put("T1C11TIME", new ColumnStatistics(ValueTime.get(MIN_TIME), ValueTime.get(MAX_TIME),
169+
colStat1.put("T1C11TIME", new ColumnStatistics(toDecimal(MIN_TIME), toDecimal(MAX_TIME),
184170
10, 1000, t1rc, 8, null, 1, 0));
185171

186-
colStat1.put("T1C12TIMESTAMP", new ColumnStatistics(ValueTimestamp.get(MIN_TIMESTAMP), ValueTimestamp.get(MAX_TIMESTAMP),
172+
colStat1.put("T1C12TIMESTAMP", new ColumnStatistics(toDecimal(MIN_TIMESTAMP), toDecimal(MAX_TIMESTAMP),
187173
20, 1000, t1rc, 8, null, 1, 0));
188174

189175
tbl1stat = new IgniteStatisticsImpl(new ObjectStatisticsImpl(1000, colStat1));
@@ -437,10 +423,9 @@ public void testIndexWithBetterSelectivityPreferred() throws Exception {
437423
int rowCnt = 10_000;
438424

439425
HashMap<String, ColumnStatistics> colStat1 = new HashMap<>();
440-
colStat1.put("T1C2STR", new ColumnStatistics(ValueString.get("A1"), ValueString.get("Z9"),
441-
0, 1, rowCnt, 2, null, 1, 0));
426+
colStat1.put("T1C2STR", new ColumnStatistics(null, null, 0, 1, rowCnt, 2, null, 1, 0));
442427

443-
colStat1.put("T1C7SHORT", new ColumnStatistics(ValueShort.get((short)1), ValueShort.get((short)5000),
428+
colStat1.put("T1C7SHORT", new ColumnStatistics(toDecimal((short)1), toDecimal((short)5000),
444429
0, rowCnt, rowCnt, 2, null, 1, 0));
445430

446431
IgniteStatisticsImpl stat = new IgniteStatisticsImpl(new ObjectStatisticsImpl(1000, colStat1));

‎modules/codegen/src/main/java/org/apache/ignite/codegen/SystemViewRowAttributeWalkerGenerator.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -153,10 +153,10 @@ public static void main(String[] args) throws Exception {
153153
gen.generateAndWrite(SqlTableColumnView.class, INDEXING_SRC_DIR);
154154
gen.generateAndWrite(SqlViewColumnView.class, INDEXING_SRC_DIR);
155155

156-
gen.generateAndWrite(StatisticsColumnConfigurationView.class, INDEXING_SRC_DIR);
157-
gen.generateAndWrite(StatisticsColumnLocalDataView.class, INDEXING_SRC_DIR);
158-
gen.generateAndWrite(StatisticsColumnGlobalDataView.class, INDEXING_SRC_DIR);
159-
gen.generateAndWrite(StatisticsColumnPartitionDataView.class, INDEXING_SRC_DIR);
156+
gen.generateAndWrite(StatisticsColumnConfigurationView.class, DFLT_SRC_DIR);
157+
gen.generateAndWrite(StatisticsColumnLocalDataView.class, DFLT_SRC_DIR);
158+
gen.generateAndWrite(StatisticsColumnGlobalDataView.class, DFLT_SRC_DIR);
159+
gen.generateAndWrite(StatisticsColumnPartitionDataView.class, DFLT_SRC_DIR);
160160
}
161161

162162
/**

0 commit comments

Comments
 (0)