diff --git a/accumulo-handler/src/test/queries/positive/accumulo_queries.q b/accumulo-handler/src/test/queries/positive/accumulo_queries.q index ed2fce3812f4..e143f3607ead 100644 --- a/accumulo-handler/src/test/queries/positive/accumulo_queries.q +++ b/accumulo-handler/src/test/queries/positive/accumulo_queries.q @@ -1,6 +1,5 @@ --! qt:dataset:src -- remove these; after HIVE-18802 is fixed -set hive.optimize.index.filter=false; set hive.optimize.ppd=false; -- remove these; after HIVE-18802 is fixed diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 01f82922c9bb..602caeaa3a61 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2486,8 +2486,6 @@ public static enum ConfVars { HIVE_SCRIPT_OPERATOR_TRUST("hive.exec.script.trust", false, ""), // Optimizer - HIVE_OPT_INDEX_FILTER("hive.optimize.index.filter", true, "Whether to enable automatic use of indexes"), - HIVE_OPT_PPD("hive.optimize.ppd", true, "Whether to enable predicate pushdown"), HIVE_OPT_PPD_WINDOWING("hive.optimize.ppd.windowing", true, @@ -4790,7 +4788,7 @@ public static enum ConfVars { "hive.tez.bucket.pruning", true, "When pruning is enabled, filters on bucket columns will be processed by \n" + "filtering the splits against a bitset of included buckets. This needs predicates \n"+ - "produced by hive.optimize.ppd and hive.optimize.index.filter."), + "produced by hive.optimize.ppd."), TEZ_OPTIMIZE_BUCKET_PRUNING_COMPAT( "hive.tez.bucket.pruning.compat", true, "When pruning is enabled, handle possibly broken inserts due to negative hashcodes.\n" + @@ -6791,8 +6789,6 @@ private static String[] convertVarsToRegex(String[] paramList) { "hive\\.fetch.task\\..*", "hive\\.groupby\\..*", "hive\\.hbase\\..*", - "hive\\.index\\..*", - "hive\\.index\\..*", "hive\\.intermediate\\..*", "hive\\.jdbc\\..*", "hive\\.join\\..*", diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java index 4d1d61cc31a3..2e96e8e695c1 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java @@ -205,7 +205,7 @@ private Scan createFilterScan(JobConf jobConf, int iKey, int iTimestamp, boolean ExprNodeDesc residualPredicate = analyzer.analyzePredicate(filterExpr, conditions); // There should be no residual since we already negotiated that earlier in - // HBaseStorageHandler.decomposePredicate. However, with hive.optimize.index.filter + // HBaseStorageHandler.decomposePredicate. However, // OpProcFactory#pushFilterToStorageHandler pushes the original filter back down again. // Since pushed-down filters are not omitted at the higher levels (and thus the // contract of negotiation is ignored anyway), just ignore the residuals. diff --git a/hbase-handler/src/test/queries/positive/hbase_pushdown.q b/hbase-handler/src/test/queries/positive/hbase_pushdown.q index 74d5ad5f76b5..5831b55cc2d0 100644 --- a/hbase-handler/src/test/queries/positive/hbase_pushdown.q +++ b/hbase-handler/src/test/queries/positive/hbase_pushdown.q @@ -20,11 +20,9 @@ explain select * from hbase_pushdown where key=90 and value like '%90%'; select * from hbase_pushdown where key=90 and value like '%90%'; -set hive.optimize.index.filter=true; -- with partial pushdown with optimization (HIVE-6650) explain select * from hbase_pushdown where key=90 and value like '%90%'; select * from hbase_pushdown where key=90 and value like '%90%'; -set hive.optimize.index.filter=false; -- with two residuals diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/test/TestHiveShell.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/test/TestHiveShell.java index c6f134aaa3ea..332519e2f2b1 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/test/TestHiveShell.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/test/TestHiveShell.java @@ -199,7 +199,6 @@ private HiveConf initializeConf() { hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_ENABLED, true); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_INFER_BUCKET_SORT, false); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_METADATA_ONLY_QUERIES, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPT_INDEX_FILTER, false); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_CONVERT_JOIN, false); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SKEW_JOIN, false); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_pruning.q b/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_pruning.q index 8b469e26dc5d..3ef5c199c2eb 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_pruning.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_pruning.q @@ -7,7 +7,6 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.tez.min.bloom.filter.entries=1; set hive.tez.bigtable.minsize.semijoin.reduction=1; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/dynamic_semijoin_reduction.q b/iceberg/iceberg-handler/src/test/queries/positive/dynamic_semijoin_reduction.q index 035b5525f883..bcdf85ef22a3 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/dynamic_semijoin_reduction.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/dynamic_semijoin_reduction.q @@ -8,7 +8,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; @@ -35,4 +34,4 @@ EXPLAIN select count(*) from srcpart_date_n7 join srcpart_small_n3 on (srcpart_d select count(*) from srcpart_date_n7 join srcpart_small_n3 on (srcpart_date_n7.key = srcpart_small_n3.key1); drop table srcpart_date_n7; -drop table srcpart_small_n3; \ No newline at end of file +drop table srcpart_small_n3; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ExternalCache.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ExternalCache.java index 60d8136af68a..cf35c5429528 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ExternalCache.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ExternalCache.java @@ -98,8 +98,7 @@ public boolean hasPpd() { public void configure(HiveConf queryConfig) { this.conf = queryConfig; this.sarg = ConvertAstToSearchArg.createFromConf(conf); - this.isPpdEnabled = HiveConf.getBoolVar(conf, ConfVars.HIVE_OPT_INDEX_FILTER) - && HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_MS_FOOTER_CACHE_PPD); + this.isPpdEnabled = HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_MS_FOOTER_CACHE_PPD); this.isInTest = HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST); this.sargIsOriginal = this.sargNotIsOriginal = null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java index 5251ef0e5a93..dfef74bae4e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java @@ -47,7 +47,7 @@ public class LeafFilterFactory { @VisibleForTesting public static final String FILTER_PREDICATE_CONVERSION_NOT_SUPPORTED = - "The conversion to Parquet FilterPredicate is not supported for %s. Please try to set the following configurations at the session level\n set hive.optimize.index.filter=false;\n" + "The conversion to Parquet FilterPredicate is not supported for %s. Please try to set the following configuration at the session level\n" + " set hive.optimize.ppd=false;\n"; class IntFilterPredicateLeafBuilder extends FilterPredicateLeafBuilder { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java index 23f773647a73..8321fb75da73 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java @@ -110,7 +110,7 @@ protected void generatePredicate(NodeProcessorCtx procCtx, if (filter == null) { return; } - // the sargs are closely tied to hive.optimize.index.filter + // the sargs are closely tied to predicate push-down (hive.optimize.ppd) SearchArgument sarg = ConvertAstToSearchArg.create(ctxt.pctx.getConf(), filter); if (sarg == null) { return; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 8a56b8ab3dc6..55545896a199 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -172,8 +172,7 @@ public void initialize(HiveConf hiveConf) { if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.TEZ_OPTIMIZE_BUCKET_PRUNING) - && HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPT_PPD) - && HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPT_INDEX_FILTER)) { + && HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPT_PPD)) { final boolean compatMode = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.TEZ_OPTIMIZE_BUCKET_PRUNING_COMPAT); transformations.add(new FixedBucketPruningOptimizer(compatMode)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index 782220bebb92..773a8683f936 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -1327,12 +1327,10 @@ private static ExprNodeGenericFuncDesc pushFilterToStorageHandler( TableScanDesc tableScanDesc = tableScanOp.getConf(); Table tbl = tableScanDesc.getTableMetadata(); - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPT_INDEX_FILTER)) { - // attach the original predicate to the table scan operator for index - // optimizations that require the pushed predicate before pcr & later - // optimizations are applied - tableScanDesc.setFilterExpr(originalPredicate); - } + // Attach the original predicate to the table scan operator so that downstream + // storage-handler and predicate push-down consumers see the pushed predicate + // before pcr and later optimizations are applied. + tableScanDesc.setFilterExpr(originalPredicate); if (!tbl.isNonNative()) { return originalPredicate; } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index ecb048bba180..ad5c52dcf495 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -210,7 +210,6 @@ public void testOrcNoPPD() throws Exception { private void testOrcPPD(boolean enablePPD) throws Exception { hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPT_PPD, enablePPD);//enables PPD - hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPT_INDEX_FILTER, enablePPD);//enables ORC PPD //create delta_0001_0001_0000 (should push predicate here) runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(new int[][]{{1, 2}, {3, 4}})); List explain; @@ -1712,7 +1711,6 @@ protected void createAbortLowWaterMark() throws Exception{ @Test public void testETLSplitStrategyForACID() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY, "ETL"); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPT_INDEX_FILTER, true); runStatementOnDriver("insert into " + Table.ACIDTBL + " values(1,2)"); runStatementOnDriver("alter table " + Table.ACIDTBL + " compact 'MAJOR'"); runWorker(hiveConf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java index 2e87043d5c29..ae5d4a65aa60 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java @@ -700,7 +700,6 @@ private void setupExternalCacheConfig(boolean isPpd, String paths) { conf.setLong(HiveConf.ConfVars.MAPRED_MIN_SPLIT_SIZE.varname, 1000); conf.setLong(HiveConf.ConfVars.MAPRED_MAX_SPLIT_SIZE.varname, 5000); conf.setBoolean(ConfVars.HIVE_ORC_MS_FOOTER_CACHE_PPD.varname, isPpd); - conf.setBoolean(ConfVars.HIVE_OPT_INDEX_FILTER.varname, isPpd); } private ObjectInspector createIO() { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java index e9b9df2cc6e3..0ad97c83384d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java @@ -41,8 +41,8 @@ /** * These test the SARG implementation. - * The xml files were generated by setting hive.optimize.index.filter - * to true and using a custom record reader that prints out the value of + * The xml files were generated with predicate push-down enabled and using + * a custom record reader that prints out the value of * hive.io.filter.expr.serialized in createRecordReader. This should be * replaced by generating the AST using the API and passing that in. *

diff --git a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java index 32ce5ec6b78c..fb86db1b66dd 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java @@ -89,7 +89,7 @@ private List getSettableParams() throws SecurityException, NoSuchFieldEx // create list with variables that match some of the regexes List confVarRegexList = Arrays.asList("hive.convert.join.bucket.mapjoin.tez", - "hive.optimize.index.filter.compact.maxsize", "hive.tez.dummy", "tez.task.dummy", + "hive.tez.dummy", "tez.task.dummy", "hive.exec.dynamic.partition", "hive.exec.dynamic.partition.mode", "hive.exec.max.dynamic.partitions", "hive.exec.max.dynamic.partitions.pernode", "oozie.HadoopAccessorService.created", "tez.queue.name", diff --git a/ql/src/test/queries/clientpositive/acid_bucket_pruning.q b/ql/src/test/queries/clientpositive/acid_bucket_pruning.q index 476db6d4a4cd..be2040ad47b8 100644 --- a/ql/src/test/queries/clientpositive/acid_bucket_pruning.q +++ b/ql/src/test/queries/clientpositive/acid_bucket_pruning.q @@ -1,7 +1,6 @@ --! qt:dataset:alltypesorc set hive.mapred.mode=nonstrict; set hive.optimize.ppd=true; -set hive.optimize.index.filter=true; set hive.tez.bucket.pruning=true; set hive.explain.user=false; set hive.fetch.task.conversion=none; diff --git a/ql/src/test/queries/clientpositive/acid_vectorization_original.q b/ql/src/test/queries/clientpositive/acid_vectorization_original.q index 37d0176613cf..cab5e1af50ba 100644 --- a/ql/src/test/queries/clientpositive/acid_vectorization_original.q +++ b/ql/src/test/queries/clientpositive/acid_vectorization_original.q @@ -4,7 +4,6 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.vectorized.execution.enabled=true; -- enable ppd -set hive.optimize.index.filter=true; set hive.explain.user=false; @@ -172,4 +171,4 @@ set hive.exec.orc.split.strategy=BI; select t, count(*) from over10k_orc group by t; -select oo.ROW__ID.writeId, oo.ROW__IS__DELETED, oo.* from over10k_orc('acid.fetch.deleted.rows'='true') oo order by si; \ No newline at end of file +select oo.ROW__ID.writeId, oo.ROW__IS__DELETED, oo.* from over10k_orc('acid.fetch.deleted.rows'='true') oo order by si; diff --git a/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q b/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q index 8f5167226aeb..2e3331ab0114 100644 --- a/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q +++ b/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q @@ -4,7 +4,6 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.vectorized.execution.enabled=true; -- enable ppd -set hive.optimize.index.filter=true; set hive.explain.user=false; diff --git a/ql/src/test/queries/clientpositive/bucketmapjoin_with_subquery.q b/ql/src/test/queries/clientpositive/bucketmapjoin_with_subquery.q index dd7274014c8b..ea4c59305eb6 100644 --- a/ql/src/test/queries/clientpositive/bucketmapjoin_with_subquery.q +++ b/ql/src/test/queries/clientpositive/bucketmapjoin_with_subquery.q @@ -10,7 +10,6 @@ set hive.convert.join.bucket.mapjoin.tez=true; set hive.fetch.task.conversion=none; set hive.merge.nway.joins=false; set hive.optimize.dynamic.partition.hashjoin=true; -set hive.optimize.index.filter=true; set hive.optimize.remove.sq_count_check=true; set hive.prewarm.enabled=false; set hive.join.inner.residual=false; @@ -49,4 +48,4 @@ OVER(PARTITION BY id ORDER BY in_date DESC ) AS ROW_NUMB FROM DUP_TEST) OUTQUER select * from DUP_TEST_TARGET T join (SELECT id , in_date , sample FROM (SELECT id , in_date , sample ,ROW_NUMBER() OVER(PARTITION BY id ORDER BY in_date DESC ) AS ROW_NUMB FROM DUP_TEST) OUTQUERY WHERE ROW_NUMB =1) as S ON T.id = S.id; -select * from DUP_TEST_TARGET T join DUP_TEST S ON T.id = S.id; \ No newline at end of file +select * from DUP_TEST_TARGET T join DUP_TEST S ON T.id = S.id; diff --git a/ql/src/test/queries/clientpositive/bucketpruning1.q b/ql/src/test/queries/clientpositive/bucketpruning1.q index d867241e2df2..bd9718e3ad93 100644 --- a/ql/src/test/queries/clientpositive/bucketpruning1.q +++ b/ql/src/test/queries/clientpositive/bucketpruning1.q @@ -1,6 +1,5 @@ set hive.mapred.mode=nonstrict; set hive.optimize.ppd=true; -set hive.optimize.index.filter=true; set hive.tez.bucket.pruning=true; set hive.explain.user=false; set hive.fetch.task.conversion=none; diff --git a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q index 015b648b4cf9..5ebf29867bcf 100644 --- a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q +++ b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q @@ -4,7 +4,6 @@ set hive.vectorized.execution.enabled=false; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.tez.bucket.pruning=true; -set hive.optimize.index.filter=true; set hive.metastore.disallow.incompatible.col.type.changes=false; create table change_allowincompatible_vectorization_false_date (ts date) partitioned by (s string) clustered by (ts) into 32 buckets stored as orc tblproperties ('transactional'='true'); diff --git a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date2.q b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date2.q index 934f43e6b4b4..2319e388484b 100644 --- a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date2.q +++ b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date2.q @@ -4,7 +4,6 @@ set hive.vectorized.execution.enabled=false; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.tez.bucket.pruning=true; -set hive.optimize.index.filter=true; set hive.metastore.disallow.incompatible.col.type.changes=false; create table change_allowincompatible_vectorization_false_date (ts date) partitioned by (s string) clustered by (ts) into 32 buckets stored as orc tblproperties ('transactional'='true'); diff --git a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date3.q b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date3.q index 3b7024eb45f4..812ee2f223f1 100644 --- a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date3.q +++ b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date3.q @@ -4,7 +4,6 @@ set hive.vectorized.execution.enabled=false; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.tez.bucket.pruning=true; -set hive.optimize.index.filter=true; set hive.metastore.disallow.incompatible.col.type.changes=false; create table change_allowincompatible_vectorization_false_date (ts timestamp) partitioned by (s string) clustered by (ts) into 32 buckets stored as orc tblproperties ('transactional'='true'); diff --git a/ql/src/test/queries/clientpositive/dynamic_partition_join_noncbo.q b/ql/src/test/queries/clientpositive/dynamic_partition_join_noncbo.q index 90426f56e667..7e457779bc83 100644 --- a/ql/src/test/queries/clientpositive/dynamic_partition_join_noncbo.q +++ b/ql/src/test/queries/clientpositive/dynamic_partition_join_noncbo.q @@ -504,7 +504,6 @@ DROP TABLE t0; DROP TABLE t1; DROP TABLE t3; -set hive.optimize.index.filter=true; set hive.cbo.enable=false; set hive.auto.convert.sortmerge.join.to.mapjoin=false; diff --git a/ql/src/test/queries/clientpositive/dynamic_partition_pruning.q b/ql/src/test/queries/clientpositive/dynamic_partition_pruning.q index e7bfae40f50b..4a058ac83c06 100644 --- a/ql/src/test/queries/clientpositive/dynamic_partition_pruning.q +++ b/ql/src/test/queries/clientpositive/dynamic_partition_pruning.q @@ -7,7 +7,6 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.tez.min.bloom.filter.entries=1; set hive.tez.bigtable.minsize.semijoin.reduction=1; diff --git a/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q b/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q index f3d94d56d16b..e86a79078b34 100644 --- a/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q +++ b/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q @@ -5,7 +5,6 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.auto.convert.join=true; set hive.auto.convert.join.noconditionaltask = true; set hive.auto.convert.join.noconditionaltask.size = 10000000; diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q index f5a582464489..707d0fbdaee2 100644 --- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q @@ -8,7 +8,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q index 4d9c0375a387..2ab6078833e3 100644 --- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q @@ -7,7 +7,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; set hive.tez.dynamic.semijoin.reduction.threshold=-999999999999; diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_3.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_3.q index 80ee0baeb0aa..9f4eefde2df2 100644 --- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_3.q +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_3.q @@ -11,7 +11,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; set hive.tez.dynamic.semijoin.reduction.threshold=-999999999999; diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_4.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_4.q index 9b1723afeafc..320d958e23d2 100644 --- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_4.q +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_4.q @@ -7,7 +7,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw.q index e4ca1843628c..173823030863 100644 --- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw.q +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw.q @@ -8,7 +8,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw2.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw2.q index 910119d44736..6009d0fe89e9 100644 --- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw2.q +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_sw2.q @@ -8,7 +8,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_user_level.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_user_level.q index 5e3a1b84fbaa..bb2fae043dac 100644 --- a/ql/src/test/queries/clientpositive/dynamic_semijoin_user_level.q +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_user_level.q @@ -9,7 +9,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q index 86e0c11bb524..e06ed2453dfe 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q @@ -176,7 +176,6 @@ select count(*) from over1k_part_buck_sort2_orc; set hive.mapred.mode=nonstrict; set hive.optimize.ppd=true; -set hive.optimize.index.filter=true; set hive.tez.bucket.pruning=true; set hive.explain.user=false; set hive.fetch.task.conversion=none; diff --git a/ql/src/test/queries/clientpositive/explainanalyze_3.q b/ql/src/test/queries/clientpositive/explainanalyze_3.q index 72a2161d97f4..413e26f1c78b 100644 --- a/ql/src/test/queries/clientpositive/explainanalyze_3.q +++ b/ql/src/test/queries/clientpositive/explainanalyze_3.q @@ -116,7 +116,6 @@ load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merg SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SET mapred.min.split.size=1000; SET mapred.max.split.size=50000; -SET hive.optimize.index.filter=true; set hive.merge.orcfile.stripe.level=false; set hive.merge.tezfiles=false; set hive.merge.mapfiles=false; diff --git a/ql/src/test/queries/clientpositive/explainuser_3.q b/ql/src/test/queries/clientpositive/explainuser_3.q index c0d3d9108aea..39efca7049c1 100644 --- a/ql/src/test/queries/clientpositive/explainuser_3.q +++ b/ql/src/test/queries/clientpositive/explainuser_3.q @@ -114,7 +114,6 @@ load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merg SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SET mapred.min.split.size=1000; SET mapred.max.split.size=50000; -SET hive.optimize.index.filter=true; set hive.merge.orcfile.stripe.level=false; set hive.merge.tezfiles=false; set hive.merge.mapfiles=false; diff --git a/ql/src/test/queries/clientpositive/groupby_groupingset_bug.q b/ql/src/test/queries/clientpositive/groupby_groupingset_bug.q index 6c4ba331c781..1177559199e0 100644 --- a/ql/src/test/queries/clientpositive/groupby_groupingset_bug.q +++ b/ql/src/test/queries/clientpositive/groupby_groupingset_bug.q @@ -1,5 +1,4 @@ set hive.explain.user=true; -set hive.optimize.index.filter=true; set hive.auto.convert.join=true; set hive.vectorized.execution.enabled=true; diff --git a/ql/src/test/queries/clientpositive/llap_acid.q b/ql/src/test/queries/clientpositive/llap_acid.q index 10a3e73afa7e..099944c9968a 100644 --- a/ql/src/test/queries/clientpositive/llap_acid.q +++ b/ql/src/test/queries/clientpositive/llap_acid.q @@ -6,7 +6,6 @@ SET hive.llap.io.enabled=false; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.fetch.task.conversion=none; set hive.support.concurrency=true; diff --git a/ql/src/test/queries/clientpositive/llap_acid2.q b/ql/src/test/queries/clientpositive/llap_acid2.q index f8945883657f..3b2a3db2fddc 100644 --- a/ql/src/test/queries/clientpositive/llap_acid2.q +++ b/ql/src/test/queries/clientpositive/llap_acid2.q @@ -6,7 +6,6 @@ SET hive.llap.io.enabled=false; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.fetch.task.conversion=none; set hive.support.concurrency=true; diff --git a/ql/src/test/queries/clientpositive/llap_acid_fast.q b/ql/src/test/queries/clientpositive/llap_acid_fast.q index eac8b387d17c..f199d4ec4468 100644 --- a/ql/src/test/queries/clientpositive/llap_acid_fast.q +++ b/ql/src/test/queries/clientpositive/llap_acid_fast.q @@ -6,7 +6,6 @@ SET hive.llap.io.enabled=true; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.fetch.task.conversion=none; set hive.support.concurrency=true; diff --git a/ql/src/test/queries/clientpositive/llap_decimal64_reader.q b/ql/src/test/queries/clientpositive/llap_decimal64_reader.q index a81febac2547..8ee328e84a22 100644 --- a/ql/src/test/queries/clientpositive/llap_decimal64_reader.q +++ b/ql/src/test/queries/clientpositive/llap_decimal64_reader.q @@ -4,7 +4,6 @@ SET hive.vectorized.execution.enabled=true; SET hive.llap.io.enabled=false; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.auto.convert.join=false; DROP TABLE orc_llap_n0; diff --git a/ql/src/test/queries/clientpositive/llap_partitioned.q b/ql/src/test/queries/clientpositive/llap_partitioned.q index b9d1a301a0aa..10971a81ab91 100644 --- a/ql/src/test/queries/clientpositive/llap_partitioned.q +++ b/ql/src/test/queries/clientpositive/llap_partitioned.q @@ -6,7 +6,6 @@ SET hive.llap.io.enabled=false; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.auto.convert.join=true; set hive.exec.dynamic.partition=true; diff --git a/ql/src/test/queries/clientpositive/llap_smb.q b/ql/src/test/queries/clientpositive/llap_smb.q index e5207c1b08c3..ddfa1933be25 100644 --- a/ql/src/test/queries/clientpositive/llap_smb.q +++ b/ql/src/test/queries/clientpositive/llap_smb.q @@ -8,7 +8,6 @@ SET hive.vectorized.execution.enabled=true; SET hive.llap.io.enabled=true; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.fetch.task.conversion=none; diff --git a/ql/src/test/queries/clientpositive/llap_stats.q b/ql/src/test/queries/clientpositive/llap_stats.q index 2692f77b4440..3641b9e8e1cf 100644 --- a/ql/src/test/queries/clientpositive/llap_stats.q +++ b/ql/src/test/queries/clientpositive/llap_stats.q @@ -6,7 +6,6 @@ SET hive.llap.io.enabled=false; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.auto.convert.join=false; DROP TABLE llap_stats; diff --git a/ql/src/test/queries/clientpositive/llap_uncompressed.q b/ql/src/test/queries/clientpositive/llap_uncompressed.q index de3cdc600ec0..864422cc4989 100644 --- a/ql/src/test/queries/clientpositive/llap_uncompressed.q +++ b/ql/src/test/queries/clientpositive/llap_uncompressed.q @@ -4,7 +4,6 @@ SET hive.vectorized.execution.enabled=true; SET hive.llap.io.enabled=false; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.auto.convert.join=false; DROP TABLE orc_llap_n0; diff --git a/ql/src/test/queries/clientpositive/load_data_using_job.q b/ql/src/test/queries/clientpositive/load_data_using_job.q index a328ff9d3f21..d41d095ec70b 100644 --- a/ql/src/test/queries/clientpositive/load_data_using_job.q +++ b/ql/src/test/queries/clientpositive/load_data_using_job.q @@ -98,7 +98,6 @@ drop table srcbucket_mapjoin_n8; -- Load into ACID table using ORC files set hive.mapred.mode=nonstrict; set hive.optimize.ppd=true; -set hive.optimize.index.filter=true; set hive.tez.bucket.pruning=true; set hive.explain.user=false; set hive.fetch.task.conversion=none; diff --git a/ql/src/test/queries/clientpositive/mapjoin_hint.q b/ql/src/test/queries/clientpositive/mapjoin_hint.q index 1723a19fdbec..0978eb9d72c3 100644 --- a/ql/src/test/queries/clientpositive/mapjoin_hint.q +++ b/ql/src/test/queries/clientpositive/mapjoin_hint.q @@ -8,7 +8,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; diff --git a/ql/src/test/queries/clientpositive/mergejoin.q b/ql/src/test/queries/clientpositive/mergejoin.q index 0da7eee61c05..6a1d624fbc9b 100644 --- a/ql/src/test/queries/clientpositive/mergejoin.q +++ b/ql/src/test/queries/clientpositive/mergejoin.q @@ -12,7 +12,6 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.vectorized.execution.enabled=true; set hive.tez.min.bloom.filter.entries=1; set hive.tez.bigtable.minsize.semijoin.reduction=1; diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q index 4fd8cf16dcb6..de0adbb4bc6f 100644 --- a/ql/src/test/queries/clientpositive/mm_all.q +++ b/ql/src/test/queries/clientpositive/mm_all.q @@ -299,7 +299,6 @@ drop table skewjoin_mm; set hive.optimize.skewjoin=false; -set hive.optimize.index.filter=true; set hive.auto.convert.join=false; CREATE TABLE parquet1_mm(id INT) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only"); INSERT INTO parquet1_mm VALUES(1), (2); diff --git a/ql/src/test/queries/clientpositive/murmur_hash_migration2.q b/ql/src/test/queries/clientpositive/murmur_hash_migration2.q index 362ead71aeb7..29d134fcab93 100644 --- a/ql/src/test/queries/clientpositive/murmur_hash_migration2.q +++ b/ql/src/test/queries/clientpositive/murmur_hash_migration2.q @@ -11,7 +11,6 @@ set hive.auto.convert.join.noconditionaltask.size=30000; set hive.optimize.bucketingsorting=false; set hive.optimize.ppd=true; -set hive.optimize.index.filter=true; set hive.tez.bucket.pruning=true; set hive.fetch.task.conversion=none; set hive.support.concurrency=true; diff --git a/ql/src/test/queries/clientpositive/optimize_filter_literal.q b/ql/src/test/queries/clientpositive/optimize_filter_literal.q index b0ce4ae31e39..9a8f08b90bf3 100644 --- a/ql/src/test/queries/clientpositive/optimize_filter_literal.q +++ b/ql/src/test/queries/clientpositive/optimize_filter_literal.q @@ -7,7 +7,6 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.vectorized.execution.enabled=true; -- SORT_QUERY_RESULTS diff --git a/ql/src/test/queries/clientpositive/orc_create.q b/ql/src/test/queries/clientpositive/orc_create.q index eba39f4fafad..a2ba29dcee3f 100644 --- a/ql/src/test/queries/clientpositive/orc_create.q +++ b/ql/src/test/queries/clientpositive/orc_create.q @@ -99,7 +99,6 @@ STORED AS orc; INSERT OVERWRITE TABLE orc_create_people PARTITION (state) SELECT * FROM orc_create_people_staging; -SET hive.optimize.index.filter=true; -- test predicate push down with partition pruning SELECT COUNT(*) FROM orc_create_people where id < 10 and state = 'Ca'; diff --git a/ql/src/test/queries/clientpositive/orc_llap.q b/ql/src/test/queries/clientpositive/orc_llap.q index 4ebc7414dae0..1461725f2bae 100644 --- a/ql/src/test/queries/clientpositive/orc_llap.q +++ b/ql/src/test/queries/clientpositive/orc_llap.q @@ -7,7 +7,6 @@ SET hive.llap.io.enabled=false; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.auto.convert.join=false; set hive.llap.memory.oversubscription.max.executors.per.query=3; diff --git a/ql/src/test/queries/clientpositive/orc_llap_counters.q b/ql/src/test/queries/clientpositive/orc_llap_counters.q index 584f6cbd0e5b..fcb6c9b85578 100644 --- a/ql/src/test/queries/clientpositive/orc_llap_counters.q +++ b/ql/src/test/queries/clientpositive/orc_llap_counters.q @@ -2,7 +2,6 @@ set hive.vectorized.execution.enabled=false; set hive.compute.query.using.stats=false; set hive.mapred.mode=nonstrict; -SET hive.optimize.index.filter=true; SET hive.cbo.enable=false; SET hive.vectorized.execution.enabled=true; SET hive.llap.io.enabled=true; diff --git a/ql/src/test/queries/clientpositive/orc_llap_counters1.q b/ql/src/test/queries/clientpositive/orc_llap_counters1.q index d7861fe95d1b..dcd68571e1ad 100644 --- a/ql/src/test/queries/clientpositive/orc_llap_counters1.q +++ b/ql/src/test/queries/clientpositive/orc_llap_counters1.q @@ -1,6 +1,5 @@ set hive.vectorized.execution.enabled=false; set hive.mapred.mode=nonstrict; -SET hive.optimize.index.filter=true; SET hive.cbo.enable=false; SET hive.vectorized.execution.enabled=true; SET hive.llap.io.enabled=true; diff --git a/ql/src/test/queries/clientpositive/orc_llap_nonvector.q b/ql/src/test/queries/clientpositive/orc_llap_nonvector.q index 31a3d63c58ac..c2733bfeca6c 100644 --- a/ql/src/test/queries/clientpositive/orc_llap_nonvector.q +++ b/ql/src/test/queries/clientpositive/orc_llap_nonvector.q @@ -9,7 +9,6 @@ SET hive.llap.io.enabled=false; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.auto.convert.join=false; set hive.fetch.task.conversion=none; diff --git a/ql/src/test/queries/clientpositive/orc_merge5.q b/ql/src/test/queries/clientpositive/orc_merge5.q index 1b42978e72b8..193d2ee8e455 100644 --- a/ql/src/test/queries/clientpositive/orc_merge5.q +++ b/ql/src/test/queries/clientpositive/orc_merge5.q @@ -11,7 +11,6 @@ load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merg SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SET mapred.min.split.size=1000; SET mapred.max.split.size=50000; -SET hive.optimize.index.filter=true; set hive.merge.orcfile.stripe.level=false; set hive.merge.tezfiles=false; set hive.merge.mapfiles=false; diff --git a/ql/src/test/queries/clientpositive/orc_merge6.q b/ql/src/test/queries/clientpositive/orc_merge6.q index 1ded245c513c..7a729761b668 100644 --- a/ql/src/test/queries/clientpositive/orc_merge6.q +++ b/ql/src/test/queries/clientpositive/orc_merge6.q @@ -12,7 +12,6 @@ load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merg SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SET mapred.min.split.size=1000; SET mapred.max.split.size=50000; -SET hive.optimize.index.filter=true; set hive.merge.orcfile.stripe.level=false; set hive.merge.tezfiles=false; set hive.merge.mapfiles=false; diff --git a/ql/src/test/queries/clientpositive/orc_merge7.q b/ql/src/test/queries/clientpositive/orc_merge7.q index 7a70c0335b60..de448f693e7b 100644 --- a/ql/src/test/queries/clientpositive/orc_merge7.q +++ b/ql/src/test/queries/clientpositive/orc_merge7.q @@ -13,7 +13,6 @@ load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merg SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SET mapred.min.split.size=1000; SET mapred.max.split.size=50000; -SET hive.optimize.index.filter=true; set hive.merge.orcfile.stripe.level=false; set hive.merge.tezfiles=false; set hive.merge.mapfiles=false; diff --git a/ql/src/test/queries/clientpositive/orc_merge8.q b/ql/src/test/queries/clientpositive/orc_merge8.q index 6679a0f18463..52ca70844d71 100644 --- a/ql/src/test/queries/clientpositive/orc_merge8.q +++ b/ql/src/test/queries/clientpositive/orc_merge8.q @@ -27,7 +27,6 @@ alter table alltypes_orc_n1 set fileformat orc; load data local inpath '../../data/files/alltypes2.txt' overwrite into table alltypes_n1; SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; -SET hive.optimize.index.filter=true; set hive.merge.orcfile.stripe.level=false; set hive.merge.tezfiles=false; set hive.merge.mapfiles=false; diff --git a/ql/src/test/queries/clientpositive/orc_merge_incompat2.q b/ql/src/test/queries/clientpositive/orc_merge_incompat2.q index 3773034a2f18..819bc1249dc9 100644 --- a/ql/src/test/queries/clientpositive/orc_merge_incompat2.q +++ b/ql/src/test/queries/clientpositive/orc_merge_incompat2.q @@ -14,7 +14,6 @@ load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merg SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SET mapred.min.split.size=1000; SET mapred.max.split.size=50000; -SET hive.optimize.index.filter=true; set hive.merge.orcfile.stripe.level=false; set hive.merge.tezfiles=false; set hive.merge.mapfiles=false; diff --git a/ql/src/test/queries/clientpositive/orc_ppd_basic.q b/ql/src/test/queries/clientpositive/orc_ppd_basic.q index d21c6d935289..3c00486a96a1 100644 --- a/ql/src/test/queries/clientpositive/orc_ppd_basic.q +++ b/ql/src/test/queries/clientpositive/orc_ppd_basic.q @@ -6,7 +6,6 @@ SET hive.vectorized.execution.enabled=false; set hive.compute.query.using.stats=false; set hive.mapred.mode=nonstrict; SET hive.fetch.task.conversion=none; -SET hive.optimize.index.filter=true; SET hive.cbo.enable=false; SET hive.map.aggr=false; -- disabling map side aggregation as that can lead to different intermediate record counts @@ -187,40 +186,31 @@ select count(*) from orc_ppd_n2 where s = "wendy king" and t < 0; select count(*) from orc_ppd_n2 where s = "wendy king" and t > 100; set hive.cbo.enable=false; -set hive.optimize.index.filter=false; -- when cbo is disabled constant gets converted to HiveDecimal -- 74.72f + 0.0 = 74.72000122070312 select count(*) from orc_ppd_n2 where f=74.72; -set hive.optimize.index.filter=true; select count(*) from orc_ppd_n2 where f=74.72; set hive.cbo.enable=true; -set hive.optimize.index.filter=false; select count(*) from orc_ppd_n2 where f=74.72; -set hive.optimize.index.filter=true; select count(*) from orc_ppd_n2 where f=74.72; -- 42.47f + 0.0 == 42.470001220703125 create temporary table orc_ppd_1 stored as orc as select * from orc_ppd_staging_n1 where d = 42.47; set hive.cbo.enable=false; -set hive.optimize.index.filter=false; -- when cbo is disabled constant gets converted to HiveDecimal select count(*) from orc_ppd_1 where d=42.47; -set hive.optimize.index.filter=true; select count(*) from orc_ppd_1 where d=42.47; set hive.cbo.enable=true; -set hive.optimize.index.filter=false; select count(*) from orc_ppd_1 where d=42.47; -set hive.optimize.index.filter=true; select count(*) from orc_ppd_1 where d=42.47; RESET; set hive.compute.query.using.stats=false; set hive.mapred.mode=nonstrict; SET hive.fetch.task.conversion=none; -SET hive.optimize.index.filter=true; SET hive.cbo.enable=false; SET hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecOrcRowGroupCountPrinter; -- these tests include timestamp column that will impact the file size when tests run across @@ -256,7 +246,6 @@ drop table if exists tbl_orcppd_2_1; create table tbl_orcppd_2_1 as select count(*) from tmp_orcppd where ctimestamp1 in (cast('2065-08-13 19:03:52' as timestamp), cast('2071-01-16 20:21:17' as timestamp), current_timestamp()); -set hive.optimize.index.filter=true; drop table if exists tmp_orcppd; create temporary table tmp_orcppd diff --git a/ql/src/test/queries/clientpositive/orc_ppd_boolean.q b/ql/src/test/queries/clientpositive/orc_ppd_boolean.q index 4d4f56ddb0d2..7928347638a8 100644 --- a/ql/src/test/queries/clientpositive/orc_ppd_boolean.q +++ b/ql/src/test/queries/clientpositive/orc_ppd_boolean.q @@ -10,29 +10,21 @@ create table newtypesorc_n0(c char(10), v varchar(10), d decimal(5,3), b boolean insert overwrite table newtypesorc_n0 select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, true from src src1 union all select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, false from src src2) uniontbl; -set hive.optimize.index.filter=false; -- char data types (EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_EQUALS, IN, BETWEEN tests) select sum(hash(*)) from newtypesorc_n0 where b=true; -set hive.optimize.index.filter=true; select sum(hash(*)) from newtypesorc_n0 where b=false; -set hive.optimize.index.filter=false; select sum(hash(*)) from newtypesorc_n0 where b!=true; -set hive.optimize.index.filter=true; select sum(hash(*)) from newtypesorc_n0 where b!=false; -set hive.optimize.index.filter=false; select sum(hash(*)) from newtypesorc_n0 where b 127; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > 127; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = 55; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = 55; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = 54; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = 54; alter table orc_ppd_n3 change column t t smallint; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > 127; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > 127; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = 55; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = 55; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = 54; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = 54; alter table orc_ppd_n3 change column t t int; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > 127; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > 127; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = 55; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = 55; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = 54; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = 54; alter table orc_ppd_n3 change column t t bigint; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > 127; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > 127; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = 55; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = 55; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = 54; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = 54; alter table orc_ppd_n3 change column t t string; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > '127'; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 0 (no row groups) select count(*) from orc_ppd_n3 where t > '127'; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = '55'; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 1000 (1 row group) select count(*) from orc_ppd_n3 where t = '55'; -SET hive.optimize.index.filter=false; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = '54'; -SET hive.optimize.index.filter=true; -- INPUT_RECORDS: 2000 (2 row groups) select count(*) from orc_ppd_n3 where t = '54'; -SET hive.optimize.index.filter=false; -- float tests select count(*) from orc_ppd_n3 where f = 74.72; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where f = 74.72; alter table orc_ppd_n3 change column f f double; -SET hive.optimize.index.filter=false; select count(*) from orc_ppd_n3 where f = 74.72000122070312; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where f = 74.72000122070312; alter table orc_ppd_n3 change column f f string; -SET hive.optimize.index.filter=false; select count(*) from orc_ppd_n3 where f = '74.72000122070312'; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where f = '74.72000122070312'; -SET hive.optimize.index.filter=false; -- string tests select count(*) from orc_ppd_n3 where s = 'bob davidson'; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where s = 'bob davidson'; alter table orc_ppd_n3 change column s s char(50); -SET hive.optimize.index.filter=false; select count(*) from orc_ppd_n3 where s = 'bob davidson'; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where s = 'bob davidson'; alter table orc_ppd_n3 change column s s varchar(50); -SET hive.optimize.index.filter=false; select count(*) from orc_ppd_n3 where s = 'bob davidson'; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where s = 'bob davidson'; alter table orc_ppd_n3 change column s s char(50); -SET hive.optimize.index.filter=false; select count(*) from orc_ppd_n3 where s = 'bob davidson'; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where s = 'bob davidson'; alter table orc_ppd_n3 change column s s string; -SET hive.optimize.index.filter=false; select count(*) from orc_ppd_n3 where s = 'bob davidson'; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where s = 'bob davidson'; alter table orc_ppd_n3 add columns (boo boolean); -SET hive.optimize.index.filter=false; -- ppd on newly added column select count(*) from orc_ppd_n3 where si = 442; select count(*) from orc_ppd_n3 where si = 442 or boo is not null or boo = false; -SET hive.optimize.index.filter=true; select count(*) from orc_ppd_n3 where si = 442; select count(*) from orc_ppd_n3 where si = 442 or boo is not null or boo = false; diff --git a/ql/src/test/queries/clientpositive/orc_ppd_str_conversion.q b/ql/src/test/queries/clientpositive/orc_ppd_str_conversion.q index 61f4cfaae947..14df6f11821d 100644 --- a/ql/src/test/queries/clientpositive/orc_ppd_str_conversion.q +++ b/ql/src/test/queries/clientpositive/orc_ppd_str_conversion.q @@ -10,9 +10,7 @@ insert overwrite table text_test select * from orc_test_n0; explain select * from text_test where col2='1'; select * from text_test where col2='1'; -set hive.optimize.index.filter=false; select * from orc_test_n0 where col2='1'; -set hive.optimize.index.filter=true; select * from orc_test_n0 where col2='1'; diff --git a/ql/src/test/queries/clientpositive/orc_ppd_timestamp.q b/ql/src/test/queries/clientpositive/orc_ppd_timestamp.q index 9719270edb63..7988648f6f9d 100644 --- a/ql/src/test/queries/clientpositive/orc_ppd_timestamp.q +++ b/ql/src/test/queries/clientpositive/orc_ppd_timestamp.q @@ -13,89 +13,60 @@ insert overwrite table newtypesorc_n2 select * from (select cast("apple" as char -- timestamp data types (EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_EQUALS, IN, BETWEEN tests) select sum(hash(*)) from newtypesorc_n2 where cast(ts as string)='2011-01-01 01:01:01'; -set hive.optimize.index.filter=true; select sum(hash(*)) from newtypesorc_n2 where cast(ts as string)='2011-01-01 01:01:01'; -set hive.optimize.index.filter=false; select sum(hash(*)) from newtypesorc_n2 where ts=cast('2011-01-01 01:01:01' as timestamp); -set hive.optimize.index.filter=true; select sum(hash(*)) from newtypesorc_n2 where ts=cast('2011-01-01 01:01:01' as timestamp); -set hive.optimize.index.filter=false; select sum(hash(*)) from newtypesorc_n2 where ts=cast('2011-01-01 01:01:01' as varchar(20)); -set hive.optimize.index.filter=true; select sum(hash(*)) from newtypesorc_n2 where ts=cast('2011-01-01 01:01:01' as varchar(20)); -set hive.optimize.index.filter=false; select sum(hash(*)) from newtypesorc_n2 where ts!=cast('2011-01-01 01:01:01' as timestamp); -set hive.optimize.index.filter=true; select sum(hash(*)) from newtypesorc_n2 where ts!=cast('2011-01-01 01:01:01' as timestamp); -set hive.optimize.index.filter=false; select sum(hash(*)) from newtypesorc_n2 where ts2 limit 1; -SET hive.optimize.index.filter=true; SELECT * FROM orc_pred WHERE t>2 limit 1; -SET hive.optimize.index.filter=false; SELECT SUM(HASH(t)) FROM orc_pred WHERE t IS NOT NULL AND t < 0 AND t > -2; -SET hive.optimize.index.filter=true; SELECT SUM(HASH(t)) FROM orc_pred WHERE t IS NOT NULL AND t < 0 AND t > -2; -SET hive.optimize.index.filter=false; EXPLAIN SELECT SUM(HASH(t)) FROM orc_pred WHERE t IS NOT NULL AND t < 0 AND t > -2; -SET hive.optimize.index.filter=true; EXPLAIN SELECT SUM(HASH(t)) FROM orc_pred WHERE t IS NOT NULL AND t < 0 AND t > -2; -SET hive.optimize.index.filter=false; SELECT t, s FROM orc_pred WHERE t <=> -1 @@ -94,13 +82,11 @@ SELECT t, s FROM orc_pred AND s LIKE 'bob%' ; -SET hive.optimize.index.filter=true; SELECT t, s FROM orc_pred WHERE t <=> -1 AND s IS NOT NULL AND s LIKE 'bob%' ; -SET hive.optimize.index.filter=false; EXPLAIN SELECT t, s FROM orc_pred WHERE t <=> -1 @@ -108,13 +94,11 @@ EXPLAIN SELECT t, s FROM orc_pred AND s LIKE 'bob%' ; -SET hive.optimize.index.filter=true; EXPLAIN SELECT t, s FROM orc_pred WHERE t <=> -1 AND s IS NOT NULL AND s LIKE 'bob%' ; -SET hive.optimize.index.filter=false; SELECT t, s FROM orc_pred WHERE s IS NOT NULL @@ -123,14 +107,12 @@ SELECT t, s FROM orc_pred AND t BETWEEN 25 AND 30 SORT BY t,s; -set hive.optimize.index.filter=true; SELECT t, s FROM orc_pred WHERE s IS NOT NULL AND s LIKE 'bob%' AND t NOT IN (-1,-2,-3) AND t BETWEEN 25 AND 30 SORT BY t,s; -set hive.optimize.index.filter=false; EXPLAIN SELECT t, s FROM orc_pred WHERE s IS NOT NULL @@ -139,14 +121,12 @@ EXPLAIN SELECT t, s FROM orc_pred AND t BETWEEN 25 AND 30 SORT BY t,s; -SET hive.optimize.index.filter=true; EXPLAIN SELECT t, s FROM orc_pred WHERE s IS NOT NULL AND s LIKE 'bob%' AND t NOT IN (-1,-2,-3) AND t BETWEEN 25 AND 30 SORT BY t,s; -SET hive.optimize.index.filter=false; SELECT t, si, d, s FROM orc_pred WHERE d >= ROUND(9.99) @@ -159,7 +139,6 @@ SELECT t, si, d, s FROM orc_pred ORDER BY s DESC LIMIT 3; -SET hive.optimize.index.filter=true; SELECT t, si, d, s FROM orc_pred WHERE d >= ROUND(9.99) AND d < 12 @@ -170,7 +149,6 @@ SELECT t, si, d, s FROM orc_pred AND si BETWEEN 300 AND 400 ORDER BY s DESC LIMIT 3; -SET hive.optimize.index.filter=false; EXPLAIN SELECT t, si, d, s FROM orc_pred WHERE d >= ROUND(9.99) @@ -183,7 +161,6 @@ EXPLAIN SELECT t, si, d, s FROM orc_pred ORDER BY s DESC LIMIT 3; -SET hive.optimize.index.filter=true; EXPLAIN SELECT t, si, d, s FROM orc_pred WHERE d >= ROUND(9.99) AND d < 12 @@ -194,7 +171,6 @@ EXPLAIN SELECT t, si, d, s FROM orc_pred AND si BETWEEN 300 AND 400 ORDER BY s DESC LIMIT 3; -SET hive.optimize.index.filter=false; SELECT t, si, d, s FROM orc_pred WHERE t > 10 @@ -209,7 +185,6 @@ SELECT t, si, d, s FROM orc_pred SORT BY s DESC LIMIT 3; -SET hive.optimize.index.filter=true; SELECT t, si, d, s FROM orc_pred WHERE t > 10 AND t <> 101 @@ -222,7 +197,6 @@ SELECT t, si, d, s FROM orc_pred AND si BETWEEN 300 AND 400 SORT BY s DESC LIMIT 3; -SET hive.optimize.index.filter=false; EXPLAIN SELECT t, si, d, s FROM orc_pred WHERE t > 10 @@ -237,7 +211,6 @@ EXPLAIN SELECT t, si, d, s FROM orc_pred SORT BY s DESC LIMIT 3; -SET hive.optimize.index.filter=true; EXPLAIN SELECT t, si, d, s FROM orc_pred WHERE t > 10 AND t <> 101 @@ -250,4 +223,3 @@ EXPLAIN SELECT t, si, d, s FROM orc_pred AND si BETWEEN 300 AND 400 SORT BY s DESC LIMIT 3; -SET hive.optimize.index.filter=false; diff --git a/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q b/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q index 0c4850447390..330c75149f0f 100644 --- a/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q +++ b/ql/src/test/queries/clientpositive/orc_schema_evolution_float.q @@ -1,5 +1,4 @@ set hive.vectorized.execution.enabled=false; -set hive.optimize.index.filter=false; set hive.metastore.disallow.incompatible.col.type.changes=false; -- SORT_QUERY_RESULTS @@ -24,7 +23,6 @@ alter table float_orc change column f f double; select f from float_orc; select f from float_orc where f=74.72000122070312; select f from float_orc where f=0.2199999988079071; -set hive.optimize.index.filter=true; select f from float_orc where f=74.72000122070312; select f from float_orc where f=0.2199999988079071; @@ -32,7 +30,6 @@ alter table float_orc change column f f decimal(14,5); select f from float_orc; select f from float_orc where f=74.72; select f from float_orc where f=0.22; -set hive.optimize.index.filter=true; select f from float_orc where f=74.72; select f from float_orc where f=0.22; diff --git a/ql/src/test/queries/clientpositive/orc_split_elimination.q b/ql/src/test/queries/clientpositive/orc_split_elimination.q index 03e0e73730ec..fddb511e5992 100644 --- a/ql/src/test/queries/clientpositive/orc_split_elimination.q +++ b/ql/src/test/queries/clientpositive/orc_split_elimination.q @@ -9,7 +9,6 @@ load data local inpath '../../data/files/orc_split_elim.orc' into table orc_spli SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; SET mapred.min.split.size=1000; SET mapred.max.split.size=5000; -SET hive.optimize.index.filter=false; -- The above table will have 5 splits with the followings stats -- Stripe 1: @@ -51,58 +50,44 @@ SET hive.optimize.index.filter=false; -- 5 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=0; -SET hive.optimize.index.filter=true; -- 0 mapper select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=0; -SET hive.optimize.index.filter=false; -- 5 mappers. count should be 0 select count(*) from orc_split_elim where userid<=0; -SET hive.optimize.index.filter=true; -- 0 mapper select count(*) from orc_split_elim where userid<=0; -SET hive.optimize.index.filter=false; -- 5 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=2; -SET hive.optimize.index.filter=true; -- 1 mapper select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=2; -SET hive.optimize.index.filter=false; -- 5 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=5; -SET hive.optimize.index.filter=true; -- 2 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=5; -SET hive.optimize.index.filter=false; -- 5 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=13; -SET hive.optimize.index.filter=true; -- 3 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=13; -SET hive.optimize.index.filter=false; -- 5 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=29; -SET hive.optimize.index.filter=true; -- 4 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=29; -SET hive.optimize.index.filter=false; -- 5 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=70; -SET hive.optimize.index.filter=true; -- 5 mappers select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=70; -SET hive.optimize.index.filter=false; -- partitioned table create table orc_split_elim_part (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) partitioned by (country string, year int) stored as orc; @@ -116,57 +101,43 @@ load data local inpath '../../data/files/orc_split_elim.orc' into table orc_spli -- 10 mapper - no split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us'; -SET hive.optimize.index.filter=true; -- 2 mapper - split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us'; -SET hive.optimize.index.filter=false; -- 10 mapper - no split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and (year=2000 or year=2001); -SET hive.optimize.index.filter=true; -- 2 mapper - split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and (year=2000 or year=2001); -SET hive.optimize.index.filter=false; -- 10 mapper - no split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and year=2000; -SET hive.optimize.index.filter=true; -- 1 mapper - split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and year=2000; -SET hive.optimize.index.filter=false; -- 10 mapper - no split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us'; -SET hive.optimize.index.filter=true; -- 4 mapper - split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us'; -SET hive.optimize.index.filter=false; -- 10 mapper - no split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and (year=2000 or year=2001); -SET hive.optimize.index.filter=true; -- 4 mapper - split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and (year=2000 or year=2001); -SET hive.optimize.index.filter=false; -- 10 mapper - no split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and year=2000; -SET hive.optimize.index.filter=true; -- 2 mapper - split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and year=2000; -SET hive.optimize.index.filter=false; -- 0 mapper - no split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='in'; select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='us' and year=2002; -SET hive.optimize.index.filter=true; -- 0 mapper - split elimination select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='in'; select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='us' and year=2002; -SET hive.optimize.index.filter=false; diff --git a/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q b/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q index cc3a7ab3ad73..f29af7a825a5 100644 --- a/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q +++ b/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q @@ -32,7 +32,6 @@ stored as ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="10 insert overwrite table testorc select * from vectororc order by s2; set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; -set hive.optimize.index.filter=true; set hive.vectorized.execution.enabled=false; -- row group (1,4) from stripe 1 and row group (1) from stripe 2 diff --git a/ql/src/test/queries/clientpositive/parquet_join2.q b/ql/src/test/queries/clientpositive/parquet_join2.q index 69816899de62..e698af3d29f3 100644 --- a/ql/src/test/queries/clientpositive/parquet_join2.q +++ b/ql/src/test/queries/clientpositive/parquet_join2.q @@ -1,5 +1,4 @@ set hive.vectorized.execution.enabled=false; -set hive.optimize.index.filter = true; set hive.auto.convert.join=false; CREATE TABLE tbl1_n6(id INT) STORED AS PARQUET; diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q b/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q index 499c844cdae2..be2278e36bbc 100644 --- a/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q +++ b/ql/src/test/queries/clientpositive/parquet_ppd_boolean.q @@ -12,7 +12,6 @@ create table newtypestbl(c char(10), v varchar(10), d decimal(5,3), b boolean) s insert overwrite table newtypestbl select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, true from src src1 union all select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, false from src src2 limit 10) uniontbl; SET hive.optimize.ppd=true; -SET hive.optimize.index.filter=true; select * from newtypestbl where b=true; select * from newtypestbl where b!=true; select * from newtypestbl where bfalse; select * from newtypestbl where b<=false; -SET hive.optimize.index.filter=false; select * from newtypestbl where b=true; select * from newtypestbl where b!=true; select * from newtypestbl where b'lemon '; -SET hive.optimize.index.filter=false; SET hive.optimize.ppd=false; SET hive.optimize.ppd.storage=false; diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_date.q b/ql/src/test/queries/clientpositive/parquet_ppd_date.q index 8027e9184bb1..50714173dcde 100644 --- a/ql/src/test/queries/clientpositive/parquet_ppd_date.q +++ b/ql/src/test/queries/clientpositive/parquet_ppd_date.q @@ -14,120 +14,82 @@ insert overwrite table newtypestbl_n2 select * from (select cast("apple" as char -- date data types (EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_EQUALS, IN, BETWEEN tests) select * from newtypestbl_n2 where da='1970-02-20'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da='1970-02-20'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da= date '1970-02-20'; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da=cast('1970-02-20' as date); -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da=cast('1970-02-20' as date); -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da=cast('1970-02-20' as varchar(20)); -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da=cast('1970-02-20' as varchar(20)); -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da!='1970-02-20'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da!='1970-02-20'; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da<'1970-02-27'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da<'1970-02-27'; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da<'1970-02-29' sort by c; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da<'1970-02-29' sort by c; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da<'1970-02-15'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da<'1970-02-15'; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da<='1970-02-20'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da<='1970-02-20'; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da<='1970-02-27' sort by c; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da<='1970-02-27' sort by c; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da in (cast('1970-02-21' as date), cast('1970-02-27' as date)); -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da in (cast('1970-02-21' as date), cast('1970-02-27' as date)); -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) sort by c; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) sort by c; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da in (cast('1970-02-21' as date), cast('1970-02-22' as date)); -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da in (cast('1970-02-21' as date), cast('1970-02-22' as date)); -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da between '1970-02-19' and '1970-02-22'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da between '1970-02-19' and '1970-02-22'; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da between '1970-02-19' and '1970-02-28' sort by c; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da between '1970-02-19' and '1970-02-28' sort by c; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da between '1970-02-18' and '1970-02-19'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da between '1970-02-18' and '1970-02-19'; insert overwrite table newtypestbl_n2 select * from (select cast("apple" as char(10)), cast("bee" as varchar(10)), 0.22, cast("999-02-20" as date) from src src1 union all select cast("hello" as char(10)), cast("world" as varchar(10)), 11.22, cast("1820-02-27" as date) from src src2 limit 10) uniontbl; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da='999-02-20'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da='999-02-20'; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da=cast('999-02-20' as date); -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da=cast('999-02-20' as date); set hive.vectorized.execution.enabled=true; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da='999-02-20'; -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da='999-02-20'; -set hive.optimize.index.filter=false; select * from newtypestbl_n2 where da=cast('999-02-20' as date); -set hive.optimize.index.filter=true; select * from newtypestbl_n2 where da=cast('999-02-20' as date); diff --git a/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q b/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q index f2bb84799e73..0529db89a814 100644 --- a/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q +++ b/ql/src/test/queries/clientpositive/parquet_ppd_decimal.q @@ -16,167 +16,112 @@ insert overwrite table newtypestbl_n5 select * from (select cast("apple" as char -- decimal data types (EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_EQUALS, IN, BETWEEN tests) select * from newtypestbl_n5 where d=0.22; -set hive.optimize.index.filter=true; select * from newtypestbl_n5 where d=0.22; -set hive.optimize.index.filter=false; select * from newtypestbl_n5 where d='0.22'; -set hive.optimize.index.filter=true; select * from newtypestbl_n5 where d='0.22'; -set hive.optimize.index.filter=false; select * from newtypestbl_n5 where d=cast('0.22' as float); -set hive.optimize.index.filter=true; select * from newtypestbl_n5 where d=cast('0.22' as float); -set hive.optimize.index.filter=false; select * from newtypestbl_n5 where d!=0.22; -set hive.optimize.index.filter=true; select * from newtypestbl_n5 where d!=0.22; -set hive.optimize.index.filter=false; select * from newtypestbl_n5 where d!='0.22'; -set hive.optimize.index.filter=true; select * from newtypestbl_n5 where d!='0.22'; -set hive.optimize.index.filter=false; select * from newtypestbl_n5 where d!=cast('0.22' as float); -set hive.optimize.index.filter=true; select * from newtypestbl_n5 where d!=cast('0.22' as float); -set hive.optimize.index.filter=false; select * from newtypestbl_n5 where d<11.22; -set hive.optimize.index.filter=true; select * from newtypestbl_n5 where d<11.22; -set hive.optimize.index.filter=false; select * from newtypestbl_n5 where d<'11.22'; -set hive.optimize.index.filter=true; select * from newtypestbl_n5 where d<'11.22'; -set hive.optimize.index.filter=false; select * from newtypestbl_n5 where d2 limit 1; -SET hive.optimize.index.filter=true; SELECT * FROM tbl_pred WHERE t>2 limit 1; -SET hive.optimize.index.filter=false; SELECT * FROM tbl_pred WHERE t IS NOT NULL @@ -69,13 +61,11 @@ SELECT * FROM tbl_pred AND t > -2 LIMIT 10; -SET hive.optimize.index.filter=true; SELECT * FROM tbl_pred WHERE t IS NOT NULL AND t < 0 AND t > -2 LIMIT 10; -SET hive.optimize.index.filter=false; EXPLAIN SELECT * FROM tbl_pred WHERE t IS NOT NULL @@ -83,13 +73,11 @@ EXPLAIN SELECT * FROM tbl_pred AND t > -2 LIMIT 10; -SET hive.optimize.index.filter=true; EXPLAIN SELECT * FROM tbl_pred WHERE t IS NOT NULL AND t < 0 AND t > -2 LIMIT 10; -SET hive.optimize.index.filter=false; SELECT t, s FROM tbl_pred WHERE t <=> -1 @@ -97,13 +85,11 @@ SELECT t, s FROM tbl_pred AND s LIKE 'bob%' ; -SET hive.optimize.index.filter=true; SELECT t, s FROM tbl_pred WHERE t <=> -1 AND s IS NOT NULL AND s LIKE 'bob%' ; -SET hive.optimize.index.filter=false; EXPLAIN SELECT t, s FROM tbl_pred WHERE t <=> -1 @@ -111,13 +97,11 @@ EXPLAIN SELECT t, s FROM tbl_pred AND s LIKE 'bob%' ; -SET hive.optimize.index.filter=true; EXPLAIN SELECT t, s FROM tbl_pred WHERE t <=> -1 AND s IS NOT NULL AND s LIKE 'bob%' ; -SET hive.optimize.index.filter=false; SELECT t, s FROM tbl_pred WHERE s IS NOT NULL @@ -126,14 +110,12 @@ SELECT t, s FROM tbl_pred AND t BETWEEN 25 AND 30 SORT BY t,s; -set hive.optimize.index.filter=true; SELECT t, s FROM tbl_pred WHERE s IS NOT NULL AND s LIKE 'bob%' AND t NOT IN (-1,-2,-3) AND t BETWEEN 25 AND 30 SORT BY t,s; -set hive.optimize.index.filter=false; EXPLAIN SELECT t, s FROM tbl_pred WHERE s IS NOT NULL @@ -142,14 +124,12 @@ EXPLAIN SELECT t, s FROM tbl_pred AND t BETWEEN 25 AND 30 SORT BY t,s; -SET hive.optimize.index.filter=true; EXPLAIN SELECT t, s FROM tbl_pred WHERE s IS NOT NULL AND s LIKE 'bob%' AND t NOT IN (-1,-2,-3) AND t BETWEEN 25 AND 30 SORT BY t,s; -SET hive.optimize.index.filter=false; SELECT t, si, d, s FROM tbl_pred WHERE d >= ROUND(9.99) @@ -162,7 +142,6 @@ SELECT t, si, d, s FROM tbl_pred ORDER BY s DESC LIMIT 3; -SET hive.optimize.index.filter=true; SELECT t, si, d, s FROM tbl_pred WHERE d >= ROUND(9.99) AND d < 12 @@ -173,7 +152,6 @@ SELECT t, si, d, s FROM tbl_pred AND si BETWEEN 300 AND 400 ORDER BY s DESC LIMIT 3; -SET hive.optimize.index.filter=false; EXPLAIN SELECT t, si, d, s FROM tbl_pred WHERE d >= ROUND(9.99) @@ -186,7 +164,6 @@ EXPLAIN SELECT t, si, d, s FROM tbl_pred ORDER BY s DESC LIMIT 3; -SET hive.optimize.index.filter=true; EXPLAIN SELECT t, si, d, s FROM tbl_pred WHERE d >= ROUND(9.99) AND d < 12 @@ -197,7 +174,6 @@ EXPLAIN SELECT t, si, d, s FROM tbl_pred AND si BETWEEN 300 AND 400 ORDER BY s DESC LIMIT 3; -SET hive.optimize.index.filter=false; SELECT t, si, d, s FROM tbl_pred WHERE t > 10 @@ -212,7 +188,6 @@ SELECT t, si, d, s FROM tbl_pred SORT BY s DESC LIMIT 3; -SET hive.optimize.index.filter=true; SELECT t, si, d, s FROM tbl_pred WHERE t > 10 AND t <> 101 @@ -225,9 +200,7 @@ SELECT t, si, d, s FROM tbl_pred AND si BETWEEN 300 AND 400 SORT BY s DESC LIMIT 3; -SET hive.optimize.index.filter=false; -SET hive.optimize.index.filter=true; SELECT f, i, b FROM tbl_pred WHERE f IS NOT NULL AND f < 123.2 @@ -246,7 +219,6 @@ SELECT f, i, b FROM tbl_pred AND b BETWEEN 4294967261 AND 4294967861 SORT BY f DESC LIMIT 3; -SET hive.optimize.index.filter=false; EXPLAIN SELECT t, si, d, s FROM tbl_pred WHERE t > 10 @@ -261,7 +233,6 @@ EXPLAIN SELECT t, si, d, s FROM tbl_pred SORT BY s DESC LIMIT 3; -SET hive.optimize.index.filter=true; EXPLAIN SELECT t, si, d, s FROM tbl_pred WHERE t > 10 AND t <> 101 @@ -274,10 +245,8 @@ EXPLAIN SELECT t, si, d, s FROM tbl_pred AND si BETWEEN 300 AND 400 SORT BY s DESC LIMIT 3; -SET hive.optimize.index.filter=false; -SET hive.optimize.index.filter=true; EXPLAIN SELECT f, i, b FROM tbl_pred WHERE f IS NOT NULL AND f < 123.2 @@ -296,4 +265,3 @@ EXPLAIN SELECT f, i, b FROM tbl_pred AND b BETWEEN 4294967261 AND 4294967861 SORT BY f DESC LIMIT 3; -SET hive.optimize.index.filter=false; diff --git a/ql/src/test/queries/clientpositive/parquet_predicate_pushdown_2.q b/ql/src/test/queries/clientpositive/parquet_predicate_pushdown_2.q index 038363050d2e..05bccfbc6533 100644 --- a/ql/src/test/queries/clientpositive/parquet_predicate_pushdown_2.q +++ b/ql/src/test/queries/clientpositive/parquet_predicate_pushdown_2.q @@ -1,6 +1,5 @@ set hive.vectorized.execution.enabled=false; SET hive.optimize.ppd=true; -SET hive.optimize.index.filter=true; create table test_parq(a int, b int) partitioned by (p int) stored as parquet; insert overwrite table test_parq partition (p=1) values (1, 1); diff --git a/ql/src/test/queries/clientpositive/parquet_vectorization_pushdown.q b/ql/src/test/queries/clientpositive/parquet_vectorization_pushdown.q index 2c52bebfb158..f41f4702cc95 100644 --- a/ql/src/test/queries/clientpositive/parquet_vectorization_pushdown.q +++ b/ql/src/test/queries/clientpositive/parquet_vectorization_pushdown.q @@ -2,7 +2,6 @@ set hive.mapred.mode=nonstrict; set hive.explain.user=false; SET hive.vectorized.execution.enabled=true; -SET hive.optimize.index.filter=true; set hive.fetch.task.conversion=none; explain vectorization SELECT AVG(cbigint) FROM alltypesparquet WHERE cbigint < cdouble; diff --git a/ql/src/test/queries/clientpositive/ppd_deterministic_expr.q b/ql/src/test/queries/clientpositive/ppd_deterministic_expr.q index 47c88493c623..fffd086da231 100644 --- a/ql/src/test/queries/clientpositive/ppd_deterministic_expr.q +++ b/ql/src/test/queries/clientpositive/ppd_deterministic_expr.q @@ -1,5 +1,4 @@ set hive.auto.convert.join=false; -set hive.optimize.index.filter=true; set hive.cbo.enable=false; CREATE TABLE `testb`( diff --git a/ql/src/test/queries/clientpositive/reopt_dpp.q b/ql/src/test/queries/clientpositive/reopt_dpp.q index 952dcbe0b86a..972ab93e9540 100644 --- a/ql/src/test/queries/clientpositive/reopt_dpp.q +++ b/ql/src/test/queries/clientpositive/reopt_dpp.q @@ -1,5 +1,4 @@ set hive.explain.user=true; -set hive.optimize.index.filter=true; set hive.auto.convert.join=true; set hive.vectorized.execution.enabled=true; diff --git a/ql/src/test/queries/clientpositive/reopt_semijoin.q b/ql/src/test/queries/clientpositive/reopt_semijoin.q index 75db410e6293..6796825c4eee 100644 --- a/ql/src/test/queries/clientpositive/reopt_semijoin.q +++ b/ql/src/test/queries/clientpositive/reopt_semijoin.q @@ -1,5 +1,4 @@ set hive.explain.user=true; -set hive.optimize.index.filter=true; set hive.auto.convert.join=true; set hive.vectorized.execution.enabled=true; @@ -45,7 +44,6 @@ alter table x1_date_dim update statistics set( set hive.auto.convert.join.noconditionaltask.size=1; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; -set hive.optimize.index.filter=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; set hive.tez.bloom.filter.factor=1.0f; diff --git a/ql/src/test/queries/clientpositive/semijoin_hint.q b/ql/src/test/queries/clientpositive/semijoin_hint.q index cdf23b7653bc..d796b018d7bc 100644 --- a/ql/src/test/queries/clientpositive/semijoin_hint.q +++ b/ql/src/test/queries/clientpositive/semijoin_hint.q @@ -11,7 +11,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; @@ -120,4 +119,4 @@ WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b); explain merge /*+ semi(s, a, t, 1000)*/ into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a WHEN MATCHED AND s.a > 8 THEN DELETE WHEN MATCHED THEN UPDATE SET b = 7 -WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b); \ No newline at end of file +WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b); diff --git a/ql/src/test/queries/clientpositive/semijoin_reddedup.q b/ql/src/test/queries/clientpositive/semijoin_reddedup.q index 8e298acb6766..01c3737c0dd4 100644 --- a/ql/src/test/queries/clientpositive/semijoin_reddedup.q +++ b/ql/src/test/queries/clientpositive/semijoin_reddedup.q @@ -12,7 +12,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; diff --git a/ql/src/test/queries/clientpositive/semijoin_removal_udf.q b/ql/src/test/queries/clientpositive/semijoin_removal_udf.q index 5fadeb04a31b..96a1c8a8439b 100644 --- a/ql/src/test/queries/clientpositive/semijoin_removal_udf.q +++ b/ql/src/test/queries/clientpositive/semijoin_removal_udf.q @@ -1,4 +1,3 @@ -set hive.optimize.index.filter=true; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.exec.dynamic.partition.mode=nonstrict; diff --git a/ql/src/test/queries/clientpositive/sharedwork_semi.q b/ql/src/test/queries/clientpositive/sharedwork_semi.q index b520f3e6dc17..c75adba29c4e 100644 --- a/ql/src/test/queries/clientpositive/sharedwork_semi.q +++ b/ql/src/test/queries/clientpositive/sharedwork_semi.q @@ -1,5 +1,4 @@ set hive.explain.user=true; -set hive.optimize.index.filter=true; set hive.auto.convert.join=true; set hive.vectorized.execution.enabled=true; set hive.optimize.shared.work.semijoin=true; @@ -46,7 +45,6 @@ alter table x1_date_dim update statistics set( set hive.auto.convert.join.noconditionaltask.size=1; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=true; -set hive.optimize.index.filter=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; set hive.tez.bloom.filter.factor=1.0f; diff --git a/ql/src/test/queries/clientpositive/swo_event_merge.q b/ql/src/test/queries/clientpositive/swo_event_merge.q index c7e543501cf3..60edebc10105 100644 --- a/ql/src/test/queries/clientpositive/swo_event_merge.q +++ b/ql/src/test/queries/clientpositive/swo_event_merge.q @@ -1,5 +1,4 @@ set hive.explain.user=true; -set hive.optimize.index.filter=true; set hive.auto.convert.join=false; set hive.vectorized.execution.enabled=true; diff --git a/ql/src/test/queries/clientpositive/temp_table_llap_partitioned.q b/ql/src/test/queries/clientpositive/temp_table_llap_partitioned.q index e496bf789256..eac3013ad2aa 100644 --- a/ql/src/test/queries/clientpositive/temp_table_llap_partitioned.q +++ b/ql/src/test/queries/clientpositive/temp_table_llap_partitioned.q @@ -6,7 +6,6 @@ SET hive.llap.io.enabled=false; SET hive.exec.orc.default.buffer.size=32768; SET hive.exec.orc.default.row.index.stride=1000; -SET hive.optimize.index.filter=true; set hive.auto.convert.join=true; set hive.exec.dynamic.partition=true; diff --git a/ql/src/test/queries/clientpositive/temp_table_parquet_ppd_partition.q b/ql/src/test/queries/clientpositive/temp_table_parquet_ppd_partition.q index 75d80eff31da..4f94560ae2e0 100644 --- a/ql/src/test/queries/clientpositive/temp_table_parquet_ppd_partition.q +++ b/ql/src/test/queries/clientpositive/temp_table_parquet_ppd_partition.q @@ -1,5 +1,4 @@ set hive.vectorized.execution.enabled=false; -SET hive.optimize.index.filter=true; SET hive.optimize.ppd=true; -- Test predicate with partitioned columns @@ -7,4 +6,4 @@ CREATE TEMPORARY TABLE part1_n1_temp (id int, content string) PARTITIONED BY (p ALTER TABLE part1_n1_temp ADD PARTITION (p='p1'); INSERT INTO TABLE part1_n1_temp PARTITION (p='p1') VALUES (1, 'a'), (2, 'b'); SELECT * FROM part1_n1_temp; -DROP TABLE part1_n1_temp PURGE; \ No newline at end of file +DROP TABLE part1_n1_temp PURGE; diff --git a/ql/src/test/queries/clientpositive/tez_fixed_bucket_pruning.q b/ql/src/test/queries/clientpositive/tez_fixed_bucket_pruning.q index 21d5907d7c37..d4cbcc81888c 100644 --- a/ql/src/test/queries/clientpositive/tez_fixed_bucket_pruning.q +++ b/ql/src/test/queries/clientpositive/tez_fixed_bucket_pruning.q @@ -159,7 +159,6 @@ set hive.explain.user=false; set hive.auto.convert.join=true; set hive.vectorized.execution.enabled=true; set hive.auto.convert.join.noconditionaltask.size=200000000; -set hive.optimize.index.filter=true; -- fixed bucket pruning off set hive.tez.bucket.pruning=false; @@ -234,4 +233,4 @@ select col_1, col_2, col_3 from test_table where col_1 <> 2 order by col_2; explain extended select col_1, col_2, col_3 from test_table where col_1 = 2 order by col_2; select col_1, col_2, col_3 from test_table where col_1 = 2 order by col_2; -drop table `test_table`; \ No newline at end of file +drop table `test_table`; diff --git a/ql/src/test/queries/clientpositive/tez_tag.q b/ql/src/test/queries/clientpositive/tez_tag.q index af510af9ff1c..913a9bd7ad15 100644 --- a/ql/src/test/queries/clientpositive/tez_tag.q +++ b/ql/src/test/queries/clientpositive/tez_tag.q @@ -10,7 +10,6 @@ set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.tez.dynamic.semijoin.reduction=false; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.stats.autogather=true; set hive.tez.bigtable.minsize.semijoin.reduction=1; set hive.tez.min.bloom.filter.entries=1; diff --git a/ql/src/test/queries/clientpositive/topnkey.q b/ql/src/test/queries/clientpositive/topnkey.q index 8a946b77655f..41a545ab107d 100644 --- a/ql/src/test/queries/clientpositive/topnkey.q +++ b/ql/src/test/queries/clientpositive/topnkey.q @@ -7,7 +7,6 @@ SET hive.optimize.ppd=true; SET hive.ppd.remove.duplicatefilters=true; SET hive.tez.dynamic.partition.pruning=true; SET hive.optimize.metadataonly=false; -SET hive.optimize.index.filter=true; SET hive.tez.min.bloom.filter.entries=1; SET hive.stats.fetch.column.stats=true; diff --git a/ql/src/test/queries/clientpositive/union_view.q b/ql/src/test/queries/clientpositive/union_view.q index a3787079d533..b86ad69252b6 100644 --- a/ql/src/test/queries/clientpositive/union_view.q +++ b/ql/src/test/queries/clientpositive/union_view.q @@ -13,7 +13,6 @@ CREATE TABLE src_union_3_n0(key int, value string) PARTITIONED BY (ds string, pa SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; -SET hive.optimize.index.filter=true; SET hive.exec.pre.hooks=; SET hive.exec.post.hooks=; diff --git a/ql/src/test/queries/clientpositive/vector_between_columns.q b/ql/src/test/queries/clientpositive/vector_between_columns.q index 63142f30b049..c4b8832d0469 100644 --- a/ql/src/test/queries/clientpositive/vector_between_columns.q +++ b/ql/src/test/queries/clientpositive/vector_between_columns.q @@ -5,7 +5,6 @@ SET hive.auto.convert.join=true; set hive.fetch.task.conversion=none; set hive.mapred.mode=nonstrict; set hive.join.inner.residual=false; -set hive.optimize.index.filter=true; -- SORT_QUERY_RESULTS -- diff --git a/ql/src/test/queries/clientpositive/vector_offset_limit.q b/ql/src/test/queries/clientpositive/vector_offset_limit.q index dd6a3e74324a..7d21874d7319 100644 --- a/ql/src/test/queries/clientpositive/vector_offset_limit.q +++ b/ql/src/test/queries/clientpositive/vector_offset_limit.q @@ -13,7 +13,6 @@ set hive.mapjoin.hybridgrace.hashtable=false; set hive.merge.mapfiles=false; set hive.merge.nway.joins=false; set hive.optimize.bucketmapjoin=true; -set hive.optimize.index.filter=true; set hive.stats.fetch.bitvector=false; set hive.stats.fetch.column.stats=true; set hive.support.quoted.identifiers=none; diff --git a/ql/src/test/queries/clientpositive/vector_outer_join_constants.q b/ql/src/test/queries/clientpositive/vector_outer_join_constants.q index 0431f0ea8046..30a8fbe2d073 100644 --- a/ql/src/test/queries/clientpositive/vector_outer_join_constants.q +++ b/ql/src/test/queries/clientpositive/vector_outer_join_constants.q @@ -8,7 +8,6 @@ set hive.optimize.reducededuplication=true; set hive.optimize.null.scan=true; set hive.mapjoin.optimized.hashtable=true; set hive.optimize.constant.propagation=true; -set hive.optimize.index.filter=true; set hive.optimize.bucketmapjoin=true; set hive.limit.optimize.enable=true; set hive.optimize.bucketmapjoin.sortedmerge=false; diff --git a/ql/src/test/queries/clientpositive/vector_topnkey.q b/ql/src/test/queries/clientpositive/vector_topnkey.q index 3412aba9dbc6..03d248a8740b 100644 --- a/ql/src/test/queries/clientpositive/vector_topnkey.q +++ b/ql/src/test/queries/clientpositive/vector_topnkey.q @@ -6,7 +6,6 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=true; set hive.tez.dynamic.partition.pruning=true; set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; set hive.tez.min.bloom.filter.entries=1; set hive.tez.dynamic.partition.pruning=true; diff --git a/ql/src/test/queries/clientpositive/vectorization_parquet_ppd_decimal.q b/ql/src/test/queries/clientpositive/vectorization_parquet_ppd_decimal.q index efd2a6a64667..16667308acbd 100644 --- a/ql/src/test/queries/clientpositive/vectorization_parquet_ppd_decimal.q +++ b/ql/src/test/queries/clientpositive/vectorization_parquet_ppd_decimal.q @@ -21,167 +21,112 @@ select * -- decimal data types (EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_EQUALS, IN, BETWEEN tests) select * from newtypestbl_n1 where d=0.22; -set hive.optimize.index.filter=true; select * from newtypestbl_n1 where d=0.22; -set hive.optimize.index.filter=false; select * from newtypestbl_n1 where d='0.22'; -set hive.optimize.index.filter=true; select * from newtypestbl_n1 where d='0.22'; -set hive.optimize.index.filter=false; select * from newtypestbl_n1 where d=cast('0.22' as float); -set hive.optimize.index.filter=true; select * from newtypestbl_n1 where d=cast('0.22' as float); -set hive.optimize.index.filter=false; select * from newtypestbl_n1 where d!=0.22; -set hive.optimize.index.filter=true; select * from newtypestbl_n1 where d!=0.22; -set hive.optimize.index.filter=false; select * from newtypestbl_n1 where d!='0.22'; -set hive.optimize.index.filter=true; select * from newtypestbl_n1 where d!='0.22'; -set hive.optimize.index.filter=false; select * from newtypestbl_n1 where d!=cast('0.22' as float); -set hive.optimize.index.filter=true; select * from newtypestbl_n1 where d!=cast('0.22' as float); -set hive.optimize.index.filter=false; select * from newtypestbl_n1 where d<11.22; -set hive.optimize.index.filter=true; select * from newtypestbl_n1 where d<11.22; -set hive.optimize.index.filter=false; select * from newtypestbl_n1 where d<'11.22'; -set hive.optimize.index.filter=true; select * from newtypestbl_n1 where d<'11.22'; -set hive.optimize.index.filter=false; select * from newtypestbl_n1 where d