Skip to content

Commit 07eaab3

Browse files
sershe-apachesershe-apache
committed
HIVE-11923 : allow qtests to run via a single client session for tez and llap (Siddharth Seth/Sergey Shelukhin, reviewed by Prasanth Jayachandran)
1 parent bb791ba commit 07eaab3

File tree

6 files changed

+69
-21
lines changed

6 files changed

+69
-21
lines changed

‎itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@ public void init() throws Exception {
6969
}
7070

7171
@Override
72-
public void createSources() throws Exception {
73-
super.createSources();
72+
public void createSources(String tname) throws Exception {
73+
super.createSources(tname);
7474

7575
conf.setBoolean("hive.test.init.phase", true);
7676

@@ -96,8 +96,8 @@ public void createSources() throws Exception {
9696
}
9797

9898
@Override
99-
public void cleanUp() throws Exception {
100-
super.cleanUp();
99+
public void cleanUp(String tname) throws Exception {
100+
super.cleanUp(tname);
101101

102102
// drop in case leftover from unsuccessful run
103103
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, HBASE_SRC_NAME);

‎itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java‎

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ public class QTestUtil {
142142
private final Set<String> qSortQuerySet;
143143
private final Set<String> qHashQuerySet;
144144
private final Set<String> qSortNHashQuerySet;
145+
private final Set<String> qNoSessionReuseQuerySet;
145146
private final Set<String> qJavaVersionSpecificOutput;
146147
private static final String SORT_SUFFIX = ".sorted";
147148
public static final HashSet<String> srcTables = new HashSet<String>();
@@ -401,6 +402,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
401402
qSortQuerySet = new HashSet<String>();
402403
qHashQuerySet = new HashSet<String>();
403404
qSortNHashQuerySet = new HashSet<String>();
405+
qNoSessionReuseQuerySet = new HashSet<String>();
404406
qJavaVersionSpecificOutput = new HashSet<String>();
405407
QTestUtil.clusterType = clusterType;
406408

@@ -556,12 +558,16 @@ public void addFile(File qf, boolean partial) throws IOException {
556558
} else if (matches(SORT_AND_HASH_QUERY_RESULTS, query)) {
557559
qSortNHashQuerySet.add(qf.getName());
558560
}
561+
if (matches(NO_SESSION_REUSE, query)) {
562+
qNoSessionReuseQuerySet.add(qf.getName());
563+
}
559564
}
560565

561566
private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF");
562567
private static final Pattern SORT_QUERY_RESULTS = Pattern.compile("-- SORT_QUERY_RESULTS");
563568
private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS");
564569
private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS");
570+
private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE");
565571

566572
private boolean matches(Pattern pattern, String query) {
567573
Matcher matcher = pattern.matcher(query);
@@ -803,8 +809,13 @@ public void clearTestSideEffects() throws Exception {
803809
}
804810

805811
public void cleanUp() throws Exception {
812+
cleanUp(null);
813+
}
814+
815+
public void cleanUp(String tname) throws Exception {
816+
boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
806817
if(!isSessionStateStarted) {
807-
startSessionState();
818+
startSessionState(canReuseSession);
808819
}
809820
if (System.getenv(QTEST_LEAVE_FILES) != null) {
810821
return;
@@ -867,8 +878,13 @@ protected void runCmd(String cmd) throws Exception {
867878
}
868879

869880
public void createSources() throws Exception {
881+
createSources(null);
882+
}
883+
884+
public void createSources(String tname) throws Exception {
885+
boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
870886
if(!isSessionStateStarted) {
871-
startSessionState();
887+
startSessionState(canReuseSession);
872888
}
873889

874890
if(cliDriver == null) {
@@ -908,8 +924,8 @@ public void init() throws Exception {
908924
}
909925

910926
public void init(String tname) throws Exception {
911-
cleanUp();
912-
createSources();
927+
cleanUp(tname);
928+
createSources(tname);
913929
cliDriver.processCmd("set hive.cli.print.header=true;");
914930
}
915931

@@ -919,8 +935,8 @@ public void cliInit(String tname) throws Exception {
919935

920936
public String cliInit(String tname, boolean recreate) throws Exception {
921937
if (recreate) {
922-
cleanUp();
923-
createSources();
938+
cleanUp(tname);
939+
createSources(tname);
924940
}
925941

926942
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
@@ -955,18 +971,24 @@ public String cliInit(String tname, boolean recreate) throws Exception {
955971
ss.setIsSilent(true);
956972
SessionState oldSs = SessionState.get();
957973

958-
if (oldSs != null && (clusterType == MiniClusterType.llap
959-
|| clusterType == MiniClusterType.spark || clusterType == MiniClusterType.miniSparkOnYarn)) {
960-
sparkSession = oldSs.getSparkSession();
961-
ss.setSparkSession(sparkSession);
962-
oldSs.setSparkSession(null);
974+
boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname);
975+
if (oldSs != null && canReuseSession
976+
&& (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap)) {
963977
// Copy the tezSessionState from the old CliSessionState.
964978
tezSessionState = oldSs.getTezSession();
965979
ss.setTezSession(tezSessionState);
966980
oldSs.setTezSession(null);
967981
oldSs.close();
968982
}
969983

984+
if (oldSs != null && (clusterType == MiniClusterType.spark
985+
|| clusterType == MiniClusterType.miniSparkOnYarn)) {
986+
sparkSession = oldSs.getSparkSession();
987+
ss.setSparkSession(sparkSession);
988+
oldSs.setSparkSession(null);
989+
oldSs.close();
990+
}
991+
970992
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
971993
oldSs.out.close();
972994
}
@@ -1008,7 +1030,7 @@ public void setSparkSession(SparkSession sparkSession) {
10081030
};
10091031
}
10101032

1011-
private CliSessionState startSessionState()
1033+
private CliSessionState startSessionState(boolean canReuseSession)
10121034
throws IOException {
10131035

10141036
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
@@ -1023,17 +1045,22 @@ private CliSessionState startSessionState()
10231045
ss.err = System.out;
10241046

10251047
SessionState oldSs = SessionState.get();
1026-
if (oldSs != null && (clusterType == MiniClusterType.llap
1027-
|| clusterType == MiniClusterType.miniSparkOnYarn || clusterType == MiniClusterType.miniSparkOnYarn)) {
1028-
sparkSession = oldSs.getSparkSession();
1029-
ss.setSparkSession(sparkSession);
1030-
oldSs.setSparkSession(null);
1048+
if (oldSs != null && canReuseSession
1049+
&& (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap)) {
10311050
// Copy the tezSessionState from the old CliSessionState.
10321051
tezSessionState = oldSs.getTezSession();
10331052
ss.setTezSession(tezSessionState);
10341053
oldSs.setTezSession(null);
10351054
oldSs.close();
10361055
}
1056+
1057+
if (oldSs != null && (clusterType == MiniClusterType.spark
1058+
|| clusterType == MiniClusterType.miniSparkOnYarn)) {
1059+
sparkSession = oldSs.getSparkSession();
1060+
ss.setSparkSession(sparkSession);
1061+
oldSs.setSparkSession(null);
1062+
oldSs.close();
1063+
}
10371064
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
10381065
oldSs.out.close();
10391066
}

‎ql/src/test/queries/clientpositive/scriptfile1.q‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
33
-- SORT_QUERY_RESULTS
44

55
-- EXCLUDE_OS_WINDOWS
6+
7+
-- NO_SESSION_REUSE
8+
69
CREATE TABLE dest1(key INT, value STRING);
710

811
ADD FILE ../../ql/src/test/scripts/testgrep;

‎ql/src/test/results/clientpositive/scriptfile1.q.out‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
PREHOOK: query: -- SORT_QUERY_RESULTS
22

33
-- EXCLUDE_OS_WINDOWS
4+
5+
-- NO_SESSION_REUSE
6+
47
CREATE TABLE dest1(key INT, value STRING)
58
PREHOOK: type: CREATETABLE
69
PREHOOK: Output: database:default
710
PREHOOK: Output: default@dest1
811
POSTHOOK: query: -- SORT_QUERY_RESULTS
912

1013
-- EXCLUDE_OS_WINDOWS
14+
15+
-- NO_SESSION_REUSE
16+
1117
CREATE TABLE dest1(key INT, value STRING)
1218
POSTHOOK: type: CREATETABLE
1319
POSTHOOK: Output: database:default

‎ql/src/test/results/clientpositive/spark/scriptfile1.q.out‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
PREHOOK: query: -- SORT_QUERY_RESULTS
22

33
-- EXCLUDE_OS_WINDOWS
4+
5+
-- NO_SESSION_REUSE
6+
47
CREATE TABLE dest1(key INT, value STRING)
58
PREHOOK: type: CREATETABLE
69
PREHOOK: Output: database:default
710
PREHOOK: Output: default@dest1
811
POSTHOOK: query: -- SORT_QUERY_RESULTS
912

1013
-- EXCLUDE_OS_WINDOWS
14+
15+
-- NO_SESSION_REUSE
16+
1117
CREATE TABLE dest1(key INT, value STRING)
1218
POSTHOOK: type: CREATETABLE
1319
POSTHOOK: Output: database:default

‎ql/src/test/results/clientpositive/tez/scriptfile1.q.out‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
PREHOOK: query: -- SORT_QUERY_RESULTS
22

33
-- EXCLUDE_OS_WINDOWS
4+
5+
-- NO_SESSION_REUSE
6+
47
CREATE TABLE dest1(key INT, value STRING)
58
PREHOOK: type: CREATETABLE
69
PREHOOK: Output: database:default
710
PREHOOK: Output: default@dest1
811
POSTHOOK: query: -- SORT_QUERY_RESULTS
912

1013
-- EXCLUDE_OS_WINDOWS
14+
15+
-- NO_SESSION_REUSE
16+
1117
CREATE TABLE dest1(key INT, value STRING)
1218
POSTHOOK: type: CREATETABLE
1319
POSTHOOK: Output: database:default

0 commit comments

Comments
 (0)