Skip to content

Commit 0ea9d35

Browse files
prongsashutoshc
prongs
authored and
ashutoshc
committed
HIVE-13676 : Tests failing because metastore doesn't come up (Rajat Khandelwal via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <[email protected]>
1 parent 882a7f0 commit 0ea9d35

File tree

7 files changed

+48
-159
lines changed

7 files changed

+48
-159
lines changed

‎hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@
3333
import org.apache.hadoop.fs.Path;
3434
import org.apache.hadoop.fs.permission.FsPermission;
3535
import org.apache.hadoop.hive.conf.HiveConf;
36-
import org.apache.hadoop.hive.metastore.HiveMetaStore;
3736
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
37+
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
3838
import org.apache.hadoop.hive.metastore.api.FieldSchema;
3939
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
4040
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@@ -89,7 +89,7 @@ public class TestHCatMultiOutputFormat {
8989
private static HiveConf hiveConf;
9090
private static File workDir;
9191

92-
private static final String msPort = "20199";
92+
private static int msPort;
9393
private static Thread t;
9494

9595
static {
@@ -98,22 +98,6 @@ public class TestHCatMultiOutputFormat {
9898
schemaMap.put(tableNames[2], new HCatSchema(ColumnHolder.hCattest3Cols));
9999
}
100100

101-
private static class RunMS implements Runnable {
102-
103-
@Override
104-
public void run() {
105-
try {
106-
String warehouseConf = HiveConf.ConfVars.METASTOREWAREHOUSE.varname + "="
107-
+ warehousedir.toString();
108-
HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", warehouseConf});
109-
} catch (Throwable t) {
110-
System.err.println("Exiting. Got exception from metastore: " + t.getMessage());
111-
t.printStackTrace();
112-
}
113-
}
114-
115-
}
116-
117101
/**
118102
* Private class which holds all the data for the test cases
119103
*/
@@ -173,10 +157,11 @@ public static void setup() throws Exception {
173157

174158
warehousedir = new Path(System.getProperty("test.warehouse.dir"));
175159

176-
// Run hive metastore server
177-
t = new Thread(new RunMS());
178-
t.start();
160+
HiveConf metastoreConf = new HiveConf();
161+
metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString());
179162

163+
// Run hive metastore server
164+
msPort = MetaStoreUtils.startMetaStore(metastoreConf);
180165
// LocalJobRunner does not work with mapreduce OutputCommitter. So need
181166
// to use MiniMRCluster. MAPREDUCE-2350
182167
Configuration conf = new Configuration(true);

‎hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java‎

Lines changed: 8 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@
3434
import org.apache.hadoop.conf.Configuration;
3535
import org.apache.hadoop.fs.Path;
3636
import org.apache.hadoop.hive.conf.HiveConf;
37-
import org.apache.hadoop.hive.metastore.HiveMetaStore;
3837
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
38+
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
3939
import org.apache.hadoop.hive.metastore.Warehouse;
4040
import org.apache.hadoop.hive.metastore.api.MetaException;
4141
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
@@ -83,41 +83,14 @@
8383

8484
public class TestHCatClient {
8585
private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class);
86-
private static final String msPort = "20101";
86+
private static int msPort;
8787
private static HiveConf hcatConf;
8888
private static boolean isReplicationTargetHCatRunning = false;
89-
private static final String replicationTargetHCatPort = "20102";
89+
private static int replicationTargetHCatPort;
9090
private static HiveConf replicationTargetHCatConf;
9191
private static SecurityManager securityManager;
9292
private static boolean useExternalMS = false;
9393

94-
public static class RunMS implements Runnable {
95-
96-
private final String msPort;
97-
private List<String> args = new ArrayList<String>();
98-
99-
public RunMS(String msPort) {
100-
this.msPort = msPort;
101-
this.args.add("-v");
102-
this.args.add("-p");
103-
this.args.add(this.msPort);
104-
}
105-
106-
public RunMS arg(String arg) {
107-
this.args.add(arg);
108-
return this;
109-
}
110-
111-
@Override
112-
public void run() {
113-
try {
114-
HiveMetaStore.main(args.toArray(new String[args.size()]));
115-
} catch (Throwable t) {
116-
LOG.error("Exiting. Got exception from metastore: ", t);
117-
}
118-
}
119-
} // class RunMS;
120-
12194
@AfterClass
12295
public static void tearDown() throws Exception {
12396
if (!useExternalMS) {
@@ -142,10 +115,7 @@ public static void startMetaStoreServer() throws Exception {
142115

143116
System.setProperty(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname,
144117
DbNotificationListener.class.getName()); // turn on db notification listener on metastore
145-
Thread t = new Thread(new RunMS(msPort));
146-
t.start();
147-
Thread.sleep(10000);
148-
118+
msPort = MetaStoreUtils.startMetaStore();
149119
securityManager = System.getSecurityManager();
150120
System.setSecurityManager(new NoExitSecurityManager());
151121
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:"
@@ -832,13 +802,10 @@ public void testDropPartitionsWithPartialSpec() throws Exception {
832802

833803
private void startReplicationTargetMetaStoreIfRequired() throws Exception {
834804
if (!isReplicationTargetHCatRunning) {
835-
Thread t = new Thread(new RunMS(replicationTargetHCatPort)
836-
.arg("--hiveconf")
837-
.arg("javax.jdo.option.ConnectionURL") // Reset, to use a different Derby instance.
838-
.arg(hcatConf.get("javax.jdo.option.ConnectionURL")
839-
.replace("metastore", "target_metastore")));
840-
t.start();
841-
Thread.sleep(10000);
805+
HiveConf conf = new HiveConf();
806+
conf.set("javax.jdo.option.ConnectionURL", hcatConf.get("javax.jdo.option.ConnectionURL")
807+
.replace("metastore", "target_metastore"));
808+
replicationTargetHCatPort = MetaStoreUtils.startMetaStore(conf);
842809
replicationTargetHCatConf = new HiveConf(hcatConf);
843810
replicationTargetHCatConf.setVar(HiveConf.ConfVars.METASTOREURIS,
844811
"thrift://localhost:" + replicationTargetHCatPort);

‎itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java‎

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -19,37 +19,13 @@
1919
package org.apache.hadoop.hive.metastore;
2020

2121
import org.apache.hadoop.hive.conf.HiveConf;
22-
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
2322

2423
public class TestMarkPartitionRemote extends TestMarkPartition {
2524

26-
private static class RunMS implements Runnable {
27-
28-
private final int port;
29-
30-
public RunMS(int port) {
31-
this.port = port;
32-
}
33-
@Override
34-
public void run() {
35-
try {
36-
HiveMetaStore.main(new String[] { String.valueOf(port) });
37-
} catch (Throwable e) {
38-
e.printStackTrace(System.err);
39-
assert false;
40-
}
41-
}
42-
43-
}
4425
@Override
4526
protected void setUp() throws Exception {
4627
super.setUp();
47-
int port = MetaStoreUtils.findFreePort();
48-
Thread t = new Thread(new RunMS(port));
49-
t.setDaemon(true);
50-
t.start();
51-
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
28+
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore());
5229
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
53-
Thread.sleep(30000);
5430
}
5531
}

‎metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1176,22 +1176,38 @@ public static void makeDir(Path path, HiveConf hiveConf) throws MetaException {
11761176
} catch (IOException e) {
11771177
throw new MetaException("Unable to : " + path);
11781178
}
1179+
}
11791180

1181+
public static int startMetaStore() throws Exception {
1182+
return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), null);
11801183
}
11811184

1182-
public static void startMetaStore(final int port,
1183-
final HadoopThriftAuthBridge bridge) throws Exception {
1184-
startMetaStore(port, bridge, new HiveConf(HMSHandler.class));
1185+
public static int startMetaStore(final HadoopThriftAuthBridge bridge, HiveConf conf) throws Exception {
1186+
int port = findFreePort();
1187+
startMetaStore(port, bridge, conf);
1188+
return port;
1189+
}
1190+
1191+
public static int startMetaStore(HiveConf conf) throws Exception {
1192+
return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf);
1193+
}
1194+
1195+
public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception {
1196+
startMetaStore(port, bridge, null);
11851197
}
11861198

11871199
public static void startMetaStore(final int port,
1188-
final HadoopThriftAuthBridge bridge, final HiveConf hiveConf)
1200+
final HadoopThriftAuthBridge bridge, HiveConf hiveConf)
11891201
throws Exception{
1202+
if (hiveConf == null) {
1203+
hiveConf = new HiveConf(HMSHandler.class);
1204+
}
1205+
final HiveConf finalHiveConf = hiveConf;
11901206
Thread thread = new Thread(new Runnable() {
11911207
@Override
11921208
public void run() {
11931209
try {
1194-
HiveMetaStore.startMetaStore(port, bridge, hiveConf);
1210+
HiveMetaStore.startMetaStore(port, bridge, finalHiveConf);
11951211
} catch (Throwable e) {
11961212
LOG.error("Metastore Thrift Server threw an exception...",e);
11971213
}

‎metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ public class TestHiveMetaStoreGetMetaConf {
4040
public ExpectedException thrown = ExpectedException.none();
4141

4242
private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStoreGetMetaConf.class);
43-
private static final String msPort = "20103";
4443
private static HiveConf hiveConf;
4544
private static SecurityManager securityManager;
4645

@@ -65,20 +64,6 @@ public void checkExit(int status) {
6564
}
6665
}
6766

68-
private static class RunMS implements Runnable {
69-
70-
@Override
71-
public void run() {
72-
try {
73-
HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf",
74-
"hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName(),
75-
"--hiveconf", "hive.metastore.try.direct.sql.ddl=false"});
76-
} catch (Throwable t) {
77-
LOG.error("Exiting. Got exception from metastore: ", t);
78-
}
79-
}
80-
}
81-
8267
@AfterClass
8368
public static void tearDown() throws Exception {
8469
LOG.info("Shutting down metastore.");
@@ -90,7 +75,11 @@ public static void startMetaStoreServer() throws Exception {
9075

9176
securityManager = System.getSecurityManager();
9277
System.setSecurityManager(new NoExitSecurityManager());
93-
78+
HiveConf metastoreConf = new HiveConf();
79+
metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname,
80+
MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class);
81+
metastoreConf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL, false);
82+
int msPort = MetaStoreUtils.startMetaStore(metastoreConf);
9483
hiveConf = new HiveConf(TestHiveMetaStoreGetMetaConf.class);
9584
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:"
9685
+ msPort);
@@ -101,8 +90,6 @@ public static void startMetaStoreServer() throws Exception {
10190

10291
System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
10392
System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
104-
105-
new Thread(new RunMS()).start();
10693
}
10794

10895
@Before

‎metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
public class TestHiveMetaStorePartitionSpecs {
5050

5151
private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStorePartitionSpecs.class);
52-
private static final String msPort = "20102";
52+
private static int msPort;
5353
private static HiveConf hiveConf;
5454
private static SecurityManager securityManager;
5555

@@ -73,18 +73,6 @@ public void checkExit(int status) {
7373
}
7474
}
7575

76-
private static class RunMS implements Runnable {
77-
78-
@Override
79-
public void run() {
80-
try {
81-
HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf",
82-
"hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName()});
83-
} catch (Throwable t) {
84-
LOG.error("Exiting. Got exception from metastore: ", t);
85-
}
86-
}
87-
}
8876

8977
@AfterClass
9078
public static void tearDown() throws Exception {
@@ -95,10 +83,10 @@ public static void tearDown() throws Exception {
9583
@BeforeClass
9684
public static void startMetaStoreServer() throws Exception {
9785

98-
Thread t = new Thread(new RunMS());
99-
t.start();
100-
Thread.sleep(10000);
101-
86+
HiveConf metastoreConf = new HiveConf();
87+
metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname,
88+
MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class);
89+
msPort = MetaStoreUtils.startMetaStore(metastoreConf);
10290
securityManager = System.getSecurityManager();
10391
System.setSecurityManager(new NoExitSecurityManager());
10492
hiveConf = new HiveConf(TestHiveMetaStorePartitionSpecs.class);

‎ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java‎

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import java.net.ServerSocket;
2323

2424
import org.apache.hadoop.hive.conf.HiveConf;
25-
import org.apache.hadoop.hive.metastore.HiveMetaStore;
25+
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
2626
import org.apache.hadoop.util.StringUtils;
2727

2828
/**
@@ -33,44 +33,14 @@
3333
*/
3434
public class TestHiveRemote extends TestHive {
3535

36-
/**
37-
* Starts a remote metastore
38-
*/
39-
private static class RunMS implements Runnable {
40-
String port;
41-
42-
public RunMS(String port) {
43-
this.port = port;
44-
}
45-
46-
@Override
47-
public void run() {
48-
try {
49-
HiveMetaStore.main(new String[] { port });
50-
} catch (Throwable e) {
51-
e.printStackTrace(System.err);
52-
assert false;
53-
}
54-
}
55-
56-
}
57-
5836
/**
5937
* Start a remote metastore and initialize a Hive object pointing at it.
6038
*/
6139
@Override
6240
protected void setUp() throws Exception {
6341
super.setUp();
6442
hiveConf = new HiveConf(this.getClass());
65-
String port = findFreePort();
66-
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
67-
68-
Thread t = new Thread(new RunMS(port));
69-
t.start();
70-
71-
// Wait a little bit for the metastore to start.
72-
Thread.sleep(5000);
73-
43+
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore());
7444

7545
try {
7646
hm = Hive.get(hiveConf);

0 commit comments

Comments
 (0)