Skip to content

Commit

Permalink
[fix](catalog) fix npe after replaying the external catalog (#45756)
Browse files Browse the repository at this point in the history
### What problem does this PR solve?

Related PR: #45433

Problem Summary:
the `confLock` should be created after replaying in `gsonPostProcess()`
of `ExternalCatalog`, or it will be null.
  • Loading branch information
morningman authored Dec 22, 2024
1 parent f6071a3 commit 82d021b
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ public abstract class ExternalCatalog
protected PreExecutionAuthenticator preExecutionAuthenticator;

private volatile Configuration cachedConf = null;
private final byte[] confLock = new byte[0];
private byte[] confLock = new byte[0];

public ExternalCatalog() {
}
Expand Down Expand Up @@ -784,6 +784,7 @@ public void gsonPostProcess() throws IOException {
}
}
this.propLock = new byte[0];
this.confLock = new byte[0];
this.initialized = false;
setDefaultPropsIfMissing(true);
if (tableAutoAnalyzePolicy == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,26 +22,31 @@
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.test.TestExternalCatalog;
import org.apache.doris.mysql.privilege.Auth;
import org.apache.doris.meta.MetaContext;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.utframe.TestWithFeService;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ExternalCatalogTest extends TestWithFeService {
private static Auth auth;
private static Env env;
private Env env;
private CatalogMgr mgr;
private ConnectContext rootCtx;

Expand All @@ -51,7 +56,6 @@ protected void runBeforeAll() throws Exception {
mgr = Env.getCurrentEnv().getCatalogMgr();
rootCtx = createDefaultCtx();
env = Env.getCurrentEnv();
auth = env.getAuth();
// 1. create test catalog
CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
"create catalog test1 properties(\n"
Expand Down Expand Up @@ -244,4 +248,32 @@ public Map<String, Map<String, List<Column>>> getMetadata() {
return MOCKED_META;
}
}

@Test
public void testSerialization() throws Exception {
MetaContext metaContext = new MetaContext();
metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT);
metaContext.setThreadLocalInfo();

// 1. Write objects to file
File file = new File("./external_catalog_persist_test.dat");
file.createNewFile();
DataOutputStream dos = new DataOutputStream(Files.newOutputStream(file.toPath()));

TestExternalCatalog ctl = (TestExternalCatalog) mgr.getCatalog("test1");
ctl.write(dos);
dos.flush();
dos.close();

// 2. Read objects from file
DataInputStream dis = new DataInputStream(Files.newInputStream(file.toPath()));

TestExternalCatalog ctl2 = (TestExternalCatalog) ExternalCatalog.read(dis);
Configuration conf = ctl2.getConfiguration();
Assertions.assertNotNull(conf);

// 3. delete files
dis.close();
file.delete();
}
}

0 comments on commit 82d021b

Please sign in to comment.