From a271c4c98167a05fc3f8ab4de2dc204db182b77e Mon Sep 17 00:00:00 2001 From: herefree <841043203@qq.com> Date: Wed, 8 Jan 2025 13:17:19 +0800 Subject: [PATCH 1/2] fix drop table in HiveCatalog and jdbcCatalog when file path is not exists. --- .../paimon/catalog/AbstractCatalog.java | 13 +++--- .../org/apache/paimon/jdbc/JdbcCatalog.java | 5 +++ .../apache/paimon/jdbc/JdbcCatalogTest.java | 29 +++++++++++++ .../org/apache/paimon/hive/HiveCatalog.java | 9 ++++ .../apache/paimon/hive/HiveCatalogTest.java | 43 +++++++++++++++++++ 5 files changed, 93 insertions(+), 6 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 439f456efb4c..c8e9f5cd5681 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -268,16 +268,13 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) throws TableNotExistException { checkNotBranch(identifier, "dropTable"); checkNotSystemTable(identifier, "dropTable"); - - try { - getTable(identifier); - } catch (TableNotExistException e) { + if (!tableExists(identifier)) { if (ignoreIfNotExists) { return; + } else { + throw new TableNotExistException(identifier); } - throw new TableNotExistException(identifier); } - dropTableImpl(identifier); } @@ -598,6 +595,10 @@ public Optional tableSchemaInFileSystem(Path tablePath, String bran }); } + protected boolean tableExists(Identifier identifier) { + return tableExistsInFileSystem(getTableLocation(identifier), DEFAULT_MAIN_BRANCH); + } + /** Table metadata. */ protected static class TableMeta { diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java index 63cb54c180f5..d1b097a218fe 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java @@ -364,6 +364,11 @@ protected TableSchema getDataTableSchema(Identifier identifier) throws TableNotE () -> new RuntimeException("There is no paimon table in " + tableLocation)); } + protected boolean tableExists(Identifier identifier) { + return JdbcUtils.tableExists( + connections, catalogKey, identifier.getDatabaseName(), identifier.getTableName()); + } + @Override public boolean caseSensitive() { return false; diff --git a/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcCatalogTest.java index 51e2bf5c779d..5bfc57a096df 100644 --- a/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcCatalogTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogTestBase; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.table.Table; @@ -33,6 +34,7 @@ import java.io.ByteArrayOutputStream; import java.io.ObjectOutputStream; import java.sql.SQLException; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -117,6 +119,33 @@ public void testSerializeTable() throws Exception { }); } + @Test + public void testDropTable() throws Exception { + String databaseName = "test_db"; + String tableName = "new_table"; + catalog.createDatabase(databaseName, false); + catalog.createTable( + Identifier.create(databaseName, tableName), DEFAULT_TABLE_SCHEMA, false); + JdbcCatalog jdbcCatalog = (JdbcCatalog) catalog; + Path path = jdbcCatalog.getTableLocation(Identifier.create(databaseName, tableName)); + // delete file path + catalog.fileIO().deleteDirectoryQuietly(path); + + assertThatThrownBy(() -> catalog.getTable(Identifier.create(databaseName, tableName))) + .isInstanceOf(RuntimeException.class) + .hasMessage( + "There is no paimon table in " + + jdbcCatalog.getTableLocation( + Identifier.create(databaseName, tableName))); + + List tablesCon = catalog.listTables(databaseName); + assertThat(tablesCon).contains(tableName); + + catalog.dropTable(Identifier.create(databaseName, tableName), false); + List tables = catalog.listTables(databaseName); + assertThat(tableName).isNotIn(tables); + } + @Test public void testAlterDatabase() throws Exception { this.alterDatabaseWhenSupportAlter(); diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index a213909beb10..75a82137da78 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -1442,6 +1442,15 @@ public static String possibleHiveConfPath() { return System.getenv("HIVE_CONF_DIR"); } + protected boolean tableExists(Identifier identifier) { + try { + getHmsTable(identifier); + } catch (Exception e) { + return false; + } + return true; + } + public int getBatchGetTableSize() { try { int size = diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index e733ec16c839..164f98e632c0 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.catalog.CatalogTestBase; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.client.ClientPool; +import org.apache.paimon.fs.Path; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; @@ -60,6 +61,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; /** Tests for {@link HiveCatalog}. */ @@ -277,6 +279,47 @@ public void testAlterHiveTableParameters() { } } + @Test + public void testDropTable() { + try { + String databaseName = "test_db"; + catalog.createDatabase(databaseName, false); + String tableName = "new_table"; + Map options = new HashMap<>(); + Schema addHiveTableParametersSchema = + new Schema( + Lists.newArrayList( + new DataField(0, "pk", DataTypes.INT()), + new DataField(1, "col1", DataTypes.STRING()), + new DataField(2, "col2", DataTypes.STRING())), + Collections.emptyList(), + Collections.emptyList(), + options, + ""); + + catalog.createTable( + Identifier.create(databaseName, tableName), + addHiveTableParametersSchema, + false); + // delete file path + HiveCatalog hiveCatalog = (HiveCatalog) catalog; + Path path = hiveCatalog.getTableLocation(Identifier.create(databaseName, tableName)); + catalog.fileIO().deleteDirectoryQuietly(path); + assertThrows( + Catalog.TableNotExistException.class, + () -> catalog.getTable(Identifier.create(databaseName, tableName))); + + List tablesCon = catalog.listTables(databaseName); + assertThat(tablesCon).contains(tableName); + + catalog.dropTable(Identifier.create(databaseName, tableName), false); + List tables = catalog.listTables(databaseName); + assertThat(tableName).isNotIn(tables); + } catch (Exception e) { + fail("Test failed due to exception: " + e.getMessage()); + } + } + @Test public void testListTablesLock() { try { From 8704d34cefaa3f03c2a5cb0c0fc9f7578117d552 Mon Sep 17 00:00:00 2001 From: herefree <841043203@qq.com> Date: Wed, 8 Jan 2025 15:37:12 +0800 Subject: [PATCH 2/2] fix --- .../java/org/apache/paimon/hive/HiveCatalog.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 75a82137da78..d93ae5c1dd23 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -1444,11 +1444,18 @@ public static String possibleHiveConfPath() { protected boolean tableExists(Identifier identifier) { try { - getHmsTable(identifier); - } catch (Exception e) { - return false; + return clients.run( + client -> + client.tableExists( + identifier.getDatabaseName(), identifier.getTableName())); + } catch (TException e) { + throw new RuntimeException( + "Cannot determine if table " + identifier.getFullName() + "exists.", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException( + "Interrupted in call to tableExists " + identifier.getFullName(), e); } - return true; } public int getBatchGetTableSize() {