Merge branch 'main' into main

apache · Jan 13, 2025 · 80e1ac0 · 80e1ac0
2 parents ce4dcb5 + e4151e9
commit 80e1ac0
Show file tree

Hide file tree

Showing 41 changed files with 1,774 additions and 472 deletions.
diff --git a/...on/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java b/...on/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java
@@ -32,25 +32,41 @@ public class PaimonPropertiesUtils {
   // will only need to set the configuration 'catalog-backend' in Gravitino and Gravitino will
   // change it to `catalogType` automatically and pass it to Paimon.
   public static final Map<String, String> GRAVITINO_CONFIG_TO_PAIMON;
+  public static final Map<String, String> PAIMON_CATALOG_CONFIG_TO_GRAVITINO;
 
   static {
-    Map<String, String> map = new HashMap();
-    map.put(PaimonConstants.CATALOG_BACKEND, PaimonConstants.CATALOG_BACKEND);
-    map.put(PaimonConstants.GRAVITINO_JDBC_DRIVER, PaimonConstants.GRAVITINO_JDBC_DRIVER);
-    map.put(PaimonConstants.GRAVITINO_JDBC_USER, PaimonConstants.PAIMON_JDBC_USER);
-    map.put(PaimonConstants.GRAVITINO_JDBC_PASSWORD, PaimonConstants.PAIMON_JDBC_PASSWORD);
-    map.put(PaimonConstants.URI, PaimonConstants.URI);
-    map.put(PaimonConstants.WAREHOUSE, PaimonConstants.WAREHOUSE);
-    map.put(PaimonConstants.CATALOG_BACKEND_NAME, PaimonConstants.CATALOG_BACKEND_NAME);
+    Map<String, String> gravitinoConfigToPaimon = new HashMap<>();
+    Map<String, String> paimonCatalogConfigToGravitino = new HashMap<>();
+    gravitinoConfigToPaimon.put(PaimonConstants.CATALOG_BACKEND, PaimonConstants.CATALOG_BACKEND);
+    gravitinoConfigToPaimon.put(
+        PaimonConstants.GRAVITINO_JDBC_DRIVER, PaimonConstants.GRAVITINO_JDBC_DRIVER);
+    gravitinoConfigToPaimon.put(
+        PaimonConstants.GRAVITINO_JDBC_USER, PaimonConstants.PAIMON_JDBC_USER);
+    gravitinoConfigToPaimon.put(
+        PaimonConstants.GRAVITINO_JDBC_PASSWORD, PaimonConstants.PAIMON_JDBC_PASSWORD);
+    gravitinoConfigToPaimon.put(PaimonConstants.URI, PaimonConstants.URI);
+    gravitinoConfigToPaimon.put(PaimonConstants.WAREHOUSE, PaimonConstants.WAREHOUSE);
+    gravitinoConfigToPaimon.put(
+        PaimonConstants.CATALOG_BACKEND_NAME, PaimonConstants.CATALOG_BACKEND_NAME);
     // S3
-    map.put(S3Properties.GRAVITINO_S3_ENDPOINT, PaimonConstants.S3_ENDPOINT);
-    map.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, PaimonConstants.S3_ACCESS_KEY);
-    map.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, PaimonConstants.S3_SECRET_KEY);
+    gravitinoConfigToPaimon.put(S3Properties.GRAVITINO_S3_ENDPOINT, PaimonConstants.S3_ENDPOINT);
+    gravitinoConfigToPaimon.put(
+        S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, PaimonConstants.S3_ACCESS_KEY);
+    gravitinoConfigToPaimon.put(
+        S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, PaimonConstants.S3_SECRET_KEY);
     // OSS
-    map.put(OSSProperties.GRAVITINO_OSS_ENDPOINT, PaimonConstants.OSS_ENDPOINT);
-    map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, PaimonConstants.OSS_ACCESS_KEY);
-    map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, PaimonConstants.OSS_SECRET_KEY);
-    GRAVITINO_CONFIG_TO_PAIMON = Collections.unmodifiableMap(map);
+    gravitinoConfigToPaimon.put(OSSProperties.GRAVITINO_OSS_ENDPOINT, PaimonConstants.OSS_ENDPOINT);
+    gravitinoConfigToPaimon.put(
+        OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, PaimonConstants.OSS_ACCESS_KEY);
+    gravitinoConfigToPaimon.put(
+        OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, PaimonConstants.OSS_SECRET_KEY);
+    GRAVITINO_CONFIG_TO_PAIMON = Collections.unmodifiableMap(gravitinoConfigToPaimon);
+    gravitinoConfigToPaimon.forEach(
+        (key, value) -> {
+          paimonCatalogConfigToGravitino.put(value, key);
+        });
+    PAIMON_CATALOG_CONFIG_TO_GRAVITINO =
+        Collections.unmodifiableMap(paimonCatalogConfigToGravitino);
   }
 
   /**

diff --git a/...oris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java b/...oris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java
@@ -567,10 +567,6 @@ protected String generateAlterTableSql(
       alterSql.add("MODIFY COMMENT \"" + newComment + "\"");
     }
 
-    if (!setProperties.isEmpty()) {
-      alterSql.add(generateTableProperties(setProperties));
-    }
-
     if (CollectionUtils.isEmpty(alterSql)) {
       return "";
     }
@@ -602,11 +598,14 @@ private String updateColumnNullabilityDefinition(
   }
 
   private String generateTableProperties(List<TableChange.SetProperty> setProperties) {
-    return setProperties.stream()
-        .map(
-            setProperty ->
-                String.format("\"%s\" = \"%s\"", setProperty.getProperty(), setProperty.getValue()))
-        .collect(Collectors.joining(",\n"));
+    String properties =
+        setProperties.stream()
+            .map(
+                setProperty ->
+                    String.format(
+                        "\"%s\" = \"%s\"", setProperty.getProperty(), setProperty.getValue()))
+            .collect(Collectors.joining(",\n"));
+    return "set (" + properties + ")";
   }
 
   private String updateColumnCommentFieldDefinition(

diff --git a/...ris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/...ris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
@@ -577,6 +577,16 @@ void testAlterDorisTable() {
         .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS)
         .untilAsserted(
             () -> assertEquals(4, tableCatalog.loadTable(tableIdentifier).columns().length));
+
+    // set property
+    tableCatalog.alterTable(tableIdentifier, TableChange.setProperty("in_memory", "true"));
+    Awaitility.await()
+        .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS)
+        .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS)
+        .untilAsserted(
+            () ->
+                assertEquals(
+                    "true", tableCatalog.loadTable(tableIdentifier).properties().get("in_memory")));
   }
 
   @Test

diff --git a/...ysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/...ysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java
@@ -106,6 +106,7 @@ protected String generateCreateTableSql(
       }
     }
 
+    validateIndexes(indexes, columns);
     appendIndexesSql(indexes, sqlBuilder);
 
     sqlBuilder.append("\n)");
@@ -642,4 +643,33 @@ private StringBuilder appendColumnDefinition(JdbcColumn column, StringBuilder sq
   private static String quote(String name) {
     return BACK_QUOTE + name + BACK_QUOTE;
   }
+
+  /**
+   * Verify the columns in the index.
+   *
+   * @param columns jdbc column
+   * @param indexes table indexes
+   */
+  private static void validateIndexes(Index[] indexes, JdbcColumn[] columns) {
+    Map<String, JdbcColumn> columnMap =
+        Arrays.stream(columns).collect(Collectors.toMap(JdbcColumn::name, c -> c));
+    for (Index index : indexes) {
+      if (index.type() == Index.IndexType.UNIQUE_KEY) {
+        // the column in the unique index must be not null
+        for (String[] colNames : index.fieldNames()) {
+          JdbcColumn column = columnMap.get(colNames[0]);
+          Preconditions.checkArgument(
+              column != null,
+              "Column %s in the unique index %s does not exist in the table",
+              colNames[0],
+              index.name());
+          Preconditions.checkArgument(
+              !column.nullable(),
+              "Column %s in the unique index %s must be a not null column",
+              colNames[0],
+              index.name());
+        }
+      }
+    }
+  }
 }
diff --git a/...sql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/...sql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java
@@ -1037,6 +1037,27 @@ void testCreateTableIndex() {
     Assertions.assertEquals(2, table.index().length);
     Assertions.assertNotNull(table.index()[0].name());
     Assertions.assertNotNull(table.index()[1].name());
+
+    Column notNullCol = Column.of("col_6", Types.LongType.get(), "id", true, false, null);
+    Exception exception =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                tableCatalog.createTable(
+                    tableIdent,
+                    new Column[] {notNullCol},
+                    table_comment,
+                    properties,
+                    Transforms.EMPTY_TRANSFORM,
+                    Distributions.NONE,
+                    new SortOrder[0],
+                    new Index[] {
+                      Indexes.of(Index.IndexType.UNIQUE_KEY, null, new String[][] {{"col_6"}}),
+                    }));
+    Assertions.assertTrue(
+        exception
+            .getMessage()
+            .contains("Column col_6 in the unique index null must be a not null column"));
   }
 
   @Test

diff --git a/.../src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java b/.../src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java
@@ -64,7 +64,7 @@ public void testOperationTable() {
             .withName("col_1")
             .withType(VARCHAR)
             .withComment("test_comment")
-            .withNullable(true)
+            .withNullable(false)
             .build());
     columns.add(
         JdbcColumn.builder()
@@ -573,7 +573,7 @@ public void testCreateAndLoadTable() {
         JdbcColumn.builder()
             .withName("col_4")
             .withType(Types.DateType.get())
-            .withNullable(true)
+            .withNullable(false)
             .withComment("date")
             .withDefaultValue(Column.DEFAULT_VALUE_NOT_SET)
             .build());