diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml index 1724b8a802..a5588bc104 100644 --- a/amoro-ams/pom.xml +++ b/amoro-ams/pom.xml @@ -33,6 +33,7 @@ 5.17.14 + 18.3.0 false @@ -409,6 +410,13 @@ test + + org.apache.amoro + amoro-format-lance + ${project.version} + test + + org.apache.curator curator-test @@ -618,6 +626,66 @@ + + support-lance-format + + + + org.apache.arrow + arrow-bom + ${lance-arrow-version} + pom + import + + + + + + org.apache.amoro + amoro-format-lance + + + org.apache.arrow + arrow-vector + + + org.apache.arrow + arrow-memory-netty + + + org.apache.arrow + arrow-c-data + + + org.apache.arrow + arrow-dataset + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + enforce-arrow-dependency-convergence + + enforce + + validate + + + + + true + + + + + + + support-all-formats @@ -629,6 +697,10 @@ org.apache.amoro amoro-format-hudi + + org.apache.amoro + amoro-format-lance + diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java index 31c8240cae..af39ba5330 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java @@ -44,10 +44,18 @@ public class CatalogBuilder { ImmutableMap.of( CATALOG_TYPE_HADOOP, Sets.newHashSet( - TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG, TableFormat.PAIMON, TableFormat.HUDI), + TableFormat.ICEBERG, + TableFormat.MIXED_ICEBERG, + TableFormat.PAIMON, + TableFormat.HUDI, + TableFormat.LANCE), CATALOG_TYPE_FILESYSTEM, Sets.newHashSet( - TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG, TableFormat.PAIMON, TableFormat.HUDI), + TableFormat.ICEBERG, + TableFormat.MIXED_ICEBERG, + TableFormat.PAIMON, + TableFormat.HUDI, + TableFormat.LANCE), CATALOG_TYPE_GLUE, Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG), CATALOG_TYPE_REST, diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java index d688f9e29e..c34bd18651 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java @@ -20,6 +20,7 @@ import static org.apache.amoro.TableFormat.HUDI; import static org.apache.amoro.TableFormat.ICEBERG; +import static org.apache.amoro.TableFormat.LANCE; import static org.apache.amoro.TableFormat.MIXED_HIVE; import static org.apache.amoro.TableFormat.MIXED_ICEBERG; import static org.apache.amoro.TableFormat.PAIMON; @@ -161,6 +162,18 @@ public class CatalogController { CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_HADOOP, PAIMON)); VALIDATE_CATALOGS.add( CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_S3, PAIMON)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE)); + VALIDATE_CATALOGS.add( + CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE)); VALIDATE_CATALOGS.add( CatalogDescriptor.of(CATALOG_TYPE_GLUE, STORAGE_CONFIGS_VALUE_TYPE_S3, ICEBERG)); VALIDATE_CATALOGS.add( diff --git a/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java b/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java index b631c595fa..281911c9ae 100644 --- a/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java +++ b/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java @@ -120,7 +120,8 @@ public AmoroTable loadTable(String database, String table) { TableFormat.MIXED_ICEBERG, TableFormat.ICEBERG, TableFormat.PAIMON, - TableFormat.HUDI) + TableFormat.HUDI, + TableFormat.LANCE) .map( formatCatalog -> { try { @@ -147,7 +148,8 @@ public List listTables(String database) { TableFormat.MIXED_ICEBERG, TableFormat.ICEBERG, TableFormat.PAIMON, - TableFormat.HUDI + TableFormat.HUDI, + TableFormat.LANCE }; Map tableNameToFormat = Maps.newHashMap(); diff --git a/amoro-common/src/main/java/org/apache/amoro/TableFormat.java b/amoro-common/src/main/java/org/apache/amoro/TableFormat.java index 62400267a8..c1179bdad7 100644 --- a/amoro-common/src/main/java/org/apache/amoro/TableFormat.java +++ b/amoro-common/src/main/java/org/apache/amoro/TableFormat.java @@ -42,6 +42,7 @@ public final class TableFormat implements Serializable { /** Open-source table formats */ public static final TableFormat ICEBERG = register("ICEBERG"); + public static final TableFormat LANCE = register("LANCE"); public static final TableFormat MIXED_ICEBERG = register("MIXED_ICEBERG"); public static final TableFormat MIXED_HIVE = register("MIXED_HIVE"); public static final TableFormat PAIMON = register("PAIMON"); diff --git a/amoro-format-lance/pom.xml b/amoro-format-lance/pom.xml new file mode 100755 index 0000000000..a4d7adafe0 --- /dev/null +++ b/amoro-format-lance/pom.xml @@ -0,0 +1,66 @@ + + + + 4.0.0 + + org.apache.amoro + amoro-parent + 0.9-SNAPSHOT + ../pom.xml + + + amoro-format-lance + Amoro Project Lance Format + + + + org.apache.amoro + amoro-common + + + + + org.lance + lance-core + 2.0.0-beta.8 + + + + + org.lance + lance-namespace-core + 0.3.2 + + + org.lance + lance-namespace-apache-client + 0.3.2 + + + org.apache.iceberg + iceberg-aws + + + org.apache.iceberg + iceberg-aliyun + + + + diff --git a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java new file mode 100755 index 0000000000..a78fa1ae35 --- /dev/null +++ b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.lance; + +import org.apache.amoro.FormatCatalog; +import org.apache.amoro.FormatCatalogFactory; +import org.apache.amoro.TableFormat; +import org.apache.amoro.table.TableMetaStore; + +import java.util.HashMap; +import java.util.Map; + +/** Lance format catalog factory. */ +public class LanceCatalogFactory implements FormatCatalogFactory { + + /** TableFormat instance for Lance. */ + public static final TableFormat LANCE = TableFormat.register("LANCE"); + + @Override + public FormatCatalog create( + String catalogName, + String metastoreType, + Map properties, + TableMetaStore metaStore) { + return new LanceDirectoryV1Catalog(catalogName, properties); + } + + @Override + public TableFormat format() { + return LANCE; + } + + @Override + public Map convertCatalogProperties( + String catalogName, String metastoreType, Map unifiedCatalogProperties) { + if (unifiedCatalogProperties == null) { + return null; + } + + return new HashMap<>(unifiedCatalogProperties); + } +} diff --git a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java new file mode 100755 index 0000000000..a751ba8bab --- /dev/null +++ b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceDirectoryV1Catalog.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.lance; + +import org.apache.amoro.AmoroTable; +import org.apache.amoro.FormatCatalog; +import org.apache.amoro.NoSuchDatabaseException; +import org.apache.amoro.NoSuchTableException; +import org.apache.amoro.properties.CatalogMetaProperties; +import org.apache.amoro.table.TableIdentifier; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.lance.Dataset; +import org.lance.namespace.LanceNamespace; +import org.lance.namespace.model.DropTableRequest; +import org.lance.namespace.model.ListTablesRequest; +import org.lance.namespace.model.ListTablesResponse; +import org.lance.namespace.model.TableExistsRequest; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Local catalog implementation for Lance. + * + *

This catalog treats a single root directory as the backing "metastore". Under the configured + * root directory, each immediate subdirectory whose name ends with ".lance" is treated as a Lance + * dataset. All tables live in a single logical database named "default". + */ +public class LanceDirectoryV1Catalog implements FormatCatalog { + + private static final String DEFAULT_DATABASE = "default"; + private final String catalogName; + private final Map namespaceProperties; + private final LanceNamespace namespace; + + public LanceDirectoryV1Catalog(String catalogName, Map catalogProperties) { + Preconditions.checkArgument( + catalogProperties != null && !catalogProperties.isEmpty(), + "Catalog properties must be set."); + this.catalogName = catalogName; + this.namespaceProperties = new HashMap<>(catalogProperties); + String root = namespaceProperties.remove(CatalogMetaProperties.KEY_WAREHOUSE); + String s3AccessKey = namespaceProperties.remove(S3FileIOProperties.ACCESS_KEY_ID); + String s3SecretKey = namespaceProperties.remove(S3FileIOProperties.SECRET_ACCESS_KEY); + + Preconditions.checkArgument( + root != null && !root.isEmpty(), "Warehouse must be set in catalogProperties."); + this.namespaceProperties.put("manifest_enabled", "false"); + this.namespaceProperties.put("root", root); + if (s3AccessKey != null) { + this.namespaceProperties.put("storage.access_key_id", s3AccessKey); + } + if (s3SecretKey != null) { + this.namespaceProperties.put("storage.secret_access_key", s3SecretKey); + } + this.namespace = initializeNamespace(new RootAllocator(Long.MAX_VALUE)); + } + + @Override + public List listDatabases() { + return Collections.singletonList(DEFAULT_DATABASE); + } + + @Override + public boolean databaseExists(String database) { + return DEFAULT_DATABASE.equals(database); + } + + @Override + public boolean tableExists(String database, String table) { + validateDatabase(database); + + try { + TableExistsRequest request = new TableExistsRequest().id(Collections.singletonList(table)); + namespace.tableExists(request); + return true; + } catch (RuntimeException e) { + return false; + } + } + + @Override + public void createDatabase(String database) { + throw new UnsupportedOperationException("Creating Lance databases is not supported."); + } + + @Override + public void dropDatabase(String database) { + throw new UnsupportedOperationException("Dropping Lance databases is not supported."); + } + + @Override + public AmoroTable loadTable(String database, String tableName) { + validateDatabase(database); + if (!tableExists(database, tableName)) { + throw new NoSuchTableException("Table: " + database + "." + tableName + " does not exist"); + } + + TableIdentifier identifier = TableIdentifier.of(catalogName, database, tableName); + Dataset dataset = + Dataset.open().namespace(namespace).tableId(Collections.singletonList(tableName)).build(); + return new LanceTable(identifier, dataset, namespaceProperties); + } + + @Override + public boolean dropTable(String database, String table, boolean purge) { + namespace.dropTable(new DropTableRequest().id(Collections.singletonList(table))); + return true; + } + + @Override + public List listTables(String database) { + validateDatabase(database); + return listTablesFromNamespace(); + } + + private List listTablesFromNamespace() { + if (namespace == null) { + return Collections.emptyList(); + } + + ListTablesRequest request = new ListTablesRequest().id(Collections.emptyList()); + ListTablesResponse response = namespace.listTables(request); + if (response == null) { + return Collections.emptyList(); + } else { + response.getTables(); + } + + return new ArrayList<>(response.getTables()); + } + + private LanceNamespace initializeNamespace(BufferAllocator allocator) { + return LanceNamespace.connect("dir", namespaceProperties, allocator); + } + + private void validateDatabase(String database) { + if (!databaseExists(database)) { + throw new NoSuchDatabaseException("Database: " + database + " does not exist"); + } + } +} diff --git a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTable.java b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTable.java new file mode 100755 index 0000000000..465d7ef283 --- /dev/null +++ b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTable.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.lance; + +import org.apache.amoro.AmoroTable; +import org.apache.amoro.TableFormat; +import org.apache.amoro.TableSnapshot; +import org.apache.amoro.table.TableIdentifier; +import org.lance.Dataset; + +import java.util.Collections; +import java.util.Map; + +/** AmoroTable wrapper for a Lance {@link Dataset}. */ +public class LanceTable implements AmoroTable { + + private final TableIdentifier identifier; + private final Dataset dataset; + private final Map tableProperties; + + public LanceTable( + TableIdentifier identifier, Dataset dataset, Map tableProperties) { + this.identifier = identifier; + this.dataset = dataset; + this.tableProperties = + tableProperties == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(tableProperties); + } + + @Override + public TableIdentifier id() { + return identifier; + } + + @Override + public TableFormat format() { + return LanceCatalogFactory.LANCE; + } + + @Override + public Map properties() { + return tableProperties; + } + + @Override + public Dataset originalTable() { + return dataset; + } + + @Override + public TableSnapshot currentSnapshot() { + throw new IllegalStateException("The method is not implemented."); + } +} diff --git a/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTableDescriptor.java b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTableDescriptor.java new file mode 100755 index 0000000000..7bb8ad68b8 --- /dev/null +++ b/amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceTableDescriptor.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.formats.lance; + +import org.apache.amoro.AmoroTable; +import org.apache.amoro.TableFormat; +import org.apache.amoro.process.ProcessStatus; +import org.apache.amoro.table.descriptor.AMSColumnInfo; +import org.apache.amoro.table.descriptor.AmoroSnapshotsOfTable; +import org.apache.amoro.table.descriptor.ConsumerInfo; +import org.apache.amoro.table.descriptor.DDLInfo; +import org.apache.amoro.table.descriptor.FormatTableDescriptor; +import org.apache.amoro.table.descriptor.OperationType; +import org.apache.amoro.table.descriptor.OptimizingProcessInfo; +import org.apache.amoro.table.descriptor.OptimizingTaskInfo; +import org.apache.amoro.table.descriptor.PartitionBaseInfo; +import org.apache.amoro.table.descriptor.PartitionFileBaseInfo; +import org.apache.amoro.table.descriptor.ServerTableMeta; +import org.apache.amoro.table.descriptor.TableSummary; +import org.apache.amoro.table.descriptor.TagOrBranchInfo; +import org.apache.amoro.utils.CommonUtil; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.commons.lang3.tuple.Pair; +import org.lance.Branch; +import org.lance.Dataset; +import org.lance.Fragment; +import org.lance.FragmentMetadata; +import org.lance.ManifestSummary; +import org.lance.Tag; +import org.lance.Version; +import org.lance.fragment.DataFile; +import org.lance.fragment.DeletionFile; + +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; + +/** Table descriptor for Lance tables. */ +public class LanceTableDescriptor implements FormatTableDescriptor { + + private ExecutorService ioExecutor; + + @Override + public void withIoExecutor(ExecutorService ioExecutor) { + this.ioExecutor = ioExecutor; + } + + @Override + public List supportFormat() { + return Collections.singletonList(LanceCatalogFactory.LANCE); + } + + @Override + public ServerTableMeta getTableDetail(AmoroTable amoroTable) { + Dataset dataset = (Dataset) amoroTable.originalTable(); + + ServerTableMeta meta = new ServerTableMeta(); + meta.setTableIdentifier(amoroTable.id()); + meta.setTableType(LanceCatalogFactory.LANCE.name()); + meta.setBaseLocation(dataset.uri()); + + // Schema + List columns = new ArrayList<>(); + Schema schema = dataset.getSchema(); + for (Field field : schema.getFields()) { + AMSColumnInfo columnInfo = new AMSColumnInfo(); + columnInfo.setField(field.getName()); + columnInfo.setType(field.getType().toString()); + columnInfo.setRequired(!field.isNullable()); + columnInfo.setComment(null); + columns.add(columnInfo); + } + meta.setSchema(columns); + meta.setPkList(Collections.emptyList()); + meta.setPartitionColumnList(Collections.emptyList()); + meta.setProperties(amoroTable.properties()); + + Version version = dataset.getVersion(); + ManifestSummary summary = version.getManifestSummary(); + + long totalFilesSize = summary.getTotalFilesSize(); + long totalDataFiles = summary.getTotalDataFiles(); + long totalDeletionFiles = summary.getTotalDeletionFiles(); + long totalFileCount = totalDataFiles + totalDeletionFiles; + long totalRows = summary.getTotalRows(); + + String totalSizeHuman = CommonUtil.byteToXB(totalFilesSize); + String averageFileSizeHuman = + CommonUtil.byteToXB(totalFileCount == 0 ? 0 : totalFilesSize / totalFileCount); + + // Table summary + String tableFormatDisplay = "Lance"; + TableSummary tableSummary = + new TableSummary( + totalFileCount, totalSizeHuman, averageFileSizeHuman, totalRows, tableFormatDisplay); + meta.setTableSummary(tableSummary); + + // Base metrics (data files only) + Map baseMetrics = new HashMap<>(); + baseMetrics.put("totalSize", CommonUtil.byteToXB(totalFilesSize)); + baseMetrics.put("fileCount", totalDataFiles); + baseMetrics.put( + "averageFileSize", + CommonUtil.byteToXB(totalDataFiles == 0 ? 0 : totalFilesSize / totalDataFiles)); + meta.setBaseMetrics(baseMetrics); + + // Change metrics (deletion files if present) + Map changeMetrics = new HashMap<>(); + if (totalDeletionFiles > 0) { + changeMetrics.put("deletionFileCount", totalDeletionFiles); + changeMetrics.put("deletedRows", summary.getTotalDeletionFileRows()); + } + meta.setChangeMetrics(changeMetrics); + + return meta; + } + + @Override + public List getSnapshots( + AmoroTable amoroTable, String ref, OperationType operationType) { + Dataset dataset = (Dataset) amoroTable.originalTable(); + List versions = dataset.listVersions(); + + if (ioExecutor == null) { + List snapshots = new ArrayList<>(); + for (Version version : versions) { + snapshots.add(buildSnapshot(version)); + } + return snapshots; + } + + List> futures = new ArrayList<>(); + for (Version version : versions) { + futures.add(CompletableFuture.supplyAsync(() -> buildSnapshot(version), ioExecutor)); + } + + List snapshots = new ArrayList<>(); + for (CompletableFuture future : futures) { + try { + snapshots.add(future.get()); + } catch (InterruptedException e) { + // ignore + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + } + return snapshots; + } + + @Override + public List getSnapshotDetail( + AmoroTable amoroTable, String snapshotId, String ref) { + Dataset dataset = (Dataset) amoroTable.originalTable(); + long versionId; + try { + versionId = Long.parseLong(snapshotId); + } catch (NumberFormatException e) { + return Collections.emptyList(); + } + + try (Dataset versionDataset = dataset.checkoutVersion(versionId)) { + Version version = versionDataset.getVersion(); + long commitTime = toMillis(version.getDataTime()); + List files = new ArrayList<>(); + + for (Fragment fragment : versionDataset.getFragments()) { + FragmentMetadata metadata = fragment.metadata(); + for (DataFile dataFile : metadata.getFiles()) { + String path = dataFile.getPath(); + long size = dataFile.getFileSizeBytes() == null ? 0L : dataFile.getFileSizeBytes(); + files.add( + new PartitionFileBaseInfo(snapshotId, "DATA_FILE", commitTime, "", 0, path, size)); + } + + DeletionFile deletionFile = metadata.getDeletionFile(); + if (deletionFile != null) { + files.add( + new PartitionFileBaseInfo(snapshotId, "DELETION_FILE", commitTime, "", 0, "", 0L)); + } + } + + return files; + } + } + + @Override + public List getTableOperations(AmoroTable amoroTable) { + return Collections.emptyList(); + } + + @Override + public List getTablePartitions(AmoroTable amoroTable) { + // Lance tables are treated as non-partitioned for now. + return Collections.emptyList(); + } + + @Override + public List getTableFiles( + AmoroTable amoroTable, String partition, Integer specId) { + Dataset dataset = (Dataset) amoroTable.originalTable(); + Version version = dataset.getVersion(); + long commitTime = toMillis(version.getDataTime()); + + List files = new ArrayList<>(); + for (Fragment fragment : dataset.getFragments()) { + FragmentMetadata metadata = fragment.metadata(); + for (DataFile dataFile : metadata.getFiles()) { + String path = dataFile.getPath(); + long size = dataFile.getFileSizeBytes() == null ? 0L : dataFile.getFileSizeBytes(); + files.add( + new PartitionFileBaseInfo( + String.valueOf(version.getId()), "DATA_FILE", commitTime, "", 0, path, size)); + } + } + return files; + } + + @Override + public Pair, Integer> getOptimizingProcessesInfo( + AmoroTable amoroTable, String type, ProcessStatus status, int limit, int offset) { + return Pair.of(Collections.emptyList(), 0); + } + + @Override + public Map getTableOptimizingTypes(AmoroTable amoroTable) { + return Collections.emptyMap(); + } + + @Override + public List getOptimizingTaskInfos( + AmoroTable amoroTable, String processId) { + return Collections.emptyList(); + } + + @Override + public List getTableTags(AmoroTable amoroTable) { + Dataset dataset = (Dataset) amoroTable.originalTable(); + List tags = dataset.tags().list(); + + List infos = new ArrayList<>(); + for (Tag tag : tags) { + TagOrBranchInfo info = + new TagOrBranchInfo(tag.getName(), tag.getVersion(), -1, 0L, 0L, TagOrBranchInfo.TAG); + infos.add(info); + } + return infos; + } + + @Override + public List getTableBranches(AmoroTable amoroTable) { + Dataset dataset = (Dataset) amoroTable.originalTable(); + List branches = dataset.branches().list(); + + List branchInfos = new ArrayList<>(); + for (Branch branch : branches) { + TagOrBranchInfo info = + new TagOrBranchInfo( + branch.getName(), branch.getParentVersion() + 1, -1, 0L, 0L, TagOrBranchInfo.BRANCH); + branchInfos.add(info); + } + branchInfos.add(TagOrBranchInfo.MAIN_BRANCH); + return branchInfos; + } + + @Override + public List getTableConsumerInfos(AmoroTable amoroTable) { + return Collections.emptyList(); + } + + private AmoroSnapshotsOfTable buildSnapshot(Version version) { + ManifestSummary summary = version.getManifestSummary(); + AmoroSnapshotsOfTable snapshot = new AmoroSnapshotsOfTable(); + snapshot.setSnapshotId(String.valueOf(version.getId())); + snapshot.setCommitTime(toMillis(version.getDataTime())); + snapshot.setFileCount((int) summary.getTotalDataFiles()); + snapshot.setFileSize(summary.getTotalFilesSize()); + snapshot.setRecords(summary.getTotalRows()); + snapshot.setOperation("WRITE"); + + Map filesSummary = new HashMap<>(); + filesSummary.put("data-files", String.valueOf(summary.getTotalDataFiles())); + filesSummary.put("delta-files", String.valueOf(summary.getTotalDeletionFiles())); + filesSummary.put("changelogs", "0"); + snapshot.setFilesSummaryForChart(filesSummary); + + return snapshot; + } + + private long toMillis(ZonedDateTime time) { + if (time == null) { + return 0L; + } + return time.toInstant().toEpochMilli(); + } +} diff --git a/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.FormatCatalogFactory b/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.FormatCatalogFactory new file mode 100644 index 0000000000..34b9298ab0 --- /dev/null +++ b/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.FormatCatalogFactory @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# * +# http://www.apache.org/licenses/LICENSE-2.0 +# * +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.amoro.formats.lance.LanceCatalogFactory \ No newline at end of file diff --git a/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.table.descriptor.FormatTableDescriptor b/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.table.descriptor.FormatTableDescriptor new file mode 100644 index 0000000000..14dc08db26 --- /dev/null +++ b/amoro-format-lance/src/main/resources/META-INF/services/org.apache.amoro.table.descriptor.FormatTableDescriptor @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# * +# http://www.apache.org/licenses/LICENSE-2.0 +# * +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.amoro.formats.lance.LanceTableDescriptor \ No newline at end of file diff --git a/amoro-web/src/assets/icons/svg/lance.svg b/amoro-web/src/assets/icons/svg/lance.svg new file mode 100755 index 0000000000..6fc7251687 --- /dev/null +++ b/amoro-web/src/assets/icons/svg/lance.svg @@ -0,0 +1,25 @@ + + + + + + diff --git a/amoro-web/src/types/common.type.ts b/amoro-web/src/types/common.type.ts index 073c78c58c..c5a29c8675 100644 --- a/amoro-web/src/types/common.type.ts +++ b/amoro-web/src/types/common.type.ts @@ -377,6 +377,7 @@ export enum tableTypeIconMap { HIVE = 'hive', PAIMON = 'paimon', HUDI = 'hudi', + LANCE = 'lance', } export type ILineChartOriginalData = Record> diff --git a/pom.xml b/pom.xml index c5c72a856a..71e5eb1511 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,7 @@ amoro-format-mixed amoro-format-hudi amoro-format-paimon + amoro-format-lance dist @@ -196,6 +197,12 @@ ${project.version} + + org.apache.amoro + amoro-format-lance + ${project.version} + + org.apache.amoro amoro-mixed-hive