Skip to content

Commit 3df6b2a

Browse files
JingsongLiJingsongLi
committed
[core] Fix Add min_partition_stats and max_partition_stats columns to manifests system table
1 parent 49e0933 commit 3df6b2a

File tree

3 files changed

+20
-19
lines changed

3 files changed

+20
-19
lines changed

‎docs/content/concepts/system-tables.md

+5-5
Original file line numberDiff line numberDiff line change
@@ -282,8 +282,8 @@ SELECT * FROM my_table$manifests;
282282
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
283283
| file_name | file_size | num_added_files | num_deleted_files | schema_id | min_partition_stats | max_partition_stats |
284284
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
285-
| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | {20230315, 00} | {20230315, 20} |
286-
| manifest-f4dcab43-ef6b-4713... | 1648 | 1 | 0 | 0 | {20230115, 00} | {20230316, 23} |
285+
| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | [20230315, 00] | [20230315, 20] |
286+
| manifest-f4dcab43-ef6b-4713... | 1648 | 1 | 0 | 0 | [20230115, 00] | [20230316, 23] |
287287
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
288288
2 rows in set
289289
*/
@@ -294,7 +294,7 @@ SELECT * FROM my_table$manifests /*+ OPTIONS('scan.snapshot-id'='1') */;
294294
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
295295
| file_name | file_size | num_added_files | num_deleted_files | schema_id | min_partition_stats | max_partition_stats |
296296
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
297-
| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | {20230315, 00} | {20230315, 20} |
297+
| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | [20230315, 00] | [20230315, 20] |
298298
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
299299
1 rows in set
300300
*/
@@ -305,7 +305,7 @@ SELECT * FROM my_table$manifests /*+ OPTIONS('scan.tag-name'='tag1') */;
305305
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
306306
| file_name | file_size | num_added_files | num_deleted_files | schema_id | min_partition_stats | max_partition_stats |
307307
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
308-
| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | {20230315, 00} | {20230315, 20} |
308+
| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | [20230315, 00] | [20230315, 20] |
309309
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
310310
1 rows in set
311311
*/
@@ -316,7 +316,7 @@ SELECT * FROM my_table$manifests /*+ OPTIONS('scan.timestamp-millis'='1678883047
316316
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
317317
| file_name | file_size | num_added_files | num_deleted_files | schema_id | min_partition_stats | max_partition_stats |
318318
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
319-
| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | {20230315, 00} | {20230315, 20} |
319+
| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | [20230315, 00] | [20230315, 20] |
320320
+--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+
321321
1 rows in set
322322
*/

‎paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java

+13-12
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020

2121
import org.apache.paimon.CoreOptions;
2222
import org.apache.paimon.Snapshot;
23-
import org.apache.paimon.casting.CastExecutor;
24-
import org.apache.paimon.casting.CastExecutors;
2523
import org.apache.paimon.data.BinaryString;
2624
import org.apache.paimon.data.GenericRow;
2725
import org.apache.paimon.data.InternalRow;
@@ -46,6 +44,7 @@
4644
import org.apache.paimon.utils.FileStorePathFactory;
4745
import org.apache.paimon.utils.IteratorRecordReader;
4846
import org.apache.paimon.utils.ProjectedRow;
47+
import org.apache.paimon.utils.RowDataToObjectArrayConverter;
4948
import org.apache.paimon.utils.SerializationUtils;
5049

5150
import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators;
@@ -58,6 +57,7 @@
5857
import java.util.Iterator;
5958
import java.util.List;
6059
import java.util.Map;
60+
import java.util.function.Function;
6161

6262
import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER;
6363

@@ -186,16 +186,17 @@ public RecordReader<InternalRow> createReader(Split split) {
186186
}
187187
List<ManifestFileMeta> manifestFileMetas = allManifests(dataTable);
188188

189-
@SuppressWarnings("unchecked")
190-
CastExecutor<InternalRow, BinaryString> partitionCastExecutor =
191-
(CastExecutor<InternalRow, BinaryString>)
192-
CastExecutors.resolveToString(
193-
dataTable.schema().logicalPartitionType());
189+
RowDataToObjectArrayConverter partRowConverter =
190+
new RowDataToObjectArrayConverter(dataTable.schema().logicalPartitionType());
191+
192+
Function<InternalRow, BinaryString> partConverter =
193+
part ->
194+
BinaryString.fromString(
195+
Arrays.toString(partRowConverter.convert(part)));
194196

195197
Iterator<InternalRow> rows =
196198
Iterators.transform(
197-
manifestFileMetas.iterator(),
198-
meta -> toRow(meta, partitionCastExecutor));
199+
manifestFileMetas.iterator(), meta -> toRow(meta, partConverter));
199200
if (readType != null) {
200201
rows =
201202
Iterators.transform(
@@ -209,15 +210,15 @@ public RecordReader<InternalRow> createReader(Split split) {
209210

210211
private InternalRow toRow(
211212
ManifestFileMeta manifestFileMeta,
212-
CastExecutor<InternalRow, BinaryString> partitionCastExecutor) {
213+
Function<InternalRow, BinaryString> partitionCastExecutor) {
213214
return GenericRow.of(
214215
BinaryString.fromString(manifestFileMeta.fileName()),
215216
manifestFileMeta.fileSize(),
216217
manifestFileMeta.numAddedFiles(),
217218
manifestFileMeta.numDeletedFiles(),
218219
manifestFileMeta.schemaId(),
219-
partitionCastExecutor.cast(manifestFileMeta.partitionStats().minValues()),
220-
partitionCastExecutor.cast(manifestFileMeta.partitionStats().maxValues()));
220+
partitionCastExecutor.apply(manifestFileMeta.partitionStats().minValues()),
221+
partitionCastExecutor.apply(manifestFileMeta.partitionStats().maxValues()));
221222
}
222223
}
223224

‎paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -180,14 +180,14 @@ private List<InternalRow> getExpectedResult(long snapshotId) {
180180
manifestFileMeta.schemaId(),
181181
BinaryString.fromString(
182182
String.format(
183-
"{%d}",
183+
"[%d]",
184184
manifestFileMeta
185185
.partitionStats()
186186
.minValues()
187187
.getInt(0))),
188188
BinaryString.fromString(
189189
String.format(
190-
"{%d}",
190+
"[%d]",
191191
manifestFileMeta
192192
.partitionStats()
193193
.maxValues()

0 commit comments

Comments
 (0)