Skip to content

Commit 39782c8

Browse files
daindain
committed
Run Iceberg connector tests on v2 and v3
1 parent 398cef6 commit 39782c8

11 files changed

+167
-17
lines changed

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergAvroConnectorTest.java renamed to plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergAvroConnectorTest.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,12 @@
1818
import static io.trino.plugin.iceberg.IcebergFileFormat.AVRO;
1919
import static org.junit.jupiter.api.Assumptions.abort;
2020

21-
public class TestIcebergAvroConnectorTest
21+
public abstract class BaseIcebergAvroConnectorTest
2222
extends BaseIcebergConnectorTest
2323
{
24-
public TestIcebergAvroConnectorTest()
24+
public BaseIcebergAvroConnectorTest(int formatVersion)
2525
{
26-
super(AVRO);
26+
super(AVRO, formatVersion);
2727
}
2828

2929
@Override

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -195,13 +195,15 @@ public abstract class BaseIcebergConnectorTest
195195
private static final Pattern WITH_CLAUSE_EXTRACTOR = Pattern.compile(".*(WITH\\s*\\([^)]*\\))\\s*$", Pattern.DOTALL);
196196

197197
protected final IcebergFileFormat format;
198+
protected final int formatVersion;
198199

199200
protected TrinoFileSystem fileSystem;
200201
protected TimeUnit storageTimePrecision;
201202

202-
protected BaseIcebergConnectorTest(IcebergFileFormat format)
203+
protected BaseIcebergConnectorTest(IcebergFileFormat format, int formatVersion)
203204
{
204205
this.format = requireNonNull(format, "format is null");
206+
this.formatVersion = formatVersion;
205207
}
206208

207209
@Override
@@ -217,6 +219,7 @@ protected IcebergQueryRunner.Builder createQueryRunnerBuilder()
217219
return IcebergQueryRunner.builder()
218220
.setIcebergProperties(ImmutableMap.<String, String>builder()
219221
.put("iceberg.file-format", format.name())
222+
.put("iceberg.format-version", String.valueOf(formatVersion))
220223
// Only allow some extra properties. Add "sorted_by" so that we can test that the property is disallowed by the connector explicitly.
221224
.put("iceberg.allowed-extra-properties", "extra.property.one,extra.property.two,extra.property.three,sorted_by")
222225
// Allows testing the sorting writer flushing to the file system with smaller tables
@@ -414,7 +417,7 @@ public void testShowCreateTable()
414417
")\n" +
415418
"WITH (\n" +
416419
" format = '" + format.name() + "',\n" +
417-
" format_version = 2,\n" +
420+
" format_version = " + formatVersion + ",\n" +
418421
" location = '\\E.*/tpch/orders-.*\\Q'\n" +
419422
")\\E");
420423
}
@@ -1994,11 +1997,12 @@ private void testCreateTableLikeForFormat(IcebergFileFormat otherFormat)
19941997
"""
19951998
WITH (
19961999
format = '%s',
1997-
format_version = 2,
2000+
format_version = %s,
19982001
location = '%s',
19992002
partitioning = ARRAY['adate']
20002003
)""",
20012004
format,
2005+
formatVersion,
20022006
tempDirPath));
20032007

20042008
assertUpdate("CREATE TABLE test_create_table_like_copy0 (LIKE test_create_table_like_original, col2 INTEGER)");
@@ -2010,21 +2014,23 @@ private void testCreateTableLikeForFormat(IcebergFileFormat otherFormat)
20102014
"""
20112015
WITH (
20122016
format = '%s',
2013-
format_version = 2,
2017+
format_version = %s,
20142018
location = '%s'
20152019
)""",
20162020
format,
2021+
formatVersion,
20172022
getTableLocation("test_create_table_like_copy1")));
20182023

20192024
assertUpdate("CREATE TABLE test_create_table_like_copy2 (LIKE test_create_table_like_original EXCLUDING PROPERTIES)");
20202025
assertThat(getTablePropertiesString("test_create_table_like_copy2")).isEqualTo(format(
20212026
"""
20222027
WITH (
20232028
format = '%s',
2024-
format_version = 2,
2029+
format_version = %s,
20252030
location = '%s'
20262031
)""",
20272032
format,
2033+
formatVersion,
20282034
getTableLocation("test_create_table_like_copy2")));
20292035
assertUpdate("DROP TABLE test_create_table_like_copy2");
20302036

@@ -5418,7 +5424,7 @@ public void testOptimize()
54185424
.hasSizeGreaterThan(workerCount);
54195425

54205426
// For optimize we need to set task_min_writer_count to 1, otherwise it will create more than one file.
5421-
computeActual(withSingleWriterPerTask(getSession()), "ALTER TABLE " + tableName + " EXECUTE OPTIMIZE");
5427+
computeActual(withSingleWriterPerTask(getSession()), "ALTER TABLE " + tableName + " EXECUTE OPTIMIZE");
54225428
assertThat(query("SELECT sum(key), listagg(value, ' ') WITHIN GROUP (ORDER BY key) FROM " + tableName))
54235429
.matches("VALUES (BIGINT '65', VARCHAR 'eleven zwölf trzynaście quatorze пʼятнадцять')");
54245430
List<String> updatedFiles = getActiveFiles(tableName);

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMinioOrcConnectorTest.java renamed to plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioOrcConnectorTest.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,14 @@
4343
* Iceberg connector test ORC and with S3-compatible storage (but without real metastore).
4444
*/
4545
@Execution(SAME_THREAD)
46-
public class TestIcebergMinioOrcConnectorTest
46+
public abstract class BaseIcebergMinioOrcConnectorTest
4747
extends BaseIcebergConnectorTest
4848
{
4949
private final String bucketName = "test-iceberg-orc-" + randomNameSuffix();
5050

51-
public TestIcebergMinioOrcConnectorTest()
51+
public BaseIcebergMinioOrcConnectorTest(int formatVersion)
5252
{
53-
super(ORC);
53+
super(ORC, formatVersion);
5454
}
5555

5656
@Override
@@ -65,6 +65,7 @@ protected QueryRunner createQueryRunner()
6565
.setIcebergProperties(
6666
ImmutableMap.<String, String>builder()
6767
.put("iceberg.file-format", format.name())
68+
.put("iceberg.format-version", String.valueOf(formatVersion))
6869
.put("fs.hadoop.enabled", "true")
6970
.put("fs.native-s3.enabled", "true")
7071
.put("s3.aws-access-key", MINIO_ACCESS_KEY)

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetConnectorTest.java renamed to plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergParquetConnectorTest.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,12 @@
3939
import static java.time.ZoneOffset.UTC;
4040
import static org.assertj.core.api.Assertions.assertThat;
4141

42-
public class TestIcebergParquetConnectorTest
42+
public abstract class BaseIcebergParquetConnectorTest
4343
extends BaseIcebergConnectorTest
4444
{
45-
public TestIcebergParquetConnectorTest()
45+
public BaseIcebergParquetConnectorTest(int formatVersion)
4646
{
47-
super(PARQUET);
47+
super(PARQUET, 3);
4848
}
4949

5050
@Override

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2AvroConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV2AvroConnectorTest
17+
extends BaseIcebergAvroConnectorTest
18+
{
19+
TestIcebergV2AvroConnectorTest()
20+
{
21+
super(2);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2MinioOrcConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV2MinioOrcConnectorTest
17+
extends BaseIcebergMinioOrcConnectorTest
18+
{
19+
TestIcebergV2MinioOrcConnectorTest()
20+
{
21+
super(2);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2ParquetConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV2ParquetConnectorTest
17+
extends BaseIcebergParquetConnectorTest
18+
{
19+
TestIcebergV2ParquetConnectorTest()
20+
{
21+
super(2);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3AvroConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV3AvroConnectorTest
17+
extends BaseIcebergAvroConnectorTest
18+
{
19+
TestIcebergV3AvroConnectorTest()
20+
{
21+
super(3);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3MinioOrcConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV3MinioOrcConnectorTest
17+
extends BaseIcebergMinioOrcConnectorTest
18+
{
19+
TestIcebergV3MinioOrcConnectorTest()
20+
{
21+
super(3);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3ParquetConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV3ParquetConnectorTest
17+
extends BaseIcebergParquetConnectorTest
18+
{
19+
TestIcebergV3ParquetConnectorTest()
20+
{
21+
super(3);
22+
}
23+
}

0 commit comments

Comments
 (0)