Destination s3: Handle nonstandard fields in jsonschema (#45700)

airbytehq · Sep 20, 2024 · adb7e97 · adb7e97
1 parent 2e61b5a
commit adb7e97
Show file tree

Hide file tree

Showing 7 changed files with 31 additions and 7 deletions.
diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -174,6 +174,7 @@ corresponds to that version.
 
 | Version    | Date       | Pull Request                                                | Subject                                                                                                                                                        |
 |:-----------|:-----------|:------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.46.1     | 2024-09-20 | [\#45700](https://github.com/airbytehq/airbyte/pull/45700)  | Destinations: Fix bug in parsing jsonschema |
 | 0.46.0     | 2024-09-18 | [\#45432](https://github.com/airbytehq/airbyte/pull/45432)  | upgrade all libraries to latest version |
 | 0.45.1     | 2024-09-17 | [\#45638](https://github.com/airbytehq/airbyte/pull/45638)  | upgrade apache mina sshd to 2.13.2 to handle openssh tcpkeepalive requests |
 | 0.45.0     | 2024-09-16 | [\#45469](https://github.com/airbytehq/airbyte/pull/45469)  | Fix some race conditions, improve thread filtering, improve test logging                                                                                                                              |

diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties
@@ -1 +1 @@
-version=0.46.0
+version=0.46.1
diff --git a/.../kotlin/io/airbyte/cdk/integrations/destination/s3/jsonschema/JsonSchemaIdentityMapper.kt b/.../kotlin/io/airbyte/cdk/integrations/destination/s3/jsonschema/JsonSchemaIdentityMapper.kt
@@ -122,7 +122,7 @@ open class JsonSchemaIdentityMapper : JsonSchemaMapper() {
             // Denormalize the (non-type) properties from the parent onto each type
             schema.fields().forEach { (key, value) ->
                 if (key != "type") {
-                    newTypeObj.set<ObjectNode>(key, value as ObjectNode)
+                    newTypeObj.set<ObjectNode>(key, value)
                 }
             }
 

diff --git a/.../test/kotlin/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTransformerTest.kt b/.../test/kotlin/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTransformerTest.kt
@@ -179,4 +179,26 @@ class JsonSchemaTransformerTest {
             mapped
         )
     }
+
+    @Test
+    fun testUnionHandlesNonstandardProperties() {
+        val inputSchema =
+            Jsons.deserialize(
+                """
+                    {
+                      "type": [
+                        "string",
+                        "integer"
+                      ],
+                      "description": "foo",
+                      "some_random_other_property": "lol, lmao, isn't jsonschema great"
+                    }
+                """.trimIndent()
+            ) as ObjectNode
+        val mapped = JsonSchemaAvroPreprocessor().mapSchema(inputSchema)
+        assertEquals(
+            Jsons.deserialize("""{"oneOf":[{"type":"string"},{"type":"integer"}]}"""),
+            mapped
+        )
+    }
 }
diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle
@@ -4,9 +4,9 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.46.0'
+    cdkVersionRequired = '0.46.1'
     features = ['db-destinations', 's3-destinations']
-    useLocalCdk = true
+    useLocalCdk = false
 }
 
 airbyteJavaConnector.addCdkDependencies()

diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
-  dockerImageTag: 1.2.0
+  dockerImageTag: 1.2.1
   dockerRepository: airbyte/destination-s3
   githubIssueLabel: destination-s3
   icon: s3.svg

diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md
@@ -536,8 +536,9 @@ To see connector limitations, or troubleshoot your S3 connector, see more [in ou
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                              |
 |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
-| 1.2.0   | 2024-09-18 | [45402](https://github.com/airbytehq/airbyte/pull/45402)   | fix exception with columnless streams                                                                                    |
-| 1.1.0   | 2024-09-18 | [45436](https://github.com/airbytehq/airbyte/pull/45436)   | upgrade all dependencies                                                                                    |
+| 1.2.1   | 2024-09-20 | [45700](https://github.com/airbytehq/airbyte/pull/45700)   | Improve resiliency to jsonschema fields                                                                                                              |
+| 1.2.0   | 2024-09-18 | [45402](https://github.com/airbytehq/airbyte/pull/45402)   | fix exception with columnless streams                                                                                                                |
+| 1.1.0   | 2024-09-18 | [45436](https://github.com/airbytehq/airbyte/pull/45436)   | upgrade all dependencies                                                                                                                             |
 | 1.0.5   | 2024-09-05 | [45143](https://github.com/airbytehq/airbyte/pull/45143)   | don't overwrite (and delete) existing files, skip indexes instead                                                                                    |
 | 1.0.4   | 2024-08-30 | [44933](https://github.com/airbytehq/airbyte/pull/44933)   | Fix: Avro/Parquet: handle empty schemas in nested objects/lists                                                                                      |
 | 1.0.3   | 2024-08-20 | [44476](https://github.com/airbytehq/airbyte/pull/44476)   | Increase message parsing limit to 100mb                                                                                                              |