Some streams either received an INCOMPLETE stream status, or did not receive a stream status at all #903
Description
Source: Big query
When trying to move data from big query source to postgres destination i get this error -> Some streams either received an INCOMPLETE stream status, or did not receive a stream status at all
sided with this error also Message contained record from a stream that was not in the catalog.
catalog
I have tried moving one table and still got the same error even though i checked the catalog and they are the same
Full error: 2025-12-10 21:28:51 - INFO - INFO main i.a.c.i.d.a.AsyncStreamConsumer(close):202 class io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer closed
2025-12-10 21:28:51 - INFO - INFO main i.a.c.i.d.a.AsyncStreamConsumer(close):214 Some streams either received an INCOMPLETE stream status, or did not receive a stream status at all: test_etl_staging.test_table_0
2025-12-10 21:28:51 - INFO - ERROR main i.a.c.i.u.ConnectorExceptionHandler(handleException):68 caught exception! java.lang.IllegalArgumentException: Message contained record from a stream that was not in the catalog.
catalog: {"streams":[{"stream":{"name":"test_table_0","json_schema":{"type":"object","properties":{"col_center_3":{"type":"string"},"col_hour_1":{"type":"number","airbyte_type":"integer"},"col_laugh_2":{"type":"number"},"col_late_0":{"type":"number","airbyte_type":"integer"},"col_office_4":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"test_etl_staging"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[],"generation_id":1,"minimum_generation_id":1,"sync_id":1}]},
message: {"type":"RECORD","record":{"streamDescriptor":{"name":"test_table_0"},"namespace":null,"stream":"test_table_0","data":null,"emitted_at":1765387716,"meta":{"changes":[]},"file":null},"state":null,"trace":null,"serialized":"{"col_center_3":"Company especially doctor pass name soon edge.","col_hour_1":7765,"col_laugh_2":"22.460241444","col_late_0":9933,"col_office_4":"2025-12-09T12:56:48Z","_airbyte_raw_id":"06939c9e-a81a-7133-8000-137e432ecb09","_airbyte_extracted_at":"2025-12-10T19:28:36.969000","_airbyte_meta":{}}"}
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer$Companion.throwUnrecognizedStream(AsyncStreamConsumer.kt:263) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer$Companion.access$throwUnrecognizedStream(AsyncStreamConsumer.kt:252) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer.validateRecord(AsyncStreamConsumer.kt:248) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer.accept(AsyncStreamConsumer.kt:119) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer$Companion$appendOnClose$1.accept(SerializedAirbyteMessageConsumer.kt:65) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion$consumeWriteStream$2$1.invoke(IntegrationRunner.kt:412) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion$consumeWriteStream$2$1.invoke(IntegrationRunner.kt:410) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion.consumeWriteStream$lambda$1$lambda$0(IntegrationRunner.kt:410) ~[airbyte-cdk-core-0.48.9.jar:?]
at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133) ~[?:?]
at java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1939) ~[?:?]
at java.base/java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:762) ~[?:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion.consumeWriteStream$io_airbyte_airbyte_cdk_java_airbyte_cdk_airbyte_cdk_core(IntegrationRunner.kt:410) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion.consumeWriteStream$io_airbyte_airbyte_cdk_java_airbyte_cdk_airbyte_cdk_core$default(IntegrationRunner.kt:402) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.runInternal(IntegrationRunner.kt:215) [airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.run(IntegrationRunner.kt:119) [airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.run$default(IntegrationRunner.kt:113) [airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.integrations.destination.postgres.PostgresDestination$Companion.main(PostgresDestination.kt:230) [io.airbyte.airbyte-integrations.connectors-destination-postgres.jar:?]
at io.airbyte.integrations.destination.postgres.PostgresDestination.main(PostgresDestination.kt) [io.airbyte.airbyte-integrations.connectors-destination-postgres.jar:?]
Suppressed: io.airbyte.commons.exceptions.TransientErrorException: Some streams were unsuccessful due to a source error. See logs for details.
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer.close(AsyncStreamConsumer.kt:217) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer$Companion$appendOnClose$1.close(SerializedAirbyteMessageConsumer.kt:70) ~[airbyte-cdk-core-0.48.9.jar:?]
at kotlin.jdk7.AutoCloseableKt.closeFinally(AutoCloseableJVM.kt:51) ~[kotlin-stdlib-2.0.0.jar:2.0.0-release-341]
at io.airbyte.cdk.integrations.base.IntegrationRunner.runInternal(IntegrationRunner.kt:215) [airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.run(IntegrationRunner.kt:119) [airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.run$default(IntegrationRunner.kt:113) [airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.integrations.destination.postgres.PostgresDestination$Companion.main(PostgresDestination.kt:230) [io.airbyte.airbyte-integrations.connectors-destination-postgres.jar:?]
at io.airbyte.integrations.destination.postgres.PostgresDestination.main(PostgresDestination.kt) [io.airbyte.airbyte-integrations.connectors-destination-postgres.jar:?]
2025-12-10 21:28:51 - INFO - ERROR main i.a.c.i.b.AirbyteExceptionHandler(uncaughtException):31 Something went wrong in the connector. See the logs for more details. java.lang.IllegalArgumentException: Message contained record from a stream that was not in the catalog.
catalog: {"streams":[{"stream":{"name":"test_table_0","json_schema":{"type":"object","properties":{"col_center_3":{"type":"string"},"col_hour_1":{"type":"number","airbyte_type":"integer"},"col_laugh_2":{"type":"number"},"col_late_0":{"type":"number","airbyte_type":"integer"},"col_office_4":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"test_etl_staging"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[],"generation_id":1,"minimum_generation_id":1,"sync_id":1}]},
message: {"type":"RECORD","record":{"streamDescriptor":{"name":"test_table_0"},"namespace":null,"stream":"test_table_0","data":null,"emitted_at":1765387716,"meta":{"changes":[]},"file":null},"state":null,"trace":null,"serialized":"{"col_center_3":"Company especially doctor pass name soon edge.","col_hour_1":7765,"col_laugh_2":"22.460241444","col_late_0":9933,"col_office_4":"2025-12-09T12:56:48Z","_airbyte_raw_id":"06939c9e-a81a-7133-8000-137e432ecb09","_airbyte_extracted_at":"2025-12-10T19:28:36.969000","_airbyte_meta":{}}"}
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer$Companion.throwUnrecognizedStream(AsyncStreamConsumer.kt:263) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer$Companion.access$throwUnrecognizedStream(AsyncStreamConsumer.kt:252) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer.validateRecord(AsyncStreamConsumer.kt:248) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer.accept(AsyncStreamConsumer.kt:119) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer$Companion$appendOnClose$1.accept(SerializedAirbyteMessageConsumer.kt:65) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion$consumeWriteStream$2$1.invoke(IntegrationRunner.kt:412) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion$consumeWriteStream$2$1.invoke(IntegrationRunner.kt:410) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion.consumeWriteStream$lambda$1$lambda$0(IntegrationRunner.kt:410) ~[airbyte-cdk-core-0.48.9.jar:?]
at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133) ~[?:?]
at java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1939) ~[?:?]
at java.base/java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:762) ~[?:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion.consumeWriteStream$io_airbyte_airbyte_cdk_java_airbyte_cdk_airbyte_cdk_core(IntegrationRunner.kt:410) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner$Companion.consumeWriteStream$io_airbyte_airbyte_cdk_java_airbyte_cdk_airbyte_cdk_core$default(IntegrationRunner.kt:402) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.runInternal(IntegrationRunner.kt:215) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.run(IntegrationRunner.kt:119) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.run$default(IntegrationRunner.kt:113) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.integrations.destination.postgres.PostgresDestination$Companion.main(PostgresDestination.kt:230) ~[io.airbyte.airbyte-integrations.connectors-destination-postgres.jar:?]
at io.airbyte.integrations.destination.postgres.PostgresDestination.main(PostgresDestination.kt) ~[io.airbyte.airbyte-integrations.connectors-destination-postgres.jar:?]
Suppressed: io.airbyte.commons.exceptions.TransientErrorException: Some streams were unsuccessful due to a source error. See logs for details.
at io.airbyte.cdk.integrations.destination.async.AsyncStreamConsumer.close(AsyncStreamConsumer.kt:217) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer$Companion$appendOnClose$1.close(SerializedAirbyteMessageConsumer.kt:70) ~[airbyte-cdk-core-0.48.9.jar:?]
at kotlin.jdk7.AutoCloseableKt.closeFinally(AutoCloseableJVM.kt:51) ~[kotlin-stdlib-2.0.0.jar:2.0.0-release-341]
at io.airbyte.cdk.integrations.base.IntegrationRunner.runInternal(IntegrationRunner.kt:215) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.run(IntegrationRunner.kt:119) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.cdk.integrations.base.IntegrationRunner.run$default(IntegrationRunner.kt:113) ~[airbyte-cdk-core-0.48.9.jar:?]
at io.airbyte.integrations.destination.postgres.PostgresDestination$Companion.main(PostgresDestination.kt:230) ~[io.airbyte.airbyte-integrations.connectors-destination-postgres.jar:?]
at io.airbyte.integrations.destination.postgres.PostgresDestination.main(PostgresDestination.kt) ~[io.airbyte.airbyte-integrations.connectors-destination-postgres.jar:?]
2025-12-10 21:28:51 - ERROR - Something went wrong in the connector. See the logs for more details.
Note: Using PyAirByte locally checked out on this pr as force_full_refresh is not functionning
pr-> #899