Rate Limits in Production but Not in Builder

[kmclaugh]

I get rate limits that break my custom connector for the Shopify partner API when I use my customer connector in production. I get no rate limit issues when I run it in the connector builder. Furthermore, there's only one page of responses, so it doesn't make sense that there'd be more than one request anyway.

It also doesn't seem to be affected by how I configure the rate limiting in the connector. It always retries every 1 second.

Errors

2024-08-15 18:32:36 source > Caught retryable error 'Too many requests.' after 1 tries. Waiting 1 seconds then retrying...
2024-08-15 18:32:39 replication-orchestrator > Stream status TRACE received of status: RUNNING for stream Subscription Events
2024-08-15 18:32:39 source > Backing off _send(...) for 1.0s (airbyte_cdk.sources.streams.http.exceptions.RateLimitBackoffException: Too many requests.)
2024-08-15 18:32:39 replication-orchestrator > Stream status TRACE with status RUNNING received of sub-type: RATE_LIMITED for stream Subscription Events
2024-08-15 18:32:39 source > Caught retryable error 'Too many requests.' after 1 tries. Waiting 1 seconds then retrying...

Connector YAML

version: 4.3.2

type: DeclarativeSource

check:
  type: CheckStream
  stream_names:
    - App Relationship Events

definitions:
  streams:
    App Relationship Events:
      type: DeclarativeStream
      name: App Relationship Events
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: "{{ config['partner_id'] }}/api/2024-07/graphql"
          http_method: POST
          request_body_json:
            query: >-
              query ($after: String) { app(id: "gid://partners/App/{{
              config['app_id'] }}") { id events(first: {{config['page_size'] }},
              after: $after, types: [RELATIONSHIP_INSTALLED,
              RELATIONSHIP_UNINSTALLED, RELATIONSHIP_REACTIVATED,
              RELATIONSHIP_DEACTIVATED]) { edges {node { type shop { id
              myshopifyDomain name avatarUrl } app { id name } occurredAt ... on
              RelationshipUninstalled { description } } cursor } pageInfo {
              hasNextPage } } } }
          error_handler:
            type: CompositeErrorHandler
            error_handlers:
              - type: DefaultErrorHandler
                max_retries: 1
                backoff_strategies:
                  - type: ConstantBackoffStrategy
                    backoff_time_in_seconds: 5
                response_filters:
                  - type: HttpResponseFilter
                    action: RATE_LIMITED
                    http_codes:
                      - 429
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - data
              - app
              - events
              - edges
        paginator:
          type: DefaultPaginator
          page_token_option:
            type: RequestOption
            inject_into: body_json
            field_name: variables
          pagination_strategy:
            type: CursorPagination
            cursor_value: >-
              {{ '{"after": "'  + last_record['cursor'] + '"}' if
              last_record['cursor'] else "" }}
            stop_condition: "{{ response.data.app.id.events.pageInfo.hasNextPage is false }}"
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/App Relationship Events"
    Subscription Events:
      type: DeclarativeStream
      name: Subscription Events
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: "{{ config['partner_id'] }}/api/2024-07/graphql"
          http_method: POST
          request_body_json:
            query: >-
              query ($after: String) { app(id: "gid://partners/App/{{
              config['app_id'] }}") { id events(first: {{config['page_size'] }},
              after: $after, types: [SUBSCRIPTION_CHARGE_CANCELED,
              SUBSCRIPTION_CHARGE_ACTIVATED]) { edges { node { type occurredAt
              ... on SubscriptionChargeActivated { occurredAt type charge { id
              amount { amount currencyCode } billingOn name test } } ... on
              SubscriptionChargeCanceled { occurredAt type charge { id amount {
              amount currencyCode } billingOn name test } } } cursor } pageInfo
              { hasNextPage } } } }
          error_handler:
            type: CompositeErrorHandler
            error_handlers:
              - type: DefaultErrorHandler
                max_retries: 1
                backoff_strategies:
                  - type: ConstantBackoffStrategy
                    backoff_time_in_seconds: 5
                response_filters:
                  - type: HttpResponseFilter
                    action: RATE_LIMITED
                    http_codes:
                      - 429
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - data
              - app
              - events
              - edges
        paginator:
          type: DefaultPaginator
          page_token_option:
            type: RequestOption
            inject_into: body_json
            field_name: variables
          pagination_strategy:
            type: CursorPagination
            cursor_value: >-
              {{ '{"after": "'  + last_record['cursor'] + '"}' if
              last_record['cursor'] else "" }}
            stop_condition: "{{ response.data.app.id.events.pageInfo.hasNextPage is false }}"
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/Subscription Events"
  base_requester:
    type: HttpRequester
    url_base: https://partners.shopify.com/
    authenticator:
      type: ApiKeyAuthenticator
      api_token: "{{ config[\"api_key_2\"] }}"
      inject_into:
        type: RequestOption
        field_name: X-Shopify-Access-Token
        inject_into: header

streams:
  - $ref: "#/definitions/streams/App Relationship Events"
  - $ref: "#/definitions/streams/Subscription Events"

spec:
  type: Spec
  connection_specification:
    type: object
    $schema: http://json-schema.org/draft-07/schema#
    required:
      - api_key_2
      - partner_id
      - app_id
      - page_size
    properties:
      api_key_2:
        type: string
        order: 0
        title: API Key
        airbyte_secret: true
      partner_id:
        type: string
        order: 1
        title: Partner ID
      app_id:
        type: string
        order: 2
        title: App Id
      page_size:
        type: integer
        order: 3
        title: Page Size
    additionalProperties: true

metadata:
  autoImportSchema:
    App Relationship Events: true
    Subscription Events: true
  testedStreams:
    App Relationship Events:
      streamHash: 04ecce297743067b077f4bb17a06fa97de11c65f
      hasResponse: true
      responsesAreSuccessful: false
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
    Subscription Events:
      hasRecords: true
      streamHash: 31d6b4d160462da24057775ee8911e99ea147a22
      hasResponse: true
      primaryKeysAreUnique: true
      primaryKeysArePresent: true
      responsesAreSuccessful: false

schemas:
  App Relationship Events:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      cursor:
        type:
          - string
          - "null"
      node:
        type:
          - object
          - "null"
        properties:
          type:
            type:
              - string
              - "null"
          app:
            type:
              - object
              - "null"
            properties:
              id:
                type:
                  - string
                  - "null"
              name:
                type:
                  - string
                  - "null"
          occurredAt:
            type:
              - string
              - "null"
          shop:
            type:
              - object
              - "null"
            properties:
              id:
                type:
                  - string
                  - "null"
              myshopifyDomain:
                type:
                  - string
                  - "null"
              name:
                type:
                  - string
                  - "null"
  Subscription Events:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      cursor:
        type:
          - string
          - "null"
      node:
        type:
          - object
          - "null"
        properties:
          type:
            type:
              - string
              - "null"
          charge:
            type:
              - object
              - "null"
            properties:
              amount:
                type:
                  - object
                  - "null"
                properties:
                  amount:
                    type:
                      - string
                      - "null"
                  currencyCode:
                    type:
                      - string
                      - "null"
              billingOn:
                type:
                  - string
                  - "null"
              id:
                type:
                  - string
                  - "null"
              name:
                type:
                  - string
                  - "null"
              test:
                type:
                  - boolean
                  - "null"
          occurredAt:
            type:
              - string
              - "null"