treat system-probe modules as components #45771

brycekahle · 2026-01-31T02:32:36Z

What does this PR do?

Motivation

Describe how you validated your changes

Additional Notes

agent-platform-auto-pr · 2026-01-31T02:38:18Z

Gitlab CI Configuration Changes

Modified Jobs

.kmt_run_secagent_tests

  .kmt_run_secagent_tests:
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    retry:
      exit_codes: 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    timeout: 1h 30m
    variables:
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

.kmt_run_secagent_tests_base

  .kmt_run_secagent_tests_base:
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    retry:
      exit_codes: 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    timeout: 1h 30m
    variables:
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

.kmt_run_secagent_tests_required

  .kmt_run_secagent_tests_required:
    allow_failure: false
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    retry:
      exit_codes: 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    timeout: 1h 30m
    variables:
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

.kmt_run_sysprobe_tests

  .kmt_run_sysprobe_tests:
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    retry:
      exit_codes: 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - cmd/system-probe/**/*
        - pkg/collector/corechecks/ebpf/**/*
        - pkg/discovery/**/*
        - pkg/ebpf/**/*
        - pkg/network/**/*
        - pkg/process/monitor/*
        - pkg/security/**/*
        - pkg/util/kernel/**/*
        - pkg/dyninst/**/*
        - pkg/gpu/**/*
        - .gitlab/test/kernel_matrix_testing/system_probe.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/system_probe.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_system_probe
    timeout: 1h 30m
    variables:
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: system-probe

.kmt_run_tests

  .kmt_run_tests:
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    retry:
      exit_codes: 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    variables:
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2

kmt_run_secagent_tests_arm64

  kmt_run_secagent_tests_arm64:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_arm64
    - upload_dependencies_secagent_arm64
    - upload_secagent_tests_arm64
    parallel:
      matrix:
      - TAG:
        - ubuntu_22.04
        - ubuntu_24.04
        - ubuntu_24.10
        - amazon_5.4
        - amazon_5.10
        - amazon_2023
        - fedora_37
        - fedora_38
        - debian_11
        - debian_12
        - oracle_8.9
        - oracle_9.3
        - rocky_8.5
        - rocky_9.3
        - rocky_9.4
        - opensuse_15.5
        TEST_SET:
        - cws_host
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:arm64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: arm64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_arm64_ad

  kmt_run_secagent_tests_arm64_ad:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_arm64
    - upload_dependencies_secagent_arm64
    - upload_secagent_tests_arm64
    parallel:
      matrix:
      - TAG:
        - ubuntu_22.04
        TEST_SET:
        - cws_ad
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:arm64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: arm64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_arm64_docker

  kmt_run_secagent_tests_arm64_docker:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_arm64
    - upload_dependencies_secagent_arm64
    - upload_secagent_tests_arm64
    parallel:
      matrix:
      - TAG:
        - ubuntu_22.04
        - ubuntu_24.04
        - ubuntu_24.10
        - amazon_5.4
        - amazon_5.10
        - amazon_2023
        - fedora_37
        - fedora_38
        - debian_11
        - debian_12
        - oracle_8.9
        - oracle_9.3
        - rocky_8.5
        - rocky_9.3
        - rocky_9.4
        TEST_SET:
        - cws_docker
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:arm64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: arm64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_arm64_ebpfless

  kmt_run_secagent_tests_arm64_ebpfless:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_arm64
    - upload_dependencies_secagent_arm64
    - upload_secagent_tests_arm64
    parallel:
      matrix:
      - TAG:
        - ubuntu_22.04
        TEST_SET:
        - cws_el
        - cws_el_ns
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:arm64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: arm64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_arm64_peds

  kmt_run_secagent_tests_arm64_peds:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: false
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_arm64
    - upload_dependencies_secagent_arm64
    - upload_secagent_tests_arm64
    parallel:
      matrix:
      - TAG:
        - ubuntu_22.04
        - ubuntu_24.04
        - ubuntu_24.10
        - amazon_5.4
        - amazon_5.10
        - amazon_2023
        - fedora_37
        - fedora_38
        - debian_11
        - debian_12
        - oracle_8.9
        - oracle_9.3
        - rocky_8.5
        - rocky_9.3
        - rocky_9.4
        - opensuse_15.5
        TEST_SET:
        - cws_peds
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:arm64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: arm64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_x64

  kmt_run_secagent_tests_x64:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_x64
    - upload_dependencies_secagent_x64
    - upload_secagent_tests_x64
    parallel:
      matrix:
      - TAG:
        - ubuntu_18.04
        - ubuntu_20.04
        - ubuntu_22.04
        - ubuntu_24.04
        - ubuntu_24.10
        - amazon_4.14
        - amazon_5.4
        - amazon_5.10
        - amazon_2023
        - fedora_37
        - fedora_38
        - debian_10
        - debian_11
        - debian_12
        - centos_7.9
        - oracle_8.9
        - oracle_9.3
        - rocky_8.5
        - rocky_9.3
        - rocky_9.4
        - opensuse_15.3
        - opensuse_15.5
        - suse_12.5
        TEST_SET:
        - cws_host
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:amd64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: x86_64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_x64_ad

  kmt_run_secagent_tests_x64_ad:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_x64
    - upload_dependencies_secagent_x64
    - upload_secagent_tests_x64
    parallel:
      matrix:
      - TAG:
        - ubuntu_22.04
        TEST_SET:
        - cws_ad
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:amd64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: x86_64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_x64_docker

  kmt_run_secagent_tests_x64_docker:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_x64
    - upload_dependencies_secagent_x64
    - upload_secagent_tests_x64
    parallel:
      matrix:
      - TAG:
        - ubuntu_18.04
        - ubuntu_20.04
        - ubuntu_22.04
        - ubuntu_24.04
        - ubuntu_24.10
        - amazon_4.14
        - amazon_5.4
        - amazon_5.10
        - amazon_2023
        - fedora_37
        - fedora_38
        - debian_10
        - debian_11
        - debian_12
        - centos_7.9
        - oracle_8.9
        - oracle_9.3
        - rocky_8.5
        - rocky_9.3
        - rocky_9.4
        TEST_SET:
        - cws_docker
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:amd64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: x86_64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_x64_ebpfless

  kmt_run_secagent_tests_x64_ebpfless:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: true
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_x64
    - upload_dependencies_secagent_x64
    - upload_secagent_tests_x64
    parallel:
      matrix:
      - TAG:
        - ubuntu_22.04
        TEST_SET:
        - cws_el
        - cws_el_ns
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:amd64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: x86_64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_x64_peds

  kmt_run_secagent_tests_x64_peds:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: false
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_x64
    - upload_dependencies_secagent_x64
    - upload_secagent_tests_x64
    parallel:
      matrix:
      - TAG:
        - ubuntu_18.04
        - ubuntu_20.04
        - ubuntu_22.04
        - ubuntu_24.04
        - ubuntu_24.10
        - amazon_4.14
        - amazon_5.4
        - amazon_5.10
        - amazon_2023
        - fedora_37
        - fedora_38
        - debian_10
        - debian_11
        - debian_12
        - centos_7.9
        - oracle_8.9
        - oracle_9.3
        - rocky_8.5
        - rocky_9.3
        - rocky_9.4
        - opensuse_15.3
        - opensuse_15.5
        - suse_12.5
        TEST_SET:
        - cws_peds
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:amd64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: x86_64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_secagent_tests_x64_required

  kmt_run_secagent_tests_x64_required:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    allow_failure: false
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_secagent_x64
    - upload_dependencies_secagent_x64
    - upload_secagent_tests_x64
    parallel:
      matrix:
      - TAG:
        - ubuntu_24.10
        TEST_SET:
        - cws_req
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - allow_failure: true
      if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/ebpf/**/*
        - pkg/security/**/*
        - pkg/eventmonitor/**/*
        - .gitlab/test/kernel_matrix_testing/security_agent.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/tests/cws/**/*
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/security_agent.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_security_agent
    tags:
    - arch:amd64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: x86_64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: security-agent

kmt_run_sysprobe_tests_arm64

  kmt_run_sysprobe_tests_arm64:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_sysprobe_arm64
    - upload_dependencies_sysprobe_arm64
    - upload_sysprobe_tests_arm64
    - upload_minimized_btfs_sysprobe_arm64
    parallel:
      matrix:
      - TAG:
        - ubuntu_18.04
        - ubuntu_20.04
        - ubuntu_22.04
        - ubuntu_24.04
        - ubuntu_24.10
        - amazon_4.14
        - amazon_5.4
        - amazon_5.10
        - amazon_2023
        - fedora_37
        - fedora_38
        - debian_10
        - debian_11
        - debian_12
        - centos_7.9
        - centos_8
        - rocky_9.4
        TEST_SET:
        - only_usm
        - no_usm
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - cmd/system-probe/**/*
        - pkg/collector/corechecks/ebpf/**/*
        - pkg/discovery/**/*
        - pkg/ebpf/**/*
        - pkg/network/**/*
        - pkg/process/monitor/*
        - pkg/security/**/*
        - pkg/util/kernel/**/*
        - pkg/dyninst/**/*
        - pkg/gpu/**/*
        - .gitlab/test/kernel_matrix_testing/system_probe.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/system_probe.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_system_probe
    tags:
    - arch:arm64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: arm64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: system-probe

kmt_run_sysprobe_tests_x64

  kmt_run_sysprobe_tests_x64:
    after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - mkdir -p $CI_PROJECT_DIR/logs
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"journalctl -u setup-ddvm.service\"" >
      $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - cat $CI_PROJECT_DIR/logs/setup-ddvm.log || true
    - ssh metal_instance "ssh ${MICRO_VM_IP} \"systemctl is-active setup-ddvm.service\""
      | tee $CI_PROJECT_DIR/logs/setup-ddvm.status || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/junit.tar.gz /home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/junit-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp ${MICRO_VM_IP}:/ci-visibility/testjson.tar.gz /home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz"
      || true
    - scp "metal_instance:/home/ubuntu/testjson-${ARCH}-${TAG}-${TEST_SET}.tar.gz" $DD_AGENT_TESTING_DIR/
      || true
    - ssh metal_instance "scp -r ${MICRO_VM_IP}:/tmp/test_pcaps /home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      || true
    - mkdir -p "$CI_PROJECT_DIR/pcaps" && scp -r "metal_instance:/home/ubuntu/test_pcaps-${ARCH}-${TAG}-${TEST_SET}"
      "$CI_PROJECT_DIR/pcaps/test_pcaps-${ARCH}-${TAG}-${TEST_SET}" || true
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    - "if [ \"${COLLECT_COMPLEXITY}\" = \"yes\" ]; then\n  ssh metal_instance \"scp\
      \ ${MICRO_VM_IP}:/verifier-complexity.tar.gz /home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ || true\n  scp \"metal_instance:/home/ubuntu/verifier-complexity-${ARCH}-${TAG}-${TEST_COMPONENT}.tar.gz\"\
      \ $DD_AGENT_TESTING_DIR/ || true\nfi\n"
    - dda inv -- -e kmt.tag-ci-job
    - tar -xzvf $DD_AGENT_TESTING_DIR/testjson-*.tar.gz
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz"
      out.json
    - "if [ \"$CI_JOB_STATUS\" != \"success\" ]; then\n  echo \"This test job failed.\
      \ KMT automatically retries tests, so we don't allow direct retries of this job.\"\
      \n  echo \"However, if you're confident that the problem is not related to the\
      \ tests, you can retry the job by running:\"\n  echo \"dda inv kmt.retry-failed-pipeline\
      \ --pipeline-id $CI_PIPELINE_ID\"\nfi\n"
    artifacts:
      expire_in: 2 weeks
      paths:
      - $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
      - $DD_AGENT_TESTING_DIR/verifier-complexity-$ARCH-$TAG-${TEST_COMPONENT}.tar.gz
      - $CI_PROJECT_DIR/logs
      - $CI_PROJECT_DIR/pcaps
      - $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - mkdir -p ~/.aws
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config
      || exit $?
    - export AWS_PROFILE=agent-qa-ci
    - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE
      || exit $?
    - echo "" >> $AWS_EC2_SSH_KEY_FILE
    - chmod 600 $AWS_EC2_SSH_KEY_FILE
    - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - PLATFORMS_FOR_COMPLEXITY_COLLECTION="debian_10 ubuntu_18.04 centos_8 opensuse_15.3
      suse_12.5 fedora_38"
    - "if [ \"${TEST_SET}\" = \"no_usm\" ] && echo \"${PLATFORMS_FOR_COMPLEXITY_COLLECTION}\"\
      \ | grep -qw \"${TAG}\" ; then\n  export COLLECT_COMPLEXITY=yes\nfi\n"
    - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}"
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - kmt_setup_env_sysprobe_x64
    - upload_dependencies_sysprobe_x64
    - upload_sysprobe_tests_x64
    - upload_minimized_btfs_sysprobe_x64
    parallel:
      matrix:
      - TAG:
        - ubuntu_16.04
        - ubuntu_18.04
        - ubuntu_20.04
        - ubuntu_22.04
        - ubuntu_24.04
        - ubuntu_24.10
        - amazon_4.14
        - amazon_5.4
        - amazon_5.10
        - amazon_2023
        - fedora_37
        - fedora_38
        - debian_9
        - debian_10
        - debian_11
        - debian_12
        - centos_7.9
        - centos_8
        - rocky_9.4
        - rocky_8.4
        TEST_SET:
        - only_usm
        - no_usm
    retry:
      exit_codes:
      - 42
      max: 2
      when:
      - job_execution_timeout
      - runner_system_failure
      - stuck_or_timeout_failure
      - unknown_failure
      - api_failure
      - scheduler_failure
      - stale_schedule
      - data_integrity_failure
    rules:
    - if: $CI_COMMIT_BRANCH == "main"
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_KMT_TESTS == 'on'
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - cmd/system-probe/**/*
        - pkg/collector/corechecks/ebpf/**/*
        - pkg/discovery/**/*
        - pkg/ebpf/**/*
        - pkg/network/**/*
        - pkg/process/monitor/*
        - pkg/security/**/*
        - pkg/util/kernel/**/*
        - pkg/dyninst/**/*
        - pkg/gpu/**/*
        - .gitlab/test/kernel_matrix_testing/system_probe.yml
        - .gitlab/test/kernel_matrix_testing/common.yml
        - .gitlab/build/source_test/ebpf.yml
        - test/new-e2e/system-probe/**/*
        - test/new-e2e/scenarios/system-probe/**/*
        - test/e2e-framework/testing/runner/**/*
        - test/e2e-framework/testing/utils/**/*
        - test/new-e2e/go.mod
        - go.mod
        - tasks/system_probe.py
        - tasks/kmt.py
        - tasks/kernel_matrix_testing/*
    - allow_failure: true
      when: manual
    script:
    - INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
    - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
    - FILTER_MANAGED="Name=tag:managed-by,Values=pulumi"
    - FILTER_STATE="Name=instance-state-name,Values=running"
    - FILTER_PIPELINE="Name=tag:pipeline-id,Values=${CI_PIPELINE_ID}"
    - FILTER_ARCH="Name=tag:arch,Values=${ARCH}"
    - FILTER_INSTANCE_TYPE="Name=tag:instance-type,Values=${INSTANCE_TYPE}"
    - FILTER_TEST_COMPONENT="Name=tag:test-component,Values=${TEST_COMPONENT}"
    - QUERY_INSTANCE_IDS='Reservations[*].Instances[*].InstanceId'
    - QUERY_PRIVATE_IPS='Reservations[*].Instances[*].PrivateIpAddress'
    - RUNNING_INSTANCES=$(aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED
      $FILTER_PIPELINE $FILTER_TEST_COMPONENT "Name=private-ip-address,Values=$INSTANCE_IP"
      --output text --query $QUERY_INSTANCE_IDS | wc -l )
    - "if [ $RUNNING_INSTANCES -eq \"0\" ]; then\n  echo \"These jobs do not permit\
      \ retries. The go tests are retried a user-specified number of times automatically.\
      \ In order to re-run the tests, you must trigger the pipeline again\"\n  'false'\n\
      fi\n"
    - MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output)
    - MICRO_VM_NAME=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET
      $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET)))
      | map(select(.tag==$TAG)) | .[].id' $CI_PROJECT_DIR/stack.output)
    - GO_VERSION=$(dda inv go-version)
    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
    - echo -e "Host metal_instance\nHostname $INSTANCE_IP\nUser ubuntu\nStrictHostKeyChecking
      no\nIdentityFile $AWS_EC2_SSH_KEY_FILE\nServerAliveInterval 10\nServerAliveCountMax
      180\n" | tee -a ~/.ssh/config
    - chmod 600 ~/.ssh/config
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh scp "$DD_AGENT_TESTING_DIR/job_env.txt" "metal_instance:/home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt"
    - ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "scp /home/ubuntu/job_env-${ARCH}-${TAG}-${TEST_SET}.txt
      ${MICRO_VM_IP}:/job_env.txt"
    - NESTED_VM_CMD="/home/ubuntu/connector -host ${MICRO_VM_IP} -user root -ssh-file
-     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=5G /opt/kmt-ramfs
?                                                                          ^
+     /home/kernel-version-testing/ddvm_rsa -vm-cmd 'mount -o remount,size=6G /opt/kmt-ramfs
?                                                                          ^
      && CI=true /root/fetch_dependencies.sh ${ARCH} && COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}
      /opt/micro-vm-init.sh -test-tools /opt/testing-tools -retry ${RETRY} -test-root
      /opt/${TEST_COMPONENT}-tests -packages-run-config /opt/${TEST_SET}.json'"
    - $CI_PROJECT_DIR/connector-$ARCH -host $INSTANCE_IP -user ubuntu -ssh-file $AWS_EC2_SSH_KEY_FILE
      -vm-cmd "${NESTED_VM_CMD}" -send-env-vars CI_COMMIT_SHA,DD_API_KEY
    - NO_RETRY_EXIT_CODE=42 ${CI_PROJECT_DIR}/tools/ci/retry.sh ssh metal_instance "ssh
      ${MICRO_VM_IP} '/opt/testing-tools/test-json-review -flakes /opt/testing-tools/flakes.yaml
      -codeowners /opt/testing-tools/CODEOWNERS -test-root /opt/${TEST_COMPONENT}-tests'"
    - '[ ! -f $CI_PROJECT_DIR/daemon-${ARCH}.log ] && ${CI_PROJECT_DIR}/tools/ci/retry.sh
      scp metal_instance:/home/ubuntu/daemon.log $CI_PROJECT_DIR/vm-metrics-daemon-${ARCH}.log'
    stage: kernel_matrix_testing_system_probe
    tags:
    - arch:amd64
    - specific:true
    timeout: 1h 30m
    variables:
      ARCH: x86_64
      AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
      RETRY: 2
      TEST_COMPONENT: system-probe

new-e2e-npm-packages

  new-e2e-npm-packages:
    after_script:
    - CODECOV_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $CODECOV token) || exit
      $?; export CODECOV_TOKEN
    - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "junit-${CI_JOB_ID}.tgz" "$E2E_RESULT_JSON"
    - "if [ -d \"$E2E_COVERAGE_OUT_DIR\" ]; then\n  dda inv -- -e coverage.process-e2e-coverage-folders\
      \ $E2E_COVERAGE_OUT_DIR\n  pip install boto3==1.38.8 # TODO: Remove this before\
      \ merging, after dda is bumped in test-infra-definitions\n  dda inv -- -e dyntest.compute-and-upload-job-index\
      \ --bucket-uri $S3_PERMANENT_ARTIFACTS_URI --coverage-folder $E2E_COVERAGE_OUT_DIR\
      \ --commit-sha $CI_COMMIT_SHA --job-id $CI_JOB_ID\nfi\n"
    artifacts:
      expire_in: 2 weeks
      paths:
      - $E2E_OUTPUT_DIR
      - $E2E_RESULT_JSON
      - junit-*.tgz
      - $E2E_COVERAGE_OUT_DIR
      reports:
        annotations:
        - $EXTERNAL_LINKS_PATH
      when: always
    before_script:
    - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache_e2e.tar.xz -C $GOPATH/pkg/mod/cache
    - rm -f modcache_e2e.tar.xz
    - mkdir -p ~/.pulumi && tar xJf pulumi_plugins.tar.xz -C ~/.pulumi
    - rm -f pulumi_plugins.tar.xz
    - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache_tools.tar.xz -C $GOPATH/pkg/mod/cache
    - rm -f modcache_tools.tar.xz
    - dda inv -- -e install-tools
    - mkdir -p ~/.aws
    - "if [ -n \"$E2E_USE_AWS_PROFILE\" ]; then\n  echo Using agent-qa-ci aws profile\n\
      \  $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config\
      \ || exit $?\n  # Now all `aws` commands target the agent-qa profile\n  export\
      \ AWS_PROFILE=agent-qa-ci\nelse\n  # Assume role to fetch only once credentials\
      \ and avoid rate limits\n  echo Assuming ddbuild-agent-ci role\n  roleoutput=\"\
      $(aws sts assume-role --role-arn arn:aws:iam::669783387624:role/ddbuild-agent-ci\
      \ --external-id ddbuild-agent-ci --role-session-name RoleSession)\"\n  export\
      \ AWS_ACCESS_KEY_ID=\"$(echo \"$roleoutput\" | jq -r '.Credentials.AccessKeyId')\"\
      \n  export AWS_SECRET_ACCESS_KEY=\"$(echo \"$roleoutput\" | jq -r '.Credentials.SecretAccessKey')\"\
      \n  export AWS_SESSION_TOKEN=\"$(echo \"$roleoutput\" | jq -r '.Credentials.SessionToken')\"\
      \nfi\n"
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_AWS_PUBLIC_KEY_PATH
      || exit $?
    - touch $E2E_AWS_PRIVATE_KEY_PATH && chmod 600 $E2E_AWS_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh
      $AGENT_QA_E2E ssh_key_rsa > $E2E_AWS_PRIVATE_KEY_PATH || exit $?
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_AZURE_PUBLIC_KEY_PATH
      || exit $?
    - touch $E2E_AZURE_PRIVATE_KEY_PATH && chmod 600 $E2E_AZURE_PRIVATE_KEY_PATH &&
      $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_AZURE_PRIVATE_KEY_PATH
      || exit $?
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_GCP_PUBLIC_KEY_PATH
      || exit $?
    - touch $E2E_GCP_PRIVATE_KEY_PATH && chmod 600 $E2E_GCP_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh
      $AGENT_QA_E2E ssh_key_rsa > $E2E_GCP_PRIVATE_KEY_PATH || exit $?
    - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
    - ARM_CLIENT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE client_id)
      || exit $?; export ARM_CLIENT_ID
    - ARM_CLIENT_SECRET=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE token)
      || exit $?; export ARM_CLIENT_SECRET
    - ARM_TENANT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE tenant_id)
      || exit $?; export ARM_TENANT_ID
    - ARM_SUBSCRIPTION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_AZURE subscription_id)
      || exit $?; export ARM_SUBSCRIPTION_ID
    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_GCP credentials_json > ~/gcp-credentials.json
      || exit $?
    - export GOOGLE_APPLICATION_CREDENTIALS=~/gcp-credentials.json
    - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
    - export DD_ENV=nativetest
    - export DD_CIVISIBILITY_AGENTLESS_ENABLED=true
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token)
      || exit $?; export DD_API_KEY
    - export WINDOWS_DDNPM_DRIVER=${WINDOWS_DDNPM_DRIVER:-$(dda inv release.get-release-json-value
      "dependencies::WINDOWS_DDNPM_DRIVER" --no-worktree)}
    - export WINDOWS_DDPROCMON_DRIVER=${WINDOWS_DDPROCMON_DRIVER:-$(dda inv release.get-release-json-value
      "dependencies::WINDOWS_DDPROCMON_DRIVER" --no-worktree)}
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
    needs:
    - go_e2e_deps
    - go_e2e_test_binaries
    - go_tools_deps
    - job: new-e2e-base-coverage
      optional: true
    - agent_deb-x64-a7
    - agent_rpm-x64-a7
    - windows_msi_and_bosh_zip_x64-a7
    rules:
    - if: $RUN_E2E_TESTS == "off"
      when: never
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - if: $RUN_E2E_TESTS == "on"
      when: on_success
    - if: $CI_COMMIT_BRANCH == "main"
      when: on_success
    - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/
      when: on_success
    - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/
      when: on_success
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - .gitlab/test/e2e/e2e.yml
        - test/e2e-framework/**/*
        - test/new-e2e/go.mod
        - go.mod
        - flakes.yaml
        - release.json
    - changes:
        compare_to: $COMPARE_TO_BRANCH
        paths:
        - pkg/network/**/*
        - test/new-e2e/tests/npm/**/*
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - allow_failure: true
      when: manual
    script:
    - export IS_DEV_BRANCH="$(dda inv -- -e pipeline.is-dev-branch)"
    - DYNAMIC_TESTS_BREAKGLASS=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $DYNAMIC_TESTS_BREAKGLASS
      value) || exit $?; export DYNAMIC_TESTS_BREAKGLASS
    - "if [ \"$DYNAMIC_TESTS_BREAKGLASS\" == \"true\" ] || [ \"$IS_DEV_BRANCH\" == \"\
      false\" ] || [ \"$RUN_E2E_TESTS\" == \"on\" ]; then\n  export DYNAMIC_TESTS_FLAG=\"\
      \"\nfi\n"
    - dda inv -- -e new-e2e-tests.run $DYNAMIC_TESTS_FLAG $PRE_BUILT_BINARIES_FLAG $MAX_RETRIES_FLAG
      --local-package $CI_PROJECT_DIR/$OMNIBUS_BASE_DIR --result-json $E2E_RESULT_JSON
      --targets $TARGETS -c ddagent:imagePullRegistry=669783387624.dkr.ecr.us-east-1.amazonaws.com
      -c ddagent:imagePullUsername=AWS -c ddagent:imagePullPassword=$(aws ecr get-login-password)
      --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS} --test-washer --logs-folder=$E2E_OUTPUT_DIR/logs
      --logs-post-processing --logs-post-processing-test-depth=$E2E_LOGS_PROCESSING_TEST_DEPTH
    stage: e2e
    tags:
    - arch:amd64
    - specific:true
    variables:
      DYNAMIC_TESTS_FLAG: --impacted
      E2E_AWS_PRIVATE_KEY_PATH: /tmp/agent-qa-aws-ssh-key
      E2E_AWS_PUBLIC_KEY_PATH: /tmp/agent-qa-aws-ssh-key.pub
      E2E_AZURE_PRIVATE_KEY_PATH: /tmp/agent-qa-azure-ssh-key
      E2E_AZURE_PUBLIC_KEY_PATH: /tmp/agent-qa-azure-ssh-key.pub
      E2E_COMMIT_SHA: $CI_COMMIT_SHORT_SHA
      E2E_COVERAGE_OUT_DIR: $CI_PROJECT_DIR/coverage
      E2E_GCP_PRIVATE_KEY_PATH: /tmp/agent-qa-gcp-ssh-key
      E2E_GCP_PUBLIC_KEY_PATH: /tmp/agent-qa-gcp-ssh-key.pub
      E2E_KEY_PAIR_NAME: datadog-agent-ci-rsa
      E2E_LOGS_PROCESSING_TEST_DEPTH: 1
      E2E_OUTPUT_DIR: $CI_PROJECT_DIR/e2e-output
      E2E_PIPELINE_ID: $CI_PIPELINE_ID
      E2E_RESULT_JSON: $CI_PROJECT_DIR/e2e_test_output.json
      E2E_USE_AWS_PROFILE: 'true'
      EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
-     EXTRA_PARAMS: --run "TestEC2(VM|VMSELinux|VMWKit)Suite"
+     EXTRA_PARAMS: --run "TestEC2(VM|VMSELinux|VMWKit|VMDirect)Suite"
?                                                     +++++++++
      FLAKY_PATTERNS_CONFIG: $CI_PROJECT_DIR/flaky-patterns-runtime.yaml
      GIT_STRATEGY: clone
      KUBERNETES_CPU_REQUEST: 6
      KUBERNETES_MEMORY_LIMIT: 16Gi
      KUBERNETES_MEMORY_REQUEST: 12Gi
      MAX_RETRIES_FLAG: ''
      ON_NIGHTLY_FIPS: 'true'
      PRE_BUILT_BINARIES_FLAG: --use-prebuilt-binaries
      REMOTE_STACK_CLEANING: 'true'
      SHOULD_RUN_IN_FLAKES_FINDER: 'true'
      TARGETS: ./tests/npm
      TEAM: cloud-network-monitoring

Changes Summary

Removed	Modified	Added	Renamed
0	19	0	0

ℹ️ Diff available in the job log.

agent-platform-auto-pr · 2026-01-31T02:50:55Z

Go Package Import Differences

Baseline: d8f3269
Comparison: 6642f6d

binary	os	arch	change
system-probe	linux	amd64	+132, -0 +github.com/DataDog/agent-payload/v5/contimage +github.com/DataDog/agent-payload/v5/contlcycle +github.com/DataDog/agent-payload/v5/sbom +github.com/DataDog/datadog-agent/comp/api/api/utils +github.com/DataDog/datadog-agent/comp/api/api/utils/stream +github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/types +github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/def +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/fx +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/impl +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/endpoints +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/resolver +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction +github.com/DataDog/datadog-agent/comp/forwarder/eventplatform +github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl +github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver +github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl +github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags +github.com/DataDog/datadog-agent/comp/networkpath/npcollector +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/model +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl/common +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl/connfilter +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl/pathteststore +github.com/DataDog/datadog-agent/comp/rdnsquerier/def +github.com/DataDog/datadog-agent/comp/rdnsquerier/fx +github.com/DataDog/datadog-agent/comp/rdnsquerier/impl +github.com/DataDog/datadog-agent/comp/rdnsquerier/impl-none +github.com/DataDog/datadog-agent/comp/system-probe/compliance/def +github.com/DataDog/datadog-agent/comp/system-probe/compliance/fx +github.com/DataDog/datadog-agent/comp/system-probe/compliance/impl +github.com/DataDog/datadog-agent/comp/system-probe/discovery/def +github.com/DataDog/datadog-agent/comp/system-probe/discovery/fx +github.com/DataDog/datadog-agent/comp/system-probe/discovery/impl +github.com/DataDog/datadog-agent/comp/system-probe/dynamicinstrumentation/def +github.com/DataDog/datadog-agent/comp/system-probe/dynamicinstrumentation/fx +github.com/DataDog/datadog-agent/comp/system-probe/dynamicinstrumentation/impl +github.com/DataDog/datadog-agent/comp/system-probe/ebpf/def +github.com/DataDog/datadog-agent/comp/system-probe/ebpf/fx +github.com/DataDog/datadog-agent/comp/system-probe/ebpf/impl +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/def +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/fx +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/impl +github.com/DataDog/datadog-agent/comp/system-probe/gpu/def +github.com/DataDog/datadog-agent/comp/system-probe/gpu/fx +github.com/DataDog/datadog-agent/comp/system-probe/gpu/impl +github.com/DataDog/datadog-agent/comp/system-probe/languagedetection/def +github.com/DataDog/datadog-agent/comp/system-probe/languagedetection/fx +github.com/DataDog/datadog-agent/comp/system-probe/languagedetection/impl +github.com/DataDog/datadog-agent/comp/system-probe/module +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/def +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/fx +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/impl +github.com/DataDog/datadog-agent/comp/system-probe/oomkill/def +github.com/DataDog/datadog-agent/comp/system-probe/oomkill/fx +github.com/DataDog/datadog-agent/comp/system-probe/oomkill/impl +github.com/DataDog/datadog-agent/comp/system-probe/ping/def +github.com/DataDog/datadog-agent/comp/system-probe/ping/fx +github.com/DataDog/datadog-agent/comp/system-probe/ping/impl +github.com/DataDog/datadog-agent/comp/system-probe/privilegedlogs/def +github.com/DataDog/datadog-agent/comp/system-probe/privilegedlogs/fx +github.com/DataDog/datadog-agent/comp/system-probe/privilegedlogs/impl +github.com/DataDog/datadog-agent/comp/system-probe/process/def +github.com/DataDog/datadog-agent/comp/system-probe/process/fx +github.com/DataDog/datadog-agent/comp/system-probe/process/impl +github.com/DataDog/datadog-agent/comp/system-probe/tcpqueuelength/def +github.com/DataDog/datadog-agent/comp/system-probe/tcpqueuelength/fx +github.com/DataDog/datadog-agent/comp/system-probe/tcpqueuelength/impl +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/def +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/fx +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/impl +github.com/DataDog/datadog-agent/comp/system-probe/types +github.com/DataDog/datadog-agent/pkg/clusteragent/api/v1 +github.com/DataDog/datadog-agent/pkg/clusteragent/clusterchecks/types +github.com/DataDog/datadog-agent/pkg/gpu/tags +github.com/DataDog/datadog-agent/pkg/hosttags +github.com/DataDog/datadog-agent/pkg/orchestrator/model +github.com/DataDog/datadog-agent/pkg/persistentcache +github.com/DataDog/datadog-agent/pkg/process/runner/endpoint +github.com/DataDog/datadog-agent/pkg/process/util/api +github.com/DataDog/datadog-agent/pkg/process/util/api/config +github.com/DataDog/datadog-agent/pkg/process/util/api/headers +github.com/DataDog/datadog-agent/pkg/util/clusteragent +github.com/DataDog/datadog-agent/pkg/util/containers/metrics +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/containerd +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/cri +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/docker +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/ecsfargate +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/ecsmanagedinstances +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/kubelet +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/provider +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/system +github.com/DataDog/datadog-agent/pkg/util/ec2/tags +github.com/DataDog/datadog-agent/pkg/util/kubernetes/cloudprovider +github.com/DataDog/datadog-agent/pkg/util/kubernetes/clusterinfo +github.com/DataDog/datadog-agent/pkg/util/kubernetes/clustername +github.com/DataDog/datadog-agent/pkg/util/kubernetes/hostinfo +github.com/DataDog/datadog-agent/pkg/util/trie +github.com/aws/aws-sdk-go-v2/aws +github.com/aws/aws-sdk-go-v2/aws/defaults +github.com/aws/aws-sdk-go-v2/aws/middleware +github.com/aws/aws-sdk-go-v2/aws/protocol/ec2query +github.com/aws/aws-sdk-go-v2/aws/protocol/query +github.com/aws/aws-sdk-go-v2/aws/ratelimit +github.com/aws/aws-sdk-go-v2/aws/retry +github.com/aws/aws-sdk-go-v2/aws/signer/v4 +github.com/aws/aws-sdk-go-v2/aws/transport/http +github.com/aws/aws-sdk-go-v2/credentials +github.com/aws/aws-sdk-go-v2/service/ec2 +github.com/aws/aws-sdk-go-v2/service/ec2/types +github.com/aws/smithy-go +github.com/aws/smithy-go/auth +github.com/aws/smithy-go/auth/bearer +github.com/aws/smithy-go/context +github.com/aws/smithy-go/document +github.com/aws/smithy-go/encoding +github.com/aws/smithy-go/encoding/httpbinding +github.com/aws/smithy-go/encoding/xml +github.com/aws/smithy-go/endpoints +github.com/aws/smithy-go/endpoints/private/rulesfn +github.com/aws/smithy-go/io +github.com/aws/smithy-go/logging +github.com/aws/smithy-go/metrics +github.com/aws/smithy-go/middleware +github.com/aws/smithy-go/ptr +github.com/aws/smithy-go/rand +github.com/aws/smithy-go/time +github.com/aws/smithy-go/tracing +github.com/aws/smithy-go/transport/http +github.com/aws/smithy-go/waiter +golang.org/x/sync/semaphore
system-probe	linux	arm64	+132, -0 +github.com/DataDog/agent-payload/v5/contimage +github.com/DataDog/agent-payload/v5/contlcycle +github.com/DataDog/agent-payload/v5/sbom +github.com/DataDog/datadog-agent/comp/api/api/utils +github.com/DataDog/datadog-agent/comp/api/api/utils/stream +github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/types +github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/def +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/fx +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/impl +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/endpoints +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/resolver +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction +github.com/DataDog/datadog-agent/comp/forwarder/eventplatform +github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl +github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver +github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl +github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags +github.com/DataDog/datadog-agent/comp/networkpath/npcollector +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/model +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl/common +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl/connfilter +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/npcollectorimpl/pathteststore +github.com/DataDog/datadog-agent/comp/rdnsquerier/def +github.com/DataDog/datadog-agent/comp/rdnsquerier/fx +github.com/DataDog/datadog-agent/comp/rdnsquerier/impl +github.com/DataDog/datadog-agent/comp/rdnsquerier/impl-none +github.com/DataDog/datadog-agent/comp/system-probe/compliance/def +github.com/DataDog/datadog-agent/comp/system-probe/compliance/fx +github.com/DataDog/datadog-agent/comp/system-probe/compliance/impl +github.com/DataDog/datadog-agent/comp/system-probe/discovery/def +github.com/DataDog/datadog-agent/comp/system-probe/discovery/fx +github.com/DataDog/datadog-agent/comp/system-probe/discovery/impl +github.com/DataDog/datadog-agent/comp/system-probe/dynamicinstrumentation/def +github.com/DataDog/datadog-agent/comp/system-probe/dynamicinstrumentation/fx +github.com/DataDog/datadog-agent/comp/system-probe/dynamicinstrumentation/impl +github.com/DataDog/datadog-agent/comp/system-probe/ebpf/def +github.com/DataDog/datadog-agent/comp/system-probe/ebpf/fx +github.com/DataDog/datadog-agent/comp/system-probe/ebpf/impl +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/def +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/fx +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/impl +github.com/DataDog/datadog-agent/comp/system-probe/gpu/def +github.com/DataDog/datadog-agent/comp/system-probe/gpu/fx +github.com/DataDog/datadog-agent/comp/system-probe/gpu/impl +github.com/DataDog/datadog-agent/comp/system-probe/languagedetection/def +github.com/DataDog/datadog-agent/comp/system-probe/languagedetection/fx +github.com/DataDog/datadog-agent/comp/system-probe/languagedetection/impl +github.com/DataDog/datadog-agent/comp/system-probe/module +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/def +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/fx +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/impl +github.com/DataDog/datadog-agent/comp/system-probe/oomkill/def +github.com/DataDog/datadog-agent/comp/system-probe/oomkill/fx +github.com/DataDog/datadog-agent/comp/system-probe/oomkill/impl +github.com/DataDog/datadog-agent/comp/system-probe/ping/def +github.com/DataDog/datadog-agent/comp/system-probe/ping/fx +github.com/DataDog/datadog-agent/comp/system-probe/ping/impl +github.com/DataDog/datadog-agent/comp/system-probe/privilegedlogs/def +github.com/DataDog/datadog-agent/comp/system-probe/privilegedlogs/fx +github.com/DataDog/datadog-agent/comp/system-probe/privilegedlogs/impl +github.com/DataDog/datadog-agent/comp/system-probe/process/def +github.com/DataDog/datadog-agent/comp/system-probe/process/fx +github.com/DataDog/datadog-agent/comp/system-probe/process/impl +github.com/DataDog/datadog-agent/comp/system-probe/tcpqueuelength/def +github.com/DataDog/datadog-agent/comp/system-probe/tcpqueuelength/fx +github.com/DataDog/datadog-agent/comp/system-probe/tcpqueuelength/impl +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/def +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/fx +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/impl +github.com/DataDog/datadog-agent/comp/system-probe/types +github.com/DataDog/datadog-agent/pkg/clusteragent/api/v1 +github.com/DataDog/datadog-agent/pkg/clusteragent/clusterchecks/types +github.com/DataDog/datadog-agent/pkg/gpu/tags +github.com/DataDog/datadog-agent/pkg/hosttags +github.com/DataDog/datadog-agent/pkg/orchestrator/model +github.com/DataDog/datadog-agent/pkg/persistentcache +github.com/DataDog/datadog-agent/pkg/process/runner/endpoint +github.com/DataDog/datadog-agent/pkg/process/util/api +github.com/DataDog/datadog-agent/pkg/process/util/api/config +github.com/DataDog/datadog-agent/pkg/process/util/api/headers +github.com/DataDog/datadog-agent/pkg/util/clusteragent +github.com/DataDog/datadog-agent/pkg/util/containers/metrics +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/containerd +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/cri +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/docker +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/ecsfargate +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/ecsmanagedinstances +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/kubelet +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/provider +github.com/DataDog/datadog-agent/pkg/util/containers/metrics/system +github.com/DataDog/datadog-agent/pkg/util/ec2/tags +github.com/DataDog/datadog-agent/pkg/util/kubernetes/cloudprovider +github.com/DataDog/datadog-agent/pkg/util/kubernetes/clusterinfo +github.com/DataDog/datadog-agent/pkg/util/kubernetes/clustername +github.com/DataDog/datadog-agent/pkg/util/kubernetes/hostinfo +github.com/DataDog/datadog-agent/pkg/util/trie +github.com/aws/aws-sdk-go-v2/aws +github.com/aws/aws-sdk-go-v2/aws/defaults +github.com/aws/aws-sdk-go-v2/aws/middleware +github.com/aws/aws-sdk-go-v2/aws/protocol/ec2query +github.com/aws/aws-sdk-go-v2/aws/protocol/query +github.com/aws/aws-sdk-go-v2/aws/ratelimit +github.com/aws/aws-sdk-go-v2/aws/retry +github.com/aws/aws-sdk-go-v2/aws/signer/v4 +github.com/aws/aws-sdk-go-v2/aws/transport/http +github.com/aws/aws-sdk-go-v2/credentials +github.com/aws/aws-sdk-go-v2/service/ec2 +github.com/aws/aws-sdk-go-v2/service/ec2/types +github.com/aws/smithy-go +github.com/aws/smithy-go/auth +github.com/aws/smithy-go/auth/bearer +github.com/aws/smithy-go/context +github.com/aws/smithy-go/document +github.com/aws/smithy-go/encoding +github.com/aws/smithy-go/encoding/httpbinding +github.com/aws/smithy-go/encoding/xml +github.com/aws/smithy-go/endpoints +github.com/aws/smithy-go/endpoints/private/rulesfn +github.com/aws/smithy-go/io +github.com/aws/smithy-go/logging +github.com/aws/smithy-go/metrics +github.com/aws/smithy-go/middleware +github.com/aws/smithy-go/ptr +github.com/aws/smithy-go/rand +github.com/aws/smithy-go/time +github.com/aws/smithy-go/tracing +github.com/aws/smithy-go/transport/http +github.com/aws/smithy-go/waiter +golang.org/x/sync/semaphore
system-probe	windows	amd64	+26, -0 +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/def +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/endpoints +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/resolver +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction +github.com/DataDog/datadog-agent/comp/networkpath/npcollector +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/model +github.com/DataDog/datadog-agent/comp/system-probe/crashdetect/def +github.com/DataDog/datadog-agent/comp/system-probe/crashdetect/fx +github.com/DataDog/datadog-agent/comp/system-probe/crashdetect/impl +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/def +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/fx +github.com/DataDog/datadog-agent/comp/system-probe/eventmonitor/impl +github.com/DataDog/datadog-agent/comp/system-probe/module +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/def +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/fx +github.com/DataDog/datadog-agent/comp/system-probe/networktracer/impl +github.com/DataDog/datadog-agent/comp/system-probe/softwareinventory/def +github.com/DataDog/datadog-agent/comp/system-probe/softwareinventory/fx +github.com/DataDog/datadog-agent/comp/system-probe/softwareinventory/impl +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/def +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/fx +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/impl +github.com/DataDog/datadog-agent/comp/system-probe/types +github.com/DataDog/datadog-agent/pkg/orchestrator/model +golang.org/x/sync/semaphore
system-probe	darwin	amd64	+24, -21 +github.com/DataDog/agent-payload/v5/process -github.com/DataDog/datadog-agent/comp/core/tagger/fx-remote -github.com/DataDog/datadog-agent/comp/core/tagger/generic_store -github.com/DataDog/datadog-agent/comp/core/tagger/impl-remote -github.com/DataDog/datadog-agent/comp/core/tagger/telemetry -github.com/DataDog/datadog-agent/comp/core/workloadfilter/baseimpl -github.com/DataDog/datadog-agent/comp/core/workloadfilter/catalog -github.com/DataDog/datadog-agent/comp/core/workloadfilter/fx-remote -github.com/DataDog/datadog-agent/comp/core/workloadfilter/impl/parse -github.com/DataDog/datadog-agent/comp/core/workloadfilter/program -github.com/DataDog/datadog-agent/comp/core/workloadfilter/proto -github.com/DataDog/datadog-agent/comp/core/workloadfilter/remoteimpl -github.com/DataDog/datadog-agent/comp/core/workloadfilter/telemetry -github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/catalog-remote -github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/sbomutil -github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx -github.com/DataDog/datadog-agent/comp/core/workloadmeta/impl -github.com/DataDog/datadog-agent/comp/core/workloadmeta/mock -github.com/DataDog/datadog-agent/comp/core/workloadmeta/proto -github.com/DataDog/datadog-agent/comp/core/workloadmeta/telemetry +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/def +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/endpoints +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/resolver +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction +github.com/DataDog/datadog-agent/comp/networkpath/npcollector +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/model +github.com/DataDog/datadog-agent/comp/system-probe/module +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/def +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/fx +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/impl +github.com/DataDog/datadog-agent/comp/system-probe/types -github.com/DataDog/datadog-agent/pkg/errors +github.com/DataDog/datadog-agent/pkg/orchestrator/model -github.com/DataDog/datadog-agent/pkg/util/retry +github.com/DataDog/mmh3 +github.com/DataDog/zstd_0 +github.com/gogo/protobuf/jsonpb +github.com/gogo/protobuf/sortkeys +github.com/gogo/protobuf/types +github.com/klauspost/compress +github.com/klauspost/compress/fse +github.com/klauspost/compress/huff0 +github.com/klauspost/compress/zstd +golang.org/x/sync/semaphore
system-probe	darwin	arm64	+24, -21 +github.com/DataDog/agent-payload/v5/process -github.com/DataDog/datadog-agent/comp/core/tagger/fx-remote -github.com/DataDog/datadog-agent/comp/core/tagger/generic_store -github.com/DataDog/datadog-agent/comp/core/tagger/impl-remote -github.com/DataDog/datadog-agent/comp/core/tagger/telemetry -github.com/DataDog/datadog-agent/comp/core/workloadfilter/baseimpl -github.com/DataDog/datadog-agent/comp/core/workloadfilter/catalog -github.com/DataDog/datadog-agent/comp/core/workloadfilter/fx-remote -github.com/DataDog/datadog-agent/comp/core/workloadfilter/impl/parse -github.com/DataDog/datadog-agent/comp/core/workloadfilter/program -github.com/DataDog/datadog-agent/comp/core/workloadfilter/proto -github.com/DataDog/datadog-agent/comp/core/workloadfilter/remoteimpl -github.com/DataDog/datadog-agent/comp/core/workloadfilter/telemetry -github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/catalog-remote -github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/sbomutil -github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx -github.com/DataDog/datadog-agent/comp/core/workloadmeta/impl -github.com/DataDog/datadog-agent/comp/core/workloadmeta/mock -github.com/DataDog/datadog-agent/comp/core/workloadmeta/proto -github.com/DataDog/datadog-agent/comp/core/workloadmeta/telemetry +github.com/DataDog/datadog-agent/comp/forwarder/connectionsforwarder/def +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/endpoints +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/resolver +github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction +github.com/DataDog/datadog-agent/comp/networkpath/npcollector +github.com/DataDog/datadog-agent/comp/networkpath/npcollector/model +github.com/DataDog/datadog-agent/comp/system-probe/module +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/def +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/fx +github.com/DataDog/datadog-agent/comp/system-probe/traceroute/impl +github.com/DataDog/datadog-agent/comp/system-probe/types -github.com/DataDog/datadog-agent/pkg/errors +github.com/DataDog/datadog-agent/pkg/orchestrator/model -github.com/DataDog/datadog-agent/pkg/util/retry +github.com/DataDog/mmh3 +github.com/DataDog/zstd_0 +github.com/gogo/protobuf/jsonpb +github.com/gogo/protobuf/sortkeys +github.com/gogo/protobuf/types +github.com/klauspost/compress +github.com/klauspost/compress/fse +github.com/klauspost/compress/huff0 +github.com/klauspost/compress/zstd +golang.org/x/sync/semaphore

brycekahle added 30 commits January 29, 2026 11:56

add direct send loop

6f74c3b

use event consumer

c90a5b8

add sender tests

abfd5fa

add telemetry

89f46f0

add e2e test for direct send

884d78d

add build constraints

6231225

do not use full event monitor

473863b

ignore not exist errors for cwd

2f9b37f

fix log for direct send consumer

968568c

add debug logs

12afa25

add process_config keys to config sync

7ffb55c

lifecycle doesn't work that deep

f6d2c09

increase size of kmt-ramfs to 6G

867d755

reuse buffer for usm encoders

0d451e7

fix process agent profiler enabled

ea0fdf1

fix container resolve modification

5f87909

reset buf before calling EncodeConnectionDirect

8dcbc96

attempt at system-probe modules as components

860bedd

wrap ebpf, oom_kill, tcpq modules

173c471

add compliance

073a58c

add discovery module

5c37864

add dynamic instrumentation

3c15f1b

add gpu

1bc8346

add language detection

6075e34

add ping module

f3603a6

add privilegedlogs module

efd1473

add process module

00e6049

add softwareinventory module

aa70741

add traceroute module

d79c72b

add eventmonitor

6395828

brycekahle added 9 commits January 30, 2026 16:20

add crashdetect

0b89161

sort modules

d0ef424

fixup fx for unsupported build tags

2e1c8c4

fix module.Component for non-linux

63aac19

remove lifecycle where not needed

658c189

remove dup fx modules

c204b3d

comp lint

193ac23

fixes

252b8e2

remove todo

0c628fc

github-actions bot added component/system-probe long review PR is complex, plan time to review it team/ebpf-platform labels Jan 31, 2026

fix CODEOWNERS

6642f6d

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

treat system-probe modules as components #45771

treat system-probe modules as components #45771

Uh oh!

brycekahle commented Jan 31, 2026

Uh oh!

agent-platform-auto-pr bot commented Jan 31, 2026 •

edited

Loading

Modified Jobs

Uh oh!

agent-platform-auto-pr bot commented Jan 31, 2026 •

edited

Loading

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

treat system-probe modules as components #45771

Are you sure you want to change the base?

treat system-probe modules as components #45771

Uh oh!

Conversation

brycekahle commented Jan 31, 2026

What does this PR do?

Motivation

Describe how you validated your changes

Additional Notes

Uh oh!

agent-platform-auto-pr bot commented Jan 31, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Gitlab CI Configuration Changes

Modified Jobs

Changes Summary

Uh oh!

agent-platform-auto-pr bot commented Jan 31, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Go Package Import Differences

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

agent-platform-auto-pr bot commented Jan 31, 2026 •

edited

Loading

agent-platform-auto-pr bot commented Jan 31, 2026 •

edited

Loading