Embedding

.gitlab-ci.yml

@@ -13,22 +13,28 @@ workflow:
         FUNCTIONAL_TEST: "no"
     - if: $CI_MERGE_REQUEST_LABELS =~ /Run tests/ && $CI_MERGE_REQUEST_TARGET_BRANCH_SHA != ""
       variables:
-        FUNCTIONAL_TEST: "yes"
-        FUNCTIONAL_TEST_SCOPE: mr
         UNIT_TEST_REPEAT: 5
         UNIT_TEST_TIMEOUT: 50
+        FUNCTIONAL_TEST: "yes"
+        FUNCTIONAL_TEST_SCOPE: mr
+        FUNCTIONAL_TEST_CLUSTER_A100: ""
+        FUNCTIONAL_TEST_CLUSTER_H100: ""
     - if: $CI_MERGE_REQUEST_LABELS =~ /Run nightly/ && $CI_MERGE_REQUEST_TARGET_BRANCH_SHA != ""
       variables:
-        FUNCTIONAL_TEST: "yes"
-        FUNCTIONAL_TEST_SCOPE: nightly
         UNIT_TEST_REPEAT: 5
         UNIT_TEST_TIMEOUT: 50
+        FUNCTIONAL_TEST: "yes"
+        FUNCTIONAL_TEST_SCOPE: nightly
+        FUNCTIONAL_TEST_CLUSTER_A100: ""
+        FUNCTIONAL_TEST_CLUSTER_H100: ""
     - if: $CI_MERGE_REQUEST_LABELS =~ /Run weekly/ && $CI_MERGE_REQUEST_TARGET_BRANCH_SHA != ""
       variables:
-        FUNCTIONAL_TEST: "yes"
-        FUNCTIONAL_TEST_SCOPE: weekly
         UNIT_TEST_REPEAT: 5
         UNIT_TEST_TIMEOUT: 50
+        FUNCTIONAL_TEST: "yes"
+        FUNCTIONAL_TEST_SCOPE: weekly
+        FUNCTIONAL_TEST_CLUSTER_A100: ""
+        FUNCTIONAL_TEST_CLUSTER_H100: ""
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_SHA != ""
       variables:
         FUNCTIONAL_TEST: "no"
@@ -58,29 +64,23 @@ variables:
       - "mr"
       - "nightly"
       - "weekly"
+      - "pre-release"
+      - "release"
     description: "Testsuite to run (only for FUNCTIONAL_TEST=yes)"
-  FUNCTIONAL_TEST_CLUSTER:
+  FUNCTIONAL_TEST_CLUSTER_A100:
     value: "dgxa100_dracooci"
     options:
       - "dgxa100_dracooci"
       - "dgxa100_dracooci-ord"
-      - "dgxh100_eos"
-    description: '"dgxa100_dracooci" for OCI-IAD, "dgxh100_eos" for EOS'
-  CONVERGENCE_TEST:
-    value: "no"
+    description: 'Cluster for A100 workloads'
+  FUNCTIONAL_TEST_CLUSTER_H100:
+    value: "dgxh100_eos"
     options:
-      - "yes"
-      - "no"
-    description: To run a convergence test
-  CONVERGENCE_TEST_SCOPE:
-    value: "release"
-    options:
-      - "release"
-      - "pre-release"
-    description: "Test suite to run (only for CONVERGENCE_TEST=yes)"
-  CONVERGENCE_TEST_RUN_NAME:
-    value: "pre-release-$$CI_PIPELINE_ID"
-    description: "Run directory of convergence test"
+      - "dgxh100_coreweave"
+      - "dgxh100_eos"
+    description: 'Cluster for H100 workloads'
+  FUNCTIONAL_TEST_NAME:
+    description: "Name of functional test run (only for pre-release and release)"
   PUBLISH: 
     value: "no"
     options: 
@@ -96,6 +96,7 @@ variables:
 
   # CI wide variables
   CI_MCORE_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_ci
+  CI_MCORE_DEV_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_ci_dev
   CI_NEMO_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/nemo_ci
   LINTING_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_linting
   UNIT_TEST_TIMEOUT: 15
@@ -105,5 +106,4 @@ include:
   - .gitlab/stages/00.pre.yml
   - .gitlab/stages/01.tests.yml
   - .gitlab/stages/02.functional-tests.yml
-  - .gitlab/stages/03.convergence-tests.yml
-  - .gitlab/stages/04.publish.yml
+  - .gitlab/stages/03.publish.yml

.gitlab/stages/00.pre.yml

@@ -76,9 +76,10 @@ clean_docker_node:
     matrix:
       - node: 8xL40S
       - node: mcore-docker-node-small
+      - node: mcore-docker-node-jet
   script:
     - export DOCKER_HOST='unix:///var/run/docker.sock'
-    - docker system prune -a --filter "until=48h" -f || true
+    - docker system prune -a --filter "until=36h" -f || true
 
 maybe_cherry_pick_commit:
   rules:
@@ -101,8 +102,13 @@ maybe_cherry_pick_commit:
     - git config --global user.email "[email protected]"
     - git config --global user.name "Mcore Bot"
     - |
-      LABELS=$(curl --header "PRIVATE-TOKEN: ${PROJECT_ACCESS_TOKEN_MCORE}" --url "https://${GITLAB_ENDPOINT}/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}" | jq '.labels | join(",")' | tr -d '"')
-
+      MR=$(curl --header "PRIVATE-TOKEN: ${PROJECT_ACCESS_TOKEN_MCORE}" --url "https://${GITLAB_ENDPOINT}/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}")
+
+      LABELS=$(echo -E $MR | jq '.labels | join(",")' | tr -d '"')
+      AUTHOR_ID=$(echo -E $MR | jq '.author.id' | tr -d '"')
+      AUTHOR_NAME=$(echo -E $MR | jq '.author.username' | tr -d '"')
+      TITLE=$(echo -E $MR | jq '.title' | tr -d '"')
+      MILESTONE_ID=$(echo -E $MR | jq '.milestone.id' | tr -d '"')
       TARGET_BRANCHES=$(echo "$LABELS" | grep -o 'core_[^,]*')
 
       if [[ $TARGET_BRANCHES == "" ]]; then
@@ -134,8 +140,11 @@ maybe_cherry_pick_commit:
             --url https://${GITLAB_ENDPOINT}/api/v4/projects/${CI_PROJECT_ID}/merge_requests \
             -d "source_branch=cherry-pick-$MR_ID-$RELEASE_BRANCH" \
             -d "target_branch=$RELEASE_BRANCH" \
-            -d "title=Cherry-pick $MR_ID into $RELEASE_BRANCH" \
-            -d "labels=cherry-pick"
+            -d "title=Cherry pick \`$TITLE ($MR_ID)\` into \`$RELEASE_BRANCH\`" \
+            -d "labels=cherry-pick" \
+            -d "reviewer_ids=$AUTHOR_ID" \
+            -d "milestone_id=$MILESTONE_ID" \
+            -d "description=[🤖]: Hi @$AUTHOR_NAME 👋,<br><br>we've cherry picked \`$TITLE ($MR_ID)\` into \`$RELEASE_BRANCH\` for you! 🚀<br><br>Please review and approve this cherry pick by your convenience\!"
 
         else
           URL=https://${GITLAB_ENDPOINT}/ADLR/megatron-lm/-/merge_requests/$MR_ID

.gitlab/stages/01.tests.yml

@@ -21,6 +21,10 @@ build_image:
         FILE: Dockerfile.ci
         BASE_IMAGE: nvcr.io/nvidia/pytorch:24.01-py3
         TAG: mcore-docker-node-large
+      - IMAGE: CI_MCORE_DEV_IMAGE
+        FILE: Dockerfile.ci.dev
+        BASE_IMAGE: nvcr.io/nvidia/pytorch:24.01-py3
+        TAG: mcore-docker-node-large
       - IMAGE: CI_NEMO_IMAGE
         FILE: Dockerfile.ci
         BASE_IMAGE: nvcr.io/nvidian/nemo:nightly
@@ -35,48 +39,44 @@ build_image:
   variables:
     STAGE: main
   script:
+    - apk add bash
     - |
-      set -x
-      env
-      eval "IMAGE=\$$IMAGE"
-
-      docker system prune -a --filter "until=24h" -f || true
-
-      if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
-        ADDITIONAL_PARAMS="--pull"
-      fi
-
-      docker pull ${IMAGE}:${CI_PIPELINE_ID} || true
-      docker pull ${IMAGE}:${CI_MERGE_REQUEST_IID:-noop} || true
-      docker pull ${IMAGE}:buildcache || true
-
-      docker build \
-        --secret id=JET_INDEX_URLS \
-        --target $STAGE \
-        -f $FILE \
-        -t ${IMAGE}:${CI_PIPELINE_ID} \
-        -t ${IMAGE}:${CI_MERGE_REQUEST_IID:-noop} \
-        --build-arg CACHEBUST=$(cat /proc/sys/kernel/random/uuid) \
-        --cache-to type=inline \
-        --cache-from type=registry,ref=${IMAGE}:buildcache \
-        --cache-from type=registry,ref=${IMAGE}:${CI_PIPELINE_ID} \
-        --cache-from type=registry,ref=${IMAGE}:${CI_MERGE_REQUEST_IID:-noop} \
-        --build-arg FROM_IMAGE_NAME=$BASE_IMAGE \
-        ${ADDITIONAL_PARAMS} .
-
-      docker push ${IMAGE}:${CI_PIPELINE_ID}
-      docker push ${IMAGE}:${CI_MERGE_REQUEST_IID:-noop}
-
-      if [[ "$CI_COMMIT_BRANCH" == "ci-nightly-a100" ]]; then
-        docker tag ${IMAGE}:${CI_PIPELINE_ID} ${IMAGE}:nightly
-        docker push ${IMAGE}:nightly
-      fi
+      bash -c '
+        set -x
+        env
+        eval "IMAGE=\$$IMAGE"
+
+        docker system prune -a --filter "until=24h" -f || true
+
+        docker buildx create --name container --driver=docker-container
+
+        ADDITIONAL_PARAMS=()
+
+        if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
+          ADDITIONAL_PARAMS+=("--pull")
+          ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:main")
+        fi
 
-      if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
-        docker tag ${IMAGE}:${CI_PIPELINE_ID} ${IMAGE}:buildcache
-        docker push ${IMAGE}:buildcache
-      fi
+        if [[ "$CI_COMMIT_BRANCH" == "ci-nightly-a100" ]]; then
+          ADDITIONAL_PARAMS+=("-t ${IMAGE}:nightly")
+        fi
 
+        DOCKER_BUILDKIT=1 docker build \
+          --secret id=JET_INDEX_URLS \
+          --target $STAGE \
+          -f $FILE \
+          -t ${IMAGE}:${CI_PIPELINE_ID} \
+          --builder=container \
+          --build-arg CACHEBUST=$(cat /proc/sys/kernel/random/uuid) \
+          --cache-to type=registry,ref=${IMAGE}-buildcache:${CI_PIPELINE_ID} \
+          --cache-to type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID:-noop} \
+          --cache-from type=registry,ref=${IMAGE}-buildcache:main \
+          --cache-from type=registry,ref=${IMAGE}-buildcache:${CI_PIPELINE_ID} \
+          --cache-from type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID:-noop} \
+          --build-arg FROM_IMAGE_NAME=$BASE_IMAGE \
+          --push \
+          ${ADDITIONAL_PARAMS[@]} .
+        '
   retry:
     max: 2
 
@@ -85,13 +85,17 @@ unit_tests:
   # the current code. This is a form of backwards compatibility testing
   # and helps in providing stable interfaces.
   extends: [.test_mr_rules]
-  image: ${CI_MCORE_IMAGE}:${CI_PIPELINE_ID}
+  image: ${IMAGE}:${CI_PIPELINE_ID}
   needs: [build_image]
   timeout: 180m
   parallel:
     matrix:
       - TAG: latest
-      - TAG: 8fc755388a03bae05cb740857008b8916e01a63c
+        IMAGE: ${CI_MCORE_IMAGE}
+      # - TAG: latest
+      #   IMAGE: ${CI_MCORE_DEV_IMAGE}
+      - TAG: core_r0.9.0
+        IMAGE: ${CI_MCORE_IMAGE}
   tags: [8xL40S]
   variables:
     GIT_STRATEGY: clone
@@ -112,11 +116,14 @@ unit_tests:
 
       for i in $(seq $UNIT_TEST_REPEAT); do
         SEED=$((RANDOM % 9000 + 1000));
-        SKIPPED=()
+        ARGS=()
         if [[ $TAG != latest ]]; then
-          SKIPPED+=(-m "not internal")
+          ARGS+=(-m "not internal")
+        fi
+        if [[ $IMAGE == ${CI_MCORE_DEV_IMAGE} ]]; then
+          ARGS+=(-m "experimental")
         fi
-        timeout ${UNIT_TEST_TIMEOUT}m torchrun --nproc_per_node=8 -m pytest --random-order --random-order-seed ${SEED} -xvs --cov-report=term --cov-report=html --cov=megatron/core --no-cov-on-fail "${SKIPPED[@]}" tests/unit_tests
+        timeout ${UNIT_TEST_TIMEOUT}m torchrun --nproc_per_node=8 -m pytest --random-order --random-order-seed ${SEED} -xvs --cov-report=term --cov-report=html --cov=megatron/core --no-cov-on-fail "${ARGS[@]}" tests/unit_tests
       done
   artifacts:
     paths:
@@ -125,10 +132,30 @@ unit_tests:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_PROTECTED != "true"
       allow_failure: true
       when: always
-    - if: '$TAG != "latest"'
-      allow_failure: true
     - when: always
 
+unit-tests-results-notify:
+  extends: [.test_mr_rules]
+  image: ${CI_MCORE_IMAGE}:${CI_PIPELINE_ID}
+  needs: [unit_tests]
+  tags:
+    - mcore-docker-node-small
+  script:
+    - env
+    - export WEBHOOK_URL=${MCORE_NOTIFICATION_HOOK}
+    - export RO_API_TOKEN=${PROJECT_ACCESS_TOKEN_MCORE}
+    - export GITLAB_ENDPOINT
+    - export DATE=$(date +"%Y-%m-%d")
+    - bash tests/functional_tests/shell_test_utils/notify_unit_tests.sh ${CI_PIPELINE_ID}
+  artifacts:
+    when: always
+    paths:
+      - scripts
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "ci-unit-test-extended"
+      when: always
+    - when: never
+
 docs_build_test:
   extends: [.test_mr_rules]
   image: ${CI_MCORE_IMAGE}:${CI_PIPELINE_ID}