diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
index 8ed58864abe8d..7466667925340 100644
--- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
+++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
@@ -57,13 +57,29 @@ parameters:
   type: string
   default: 2.42.0.251225
 
+# Equals values from common-variables.yml. TODO: Single source of truth.
 - name: CudaVersion
   displayName: CUDA version
-  type: string
-  default: '12.8'
-  values:
-  - 12.8
-  - 13.0
+  type: object
+  default:
+    cuda_12:
+      major: "12"
+      minor: "8"
+      archs: '75-real;86-real;89-real;90-virtual'
+      trt_version: '10.14.1.48'
+      win_cuda_home: '$(Agent.TempDirectory)\v12.8'
+      win_trt_home: '$(Agent.TempDirectory)\TensorRT-10.14.1.48.Windows.win10.cuda-12.9'
+      linux_trt_version: 10.14.1.48-1.cuda12.9
+      win_cudnn_home: ''
+    cuda_13:
+      major: "13"
+      minor: "0"
+      archs:  '75-real;80-real;86-real;89-real;90-real;100-real;120-real;120-virtual'
+      trt_version: '10.14.1.48'
+      win_cuda_home: '$(Agent.TempDirectory)\v13.0'
+      win_trt_home: '$(Agent.TempDirectory)\TensorRT-10.14.1.48.Windows.win10.cuda-13.0'
+      linux_trt_version: 10.14.1.48-1.cuda13.0
+      win_cudnn_home: '$(Agent.TempDirectory)\9.14.0.64_cuda13'
 
 resources:
   repositories:
@@ -76,29 +92,6 @@ resources:
     name: 1ESPipelineTemplates/1ESPipelineTemplates
     ref: refs/tags/release
 
-variables:
-- template: templates/common-variables.yml
-- name: win_trt_home
-  ${{ if eq(parameters.CudaVersion, '12.8') }}:
-    value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda12 }}
-  ${{ if eq(parameters.CudaVersion, '13.0') }}:
-    value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda13 }}
-- name: win_cuda_home
-  ${{ if eq(parameters.CudaVersion, '12.8') }}:
-    value: $(Agent.TempDirectory)\v12.8
-  ${{ if eq(parameters.CudaVersion, '13.0') }}:
-    value: $(Agent.TempDirectory)\v13.0
-- name: win_cudnn_home
-  ${{ if eq(parameters.CudaVersion, '12.8') }}:
-    value: ''
-  ${{ if eq(parameters.CudaVersion, '13.0') }}:
-    value: $(Agent.TempDirectory)\9.14.0.64_cuda13
-- name: CudaArchs
-  ${{ if eq(parameters.CudaVersion, '12.8') }}:
-    value: '75-real;86-real;89-real;90-virtual'
-  ${{ if eq(parameters.CudaVersion, '13.0') }}:
-    value: '75-real;80-real;86-real;89-real;90-real;100-real;120-real;120-virtual'
-
 extends:
   # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
   # For non-production pipelines, use "Unofficial" as defined below.
@@ -159,14 +152,14 @@ extends:
         is1ES: true
 
     - template: stages/java-cuda-packaging-stage.yml
+      parameters:
+        CudaVersion: ${{ parameters.CudaVersion }}
 
     - template: stages/nuget-combine-cuda-stage.yml
       parameters:
         CudaVersion: ${{ parameters.CudaVersion }}
         RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
         UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
-        win_trt_home: ${{ variables.win_trt_home }}
-        win_cuda_home: ${{ variables.win_cuda_home }}
         DoEsrp: true
         IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
         buildJava: true
@@ -175,8 +168,6 @@ extends:
         BuildId: ${{ parameters.BuildId }}
         PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
         PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
-        CudaArchs: ${{ variables.CudaArchs }}
-        win_cudnn_home: ${{ variables.win_cudnn_home }}
 
     - template: stages/nodejs-win-packaging-stage.yml
       parameters:
diff --git a/tools/ci_build/github/azure-pipelines/stages/c-api-linux-cpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/c-api-linux-cpu-stage.yml
deleted file mode 100644
index ea706a65fb4c9..0000000000000
--- a/tools/ci_build/github/azure-pipelines/stages/c-api-linux-cpu-stage.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-stages:
-- stage: Linux_C_API_Packaging_CPU
-  dependsOn: []
-  jobs:
-  - template: ../templates/c-api-linux-cpu.yml
-    parameters:
-      OnnxruntimeArch: 'x64'
-      OnnxruntimeNodejsBindingArch: 'x64'
-      PoolName: 'onnxruntime-Ubuntu2404-AMD-CPU'
-      PackageJava: false
-      PackageNodeJS: false
\ No newline at end of file
diff --git a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml
index da7d52f5e7760..698fed86b3ecc 100644
--- a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml
@@ -1,31 +1,37 @@
+parameters:
+  CudaVersion:
+    type: object
+
+
 stages:
-- stage: Jar_Packaging_GPU
-  dependsOn:
-  - Linux_C_API_Packaging_GPU
-  - Windows_Packaging_CUDA
-  - Windows_Packaging_TensorRT
-  jobs:
-  - job: Jar_Packaging_GPU
-    workspace:
-      clean: all
-    templateContext:
-      inputs:
-      - input: pipelineArtifact
-        artifactName: drop-onnxruntime-java-win-x64-tensorrt
-        targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'
+- ${{ each cuda_version in parameters.CudaVersion }}:
+  - stage: Jar_Packaging_GPU_${{cuda_version.key}}
+    dependsOn:
+    - Linux_C_API_Packaging_GPU_${{cuda_version.key}}
+    - Windows_Packaging_CUDA_${{cuda_version.key}}
+    - Windows_Packaging_TensorRT_${{cuda_version.key}}
+    jobs:
+    - job: Jar_Packaging_GPU
+      workspace:
+        clean: all
+      templateContext:
+        inputs:
+        - input: pipelineArtifact
+          artifactName: drop-onnxruntime-java-win-x64-tensorrt-${{cuda_version.key}}
+          targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'
+
+        - input: pipelineArtifact
+          artifactName: drop-onnxruntime-java-linux-x64-tensorrt-${{cuda_version.key}}
+          targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64'
 
-      - input: pipelineArtifact
-        artifactName: drop-onnxruntime-java-linux-x64-tensorrt
-        targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64'
-        
-      outputs:
-      - output: pipelineArtifact
-        targetPath: $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64
-        artifactName: onnxruntime-java-gpu
-    pool: 'onnxruntime-Win-CPU-VS2022-Latest'
-    dependsOn: []
-    condition: succeeded()
-    steps:
-    - template: ../templates/jar-packaging.yml
-      parameters:
-        package_type: gpu
\ No newline at end of file
+        outputs:
+        - output: pipelineArtifact
+          targetPath: $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64
+          artifactName: onnxruntime-java-gpu-${{cuda_version.key}}
+      pool: 'onnxruntime-Win-CPU-VS2022-Latest'
+      dependsOn: []
+      condition: succeeded()
+      steps:
+      - template: ../templates/jar-packaging.yml
+        parameters:
+          package_type: gpu
diff --git a/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml
index ff35d3e35ef6c..529a2aba60080 100644
--- a/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml
@@ -1,10 +1,11 @@
 parameters:
 - name: CudaVersion
+  type: object
+- name: StageToken
   type: string
-  default: '12.8'
 
 stages:
-- stage: Linux_Nodejs_Packaging_x64
+- stage: Linux_Nodejs_Packaging_x64_${{parameters.StageToken}}
   dependsOn: []
   jobs:
   - job: Linux_Nodejs_Packaging_x64
@@ -16,19 +17,12 @@ stages:
       name: 'onnxruntime-Ubuntu2404-AMD-CPU'
       os: linux
     variables:
-    - template: ../templates/common-variables.yml
     - name: CUDA_VERSION_MAJOR
-      ${{ if eq(parameters.CudaVersion, '13.0') }}:
-        value: '13'
-      ${{ if eq(parameters.CudaVersion, '12.8') }}:
-        value: '12'
+      value: ${{ parameters.CudaVersion.major }}
     - name: CUDA_VERSION
-      value: ${{ parameters.CudaVersion }}
+      value: "${{ parameters.CudaVersion.major }}.${{ parameters.CudaVersion.minor }}"
     - name: linux_trt_version
-      ${{ if eq(parameters.CudaVersion, '13.0') }}:
-        value: ${{ variables.linux_trt_version_cuda13 }}
-      ${{ if eq(parameters.CudaVersion, '12.8') }}:
-        value: ${{ variables.linux_trt_version_cuda12 }}
+      value: ${{ parameters.CudaVersion.linux_trt_version }}
     steps:
     - checkout: self
       clean: true
@@ -52,6 +46,6 @@ stages:
       parameters:
         arch: 'x64'
         os: 'linux'
-        artifactName: 'drop-onnxruntime-nodejs-linux-x64'
+        artifactName: 'drop-onnxruntime-nodejs-linux-x64-${{parameters.StageToken}}'
 
     - template: ../templates/clean-agent-build-directory-step.yml
diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml
index 0400b4b2233ec..072ff74c730e6 100644
--- a/tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml
@@ -6,13 +6,7 @@ parameters:
   type: boolean
 
 - name: CudaVersion
-  type: string
-
-- name: win_trt_home
-  type: string
-
-- name: win_cuda_home
-  type: string
+  type: object
 
 - name: UseIncreasedTimeoutForTests
   type: boolean
@@ -45,40 +39,35 @@ parameters:
   displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package.
   type: number
 
-- name: CudaArchs
-  type: string
-- name: win_cudnn_home
-  type: string
-  default: ''
-
 stages:
-- template: nuget-linux-cuda-packaging-stage.yml
-  parameters:
-    CudaVersion: ${{ parameters.CudaVersion }}
-    buildJava: ${{ parameters.buildJava }}
-    buildNodejs: ${{ parameters.buildNodejs }}
-
-- ${{ if eq(parameters.buildNodejs, 'true') }}:
-  - template: nodejs-linux-packaging-stage.yml
+- ${{ each cuda_version in parameters.CudaVersion }}:
+  - template: nuget-linux-cuda-packaging-stage.yml
     parameters:
-      CudaVersion: ${{ parameters.CudaVersion }}
-
-- template: nuget-win-cuda-packaging-stage.yml
-  parameters:
-    RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
-    UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
-    CudaVersion: ${{ parameters.CudaVersion }}
-    win_trt_home: ${{ parameters.win_trt_home }}
-    win_cuda_home: ${{ parameters.win_cuda_home }}
-    buildJava: ${{ parameters.buildJava }}
-    PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
-    PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
-    CudaArchs: ${{ parameters.CudaArchs }}
-    win_cudnn_home: ${{ parameters.win_cudnn_home }}
-
-- template: nuget-cuda-packaging-stage.yml
-  parameters:
-    DoEsrp: ${{ parameters.DoEsrp }}
-    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
-    SpecificArtifact: ${{ parameters.SpecificArtifact }}
-    BuildId: ${{ parameters.BuildId }}
+      CudaVersion: ${{ cuda_version.value }}
+      buildJava: ${{ parameters.buildJava }}
+      buildNodejs: ${{ parameters.buildNodejs }}
+      StageToken: ${{ cuda_version.key }}
+
+  - ${{ if eq(parameters.buildNodejs, 'true') }}:
+    - template: nodejs-linux-packaging-stage.yml
+      parameters:
+        StageToken: ${{ cuda_version.key }}
+        CudaVersion: ${{ cuda_version.value }}
+
+  - template: nuget-win-cuda-packaging-stage.yml
+    parameters:
+      RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
+      UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
+      CudaVersion: ${{ cuda_version.value }}
+      StageToken: ${{ cuda_version.key }}
+      buildJava: ${{ parameters.buildJava }}
+      PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
+      PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
+
+  - template: nuget-cuda-packaging-stage.yml
+    parameters:
+      DoEsrp: ${{ parameters.DoEsrp }}
+      IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
+      SpecificArtifact: ${{ parameters.SpecificArtifact }}
+      BuildId: ${{ parameters.BuildId }}
+      StageToken: ${{ cuda_version.key }}
diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml
index 55e78ae79b208..d4f86e9b07f1c 100644
--- a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml
@@ -13,15 +13,18 @@ parameters:
   type: string
   default: '0'
 
+- name: StageToken
+  type: string
+
 stages:
 ######## Nuget ########
 # Win/Linux CUDA Combined packaging
-- stage: NuGet_Packaging_GPU
+- stage: NuGet_Packaging_GPU_${{parameters.StageToken}}
   dependsOn:
   - Setup
-  - Windows_Packaging_CUDA
-  - Windows_Packaging_TensorRT
-  - Linux_C_API_Packaging_GPU
+  - Windows_Packaging_CUDA_${{parameters.StageToken}}
+  - Windows_Packaging_TensorRT_${{parameters.StageToken}}
+  - Linux_C_API_Packaging_GPU_${{parameters.StageToken}}
   #    This is need for Download Linux CustomOp TestData
   - Linux_C_API_Packaging_CPU
   condition: succeeded()
@@ -43,7 +46,7 @@ stages:
     - template: ../templates/flex-downloadPipelineArtifact.yml
       parameters:
         StepName: 'Download Pipeline Artifact - NuGet'
-        ArtifactName: 'onnxruntime-win-x64-cuda'
+        ArtifactName: 'onnxruntime-win-x64-${{parameters.StageToken}}'
         TargetPath: '$(Build.BinariesDirectory)/nuget-artifact'
         SpecificArtifact: ${{ parameters.SpecificArtifact }}
         BuildId: ${{ parameters.BuildId }}
@@ -51,7 +54,7 @@ stages:
     - template: ../templates/flex-downloadPipelineArtifact.yml
       parameters:
         StepName: 'Download Pipeline Artifact - NuGet'
-        ArtifactName: 'onnxruntime-win-x64-tensorrt'
+        ArtifactName: 'onnxruntime-win-x64-tensorrt-${{parameters.StageToken}}'
         TargetPath: '$(Build.BinariesDirectory)/nuget-artifact'
         SpecificArtifact: ${{ parameters.SpecificArtifact }}
         BuildId: ${{ parameters.BuildId }}
@@ -59,7 +62,7 @@ stages:
     - template: ../templates/flex-downloadPipelineArtifact.yml
       parameters:
         StepName: 'Download Pipeline Artifact - NuGet'
-        ArtifactName: 'onnxruntime-linux-x64-cuda'
+        ArtifactName: 'onnxruntime-linux-x64-${{parameters.StageToken}}'
         TargetPath: '$(Build.BinariesDirectory)/nuget-artifact'
         SpecificArtifact: ${{ parameters.SpecificArtifact }}
         BuildId: ${{ parameters.BuildId }}
@@ -67,7 +70,7 @@ stages:
     - template: ../templates/flex-downloadPipelineArtifact.yml
       parameters:
         StepName: 'Download Pipeline Artifact - NuGet'
-        ArtifactName: 'onnxruntime-linux-x64-tensorrt'
+        ArtifactName: 'onnxruntime-linux-x64-tensorrt-${{parameters.StageToken}}'
         TargetPath: '$(Build.BinariesDirectory)/nuget-artifact'
         SpecificArtifact: ${{ parameters.SpecificArtifact }}
         BuildId: ${{ parameters.BuildId }}
@@ -212,8 +215,16 @@ stages:
         msbuildArguments: '-t:Clean -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu'
         workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
+    # Backwards compat shim for currently publishing pipelines:
+    - ${{if eq(parameters.StageToken, 'cuda_12')}}:
+      - task: 1ES.PublishPipelineArtifact@1
+        displayName: 'Publish Pipeline NuGet Artifact'
+        inputs:
+          artifactName: 'drop-signed-nuget-GPU'
+          targetPath: '$(Build.ArtifactStagingDirectory)'
+
     - task: 1ES.PublishPipelineArtifact@1
       displayName: 'Publish Pipeline NuGet Artifact'
       inputs:
-        artifactName: 'drop-signed-nuget-GPU'
+        artifactName: 'drop-signed-nuget-GPU-${{parameters.StageToken}}'
         targetPath: '$(Build.ArtifactStagingDirectory)'
diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml
index 486efa85877cb..d49a0c3330bbd 100644
--- a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml
@@ -1,14 +1,15 @@
 parameters:
 - name: CudaVersion
-  type: string
-  default: '12.8'
+  type: object
 - name: buildJava
   type: boolean
 - name: buildNodejs
   type: boolean
+- name: StageToken
+  type: string
 
 stages:
-- stage: Linux_C_API_Packaging_GPU
+- stage: Linux_C_API_Packaging_GPU_${{parameters.StageToken}}
   dependsOn: []
   jobs:
   - job: Linux_C_API_Packaging_CUDA
@@ -20,12 +21,9 @@ stages:
       os: linux
     variables:
     - name: CUDA_VERSION_MAJOR
-      ${{ if eq(parameters.CudaVersion, '13.0') }}:
-        value: '13'
-      ${{ if eq(parameters.CudaVersion, '12.8') }}:
-        value: '12'
+      value: ${{ parameters.CudaVersion.major }}
     - name: CUDA_VERSION
-      value: ${{ parameters.CudaVersion }}
+      value: "${{ parameters.CudaVersion.major }}.${{ parameters.CudaVersion.minor }}"
     steps:
     - template: ../templates/set-version-number-variables-step.yml
     - task: UsePythonVersion@0
@@ -48,20 +46,20 @@ stages:
         parameters:
           arch: 'linux-x64'
           buildConfig: 'Release'
-          artifactName: 'onnxruntime-java-linux-x64-cuda'
+          artifactName: 'onnxruntime-java-linux-x64-${{parameters.StageToken}}'
           libraryName: 'libonnxruntime.so'
           nativeLibraryName: 'libonnxruntime4j_jni.so'
 
     - template: ../templates/c-api-artifacts-package-and-publish-steps-posix.yml
       parameters:
         buildConfig: 'Release'
-        artifactName: 'onnxruntime-linux-x64-cuda-$(OnnxRuntimeVersion)'
-        artifactNameNoVersionString: 'onnxruntime-linux-x64-cuda'
+        artifactName: 'onnxruntime-linux-x64-${{parameters.StageToken}}-$(OnnxRuntimeVersion)'
+        artifactNameNoVersionString: 'onnxruntime-linux-x64-${{parameters.StageToken}}'
         libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
 
     - template: ../templates/clean-agent-build-directory-step.yml
   # Linux CUDA with TensorRT Packaging
-  - job: Linux_C_API_Packaging_TensorRT
+  - job: Linux_C_API_Packaging_TensorRT_${{parameters.StageToken}}
     dependsOn: []
     workspace:
       clean: all
@@ -70,19 +68,12 @@ stages:
       name: 'onnxruntime-Ubuntu2404-AMD-CPU'
       os: linux
     variables:
-    - template: ../templates/common-variables.yml
     - name: CUDA_VERSION_MAJOR
-      ${{ if eq(parameters.CudaVersion, '13.0') }}:
-        value: '13'
-      ${{ if eq(parameters.CudaVersion, '12.8') }}:
-        value: '12'
+      value: ${{ parameters.CudaVersion.major }}
     - name: CUDA_VERSION
-      value: ${{ parameters.CudaVersion }}
+      value: "${{ parameters.CudaVersion.major }}.${{ parameters.CudaVersion.minor }}"
     - name: linux_trt_version
-      ${{ if eq(parameters.CudaVersion, '13.0') }}:
-        value: ${{ variables.linux_trt_version_cuda13 }}
-      ${{ if eq(parameters.CudaVersion, '12.8') }}:
-        value: ${{ variables.linux_trt_version_cuda12 }}
+      value: ${{ parameters.CudaVersion.linux_trt_version }}
     steps:
     - checkout: self
       clean: true
@@ -111,22 +102,22 @@ stages:
         parameters:
           arch: 'linux-x64'
           buildConfig: 'Release'
-          artifactName: 'onnxruntime-java-linux-x64-tensorrt'
+          artifactName: 'onnxruntime-java-linux-x64-tensorrt-${{parameters.StageToken}}'
           libraryName: 'libonnxruntime.so'
           nativeLibraryName: 'libonnxruntime4j_jni.so'
 
     - template: ../templates/c-api-artifacts-package-and-publish-steps-posix.yml
       parameters:
         buildConfig: 'Release'
-        artifactName: 'onnxruntime-linux-x64-tensorrt-$(OnnxRuntimeVersion)'
-        artifactNameNoVersionString: 'onnxruntime-linux-x64-tensorrt'
+        artifactName: 'onnxruntime-linux-x64-tensorrt-${{parameters.StageToken}}-$(OnnxRuntimeVersion)'
+        artifactNameNoVersionString: 'onnxruntime-linux-x64-tensorrt-${{parameters.StageToken}}'
         libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
 
     - template: ../templates/clean-agent-build-directory-step.yml
   # Linux CUDA Combined Testing and Publishing
-- stage: Linux_Packaging_combined_CUDA
+- stage: Linux_Packaging_combined_CUDA_${{parameters.StageToken}}
   dependsOn:
-  - Linux_C_API_Packaging_GPU
+  - Linux_C_API_Packaging_GPU_${{parameters.StageToken}}
   jobs:
   - job: Linux_Packaging_combined_CUDA
     condition: succeeded()
@@ -136,24 +127,17 @@ stages:
       name: 'Onnxruntime-Linux-GPU-A10'
       os: linux
     variables:
-    - template: ../templates/common-variables.yml
     - name: CUDA_VERSION_MAJOR
-      ${{ if eq(parameters.CudaVersion, '13.0') }}:
-        value: '13'
-      ${{ if eq(parameters.CudaVersion, '12.8') }}:
-        value: '12'
+      value: ${{ parameters.CudaVersion.major }}
     - name: linux_trt_version
-      ${{ if eq(parameters.CudaVersion, '13.0') }}:
-        value: ${{ variables.linux_trt_version_cuda13 }}
-      ${{ if eq(parameters.CudaVersion, '12.8') }}:
-        value: ${{ variables.linux_trt_version_cuda12 }}
+      value: ${{ parameters.CudaVersion.linux_trt_version }}
+
     steps:
     - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
       submodules: false
     - checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
       submodules: false
 
-
     - template: ../templates/get-docker-image-steps.yml
       parameters:
         ScriptName: $(Build.SourcesDirectory)/onnxruntime/tools/ci_build/get_docker_image.py
@@ -169,13 +153,13 @@ stages:
     - task: DownloadPipelineArtifact@2
       displayName: 'Download Pipeline Artifact - Combined GPU'
       inputs:
-        artifactName: 'onnxruntime-linux-x64-cuda'
+        artifactName: 'onnxruntime-linux-x64-${{parameters.StageToken}}'
         targetPath: '$(Build.BinariesDirectory)/tgz-artifacts'
 
     - task: DownloadPipelineArtifact@2
       displayName: 'Download Pipeline Artifact - Combined GPU'
       inputs:
-        artifactName: 'onnxruntime-linux-x64-tensorrt'
+        artifactName: 'onnxruntime-linux-x64-tensorrt-${{parameters.StageToken}}'
         targetPath: '$(Build.BinariesDirectory)/tgz-artifacts'
 
     - task: ShellScript@2
@@ -218,4 +202,4 @@ stages:
     - task: 1ES.PublishPipelineArtifact@1
       inputs:
         targetPath: '$(Build.ArtifactStagingDirectory)'
-        artifactName: 'onnxruntime-linux-x64-gpu'
+        artifactName: 'onnxruntime-linux-x64-gpu-${{parameters.StageToken}}'
diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-win-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-win-cuda-packaging-stage.yml
index 9aaf055001bbd..ec8a0cf3c0a91 100644
--- a/tools/ci_build/github/azure-pipelines/stages/nuget-win-cuda-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/nuget-win-cuda-packaging-stage.yml
@@ -11,18 +11,11 @@ parameters:
   type: boolean
   default: true
 
-- name: CudaVersion
+- name: StageToken
   type: string
-  default: '13.0'
-  values:
-   - 12.8
-   - 13.0
 
-- name: win_cuda_home
-  type: string
-
-- name: win_trt_home
-  type: string
+- name: CudaVersion
+  type: object
 
 - name: SpecificArtifact
   displayName: Use Specific Artifact
@@ -50,12 +43,6 @@ parameters:
   displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package.
   type: number
 
-- name: CudaArchs
-  type: string
-- name: win_cudnn_home
-  type: string
-  default: ''
-
 stages:
 # Windows CUDA without TensorRT Packaging
 - template: ../templates/win-ci.yml
@@ -64,15 +51,15 @@ stages:
     PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
     ort_build_pool_name: 'onnxruntime-Win2022-GPU-A10'
     DoEsrp: ${{ parameters.DoEsrp }}
-    stage_name_suffix: CUDA
+    stage_name_suffix: CUDA_${{parameters.StageToken}}
     buildArch: x64
     msbuildPlatform: x64
-    packageName: x64-cuda
-    CudaVersion: ${{ parameters.CudaVersion }}
-    ${{ if ne(parameters.win_cudnn_home, '') }}:
-      buildparameter: --use_cuda --cuda_home=${{ parameters.win_cuda_home }} --cudnn_home=${{ parameters.win_cudnn_home }} --enable_onnx_tests --enable_wcos --parallel 4 --nvcc_threads 1 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=${{ parameters.CudaArchs }}"
+    packageName: x64-${{parameters.StageToken}}
+    CudaVersion: "${{ parameters.CudaVersion.major }}.${{ parameters.CudaVersion.minor }}"
+    ${{ if ne(parameters.CudaVersion.win_cudnn_home, '') }}:
+      buildparameter: --use_cuda --cuda_home=${{ parameters.CudaVersion.win_cuda_home }} --cudnn_home=${{ parameters.CudaVersion.win_cudnn_home }} --enable_onnx_tests --enable_wcos --parallel 4 --nvcc_threads 1 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=${{ parameters.CudaVersion.archs }}"
     ${{ else }}:
-      buildparameter: --use_cuda --cuda_home=${{ parameters.win_cuda_home }} --enable_onnx_tests --enable_wcos --parallel 4 --nvcc_threads 1 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=${{ parameters.CudaArchs }}"
+      buildparameter: --use_cuda --cuda_home=${{ parameters.CudaVersion.win_cuda_home }} --enable_onnx_tests --enable_wcos --parallel 4 --nvcc_threads 1 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=${{ parameters.CudaVersion.archs }}"
     runTests: ${{ parameters.RunOnnxRuntimeTests }}
     buildJava: ${{ parameters.buildJava }}
     java_artifact_id: onnxruntime_gpu
@@ -87,15 +74,15 @@ stages:
     PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
     ort_build_pool_name: 'onnxruntime-Win2022-GPU-A10'
     DoEsrp: ${{ parameters.DoEsrp }}
-    stage_name_suffix: TensorRT
+    stage_name_suffix: TensorRT_${{parameters.StageToken}}
     buildArch: x64
     msbuildPlatform: x64
-    CudaVersion: ${{ parameters.CudaVersion }}
-    packageName: x64-tensorrt
-    ${{ if ne(parameters.win_cudnn_home, '') }}:
-      buildparameter: --use_tensorrt --tensorrt_home=${{ parameters.win_trt_home }} --cuda_home=${{ parameters.win_cuda_home }} --cudnn_home=${{ parameters.win_cudnn_home }} --enable_onnx_tests --enable_wcos --parallel 4 --nvcc_threads 1 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=${{ parameters.CudaArchs }}"
+    CudaVersion: "${{ parameters.CudaVersion.major }}.${{ parameters.CudaVersion.minor }}"
+    packageName: x64-tensorrt-${{parameters.StageToken}}
+    ${{ if ne(parameters.CudaVersion.win_cudnn_home, '') }}:
+      buildparameter: --use_tensorrt --tensorrt_home=${{ parameters.CudaVersion.win_trt_home }} --cuda_home=${{ parameters.CudaVersion.win_cuda_home }} --cudnn_home=${{ parameters.CudaVersion.win_cudnn_home }} --enable_onnx_tests --enable_wcos --parallel 4 --nvcc_threads 1 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=${{ parameters.CudaVersion.archs }}"
     ${{ else }}:
-      buildparameter: --use_tensorrt --tensorrt_home=${{ parameters.win_trt_home }} --cuda_home=${{ parameters.win_cuda_home }} --enable_onnx_tests --enable_wcos --parallel 4 --nvcc_threads 1 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=${{ parameters.CudaArchs }}"
+      buildparameter: --use_tensorrt --tensorrt_home=${{ parameters.CudaVersion.win_trt_home }} --cuda_home=${{ parameters.CudaVersion.win_cuda_home }} --enable_onnx_tests --enable_wcos --parallel 4 --nvcc_threads 1 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=${{ parameters.CudaVersion.archs }}"
     runTests: ${{ parameters.RunOnnxRuntimeTests }}
     buildJava: ${{ parameters.buildJava }}
     java_artifact_id: onnxruntime_gpu
@@ -104,10 +91,10 @@ stages:
     BuildId: ${{ parameters.BuildId }}
 
 # Windows CUDA Combined Testing and Publishing
-- stage: Windows_Packaging_combined_GPU
+- stage: Windows_Packaging_combined_GPU_${{parameters.StageToken}}
   dependsOn:
-  - Windows_Packaging_CUDA
-  - Windows_Packaging_TensorRT
+  - Windows_Packaging_CUDA_${{parameters.StageToken}}
+  - Windows_Packaging_TensorRT_${{parameters.StageToken}}
   condition: succeeded()
 
   jobs:
@@ -117,6 +104,7 @@ stages:
     pool:
       name: 'onnxruntime-Win2022-GPU-A10'
     variables:
+      CUDA_VERSION_MAJOR: ${{ parameters.CudaVersion.major }}
       CUDA_MODULE_LOADINGL: 'LAZY'
       GRADLE_OPTS: '-Dorg.gradle.daemon=false'
     steps:
@@ -129,22 +117,22 @@ stages:
       parameters:
         DownloadCUDA: true
         DownloadTRT: true
-        CudaVersion: ${{ parameters.CudaVersion }}
+        CudaVersion: "${{ parameters.CudaVersion.major }}.${{ parameters.CudaVersion.minor }}"
 
     - template: ../templates/set-version-number-variables-step.yml
       parameters:
         versionFileDirectory: '$(Build.SourcesDirectory)\onnxruntime'
         workingDirectory: '$(Build.SourcesDirectory)\onnxruntime'
     - task: DownloadPipelineArtifact@2
-      displayName: 'Download Pipeline Artifact - onnxruntime-win-x64-cuda'
+      displayName: 'Download Pipeline Artifact - onnxruntime-win-x64-${{parameters.StageToken}}'
       inputs:
-        artifactName: 'onnxruntime-win-x64-cuda'
+        artifactName: 'onnxruntime-win-x64-${{parameters.StageToken}}'
         targetPath: '$(Build.BinariesDirectory)/zip-artifacts'
 
     - task: DownloadPipelineArtifact@2
-      displayName: 'Download Pipeline Artifact - onnxruntime-win-x64-tensorrt'
+      displayName: 'Download Pipeline Artifact - onnxruntime-win-x64-tensorrt-${{parameters.StageToken}}'
       inputs:
-        artifactName: 'onnxruntime-win-x64-tensorrt'
+        artifactName: 'onnxruntime-win-x64-tensorrt-${{parameters.StageToken}}'
         targetPath: '$(Build.BinariesDirectory)/zip-artifacts'
 
     - task: PowerShell@2
@@ -183,6 +171,7 @@ stages:
 
     - task: BatchScript@1
       displayName: 'Test C API application for GPU package'
+      condition: and(succeeded(), ne(variables['CUDA_VERSION_MAJOR'], '13'))
       inputs:
         filename: $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet\run_capi_application.bat
         arguments: $(Build.SourcesDirectory)\onnxruntime $(Build.ArtifactStagingDirectory)\onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet
@@ -191,5 +180,5 @@ stages:
     - task: 1ES.PublishPipelineArtifact@1
       displayName: 'Publish Pipeline Combined GPU Package Artifact'
       inputs:
-        artifactName: 'onnxruntime-win-x64-gpu'
+        artifactName: 'onnxruntime-win-x64-gpu-${{parameters.StageToken}}'
         targetPath: '$(Build.ArtifactStagingDirectory)'
diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
index 5025046a02b0e..2177b5fbf8e1b 100644
--- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
@@ -53,6 +53,7 @@ parameters:
   type: boolean
   default: false
 
+
 stages:
 - template: linux-cpu-packaging-pipeline.yml
   parameters:
@@ -378,12 +379,12 @@ stages:
         DoEsrp: true
 
 
-- stage: Nodejs_Packaging
+- stage: Nodejs_Packaging_Cuda_12
   dependsOn:
   - Setup
   - Windows_Nodejs_Packaging_x64
   - Windows_Nodejs_Packaging_arm64
-  - Linux_Nodejs_Packaging_x64
+  - Linux_Nodejs_Packaging_x64_cuda_12
   - Linux_C_API_Packaging_CPU
   - MacOS_C_API_Packaging_CPU
   condition: succeeded()
@@ -507,7 +508,7 @@ stages:
     - task: DownloadPipelineArtifact@0
       displayName: 'Download Pipeline Artifact - Nodejs (Linux x64)'
       inputs:
-        artifactName: 'drop-onnxruntime-nodejs-linux-x64'
+        artifactName: 'drop-onnxruntime-nodejs-linux-x64-cuda_12'
         targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/linux/x64/'
 
     - task: DownloadPipelineArtifact@0
diff --git a/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh b/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh
index 04ac0e35a6d78..2b0c724baf2ed 100755
--- a/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh
+++ b/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh
@@ -12,7 +12,7 @@ EXIT_CODE=1
 
 uname -a
 
-cd $ARTIFACT_DIR 
+cd $ARTIFACT_DIR
 
 mkdir -p $ARTIFACT_DIR/onnxruntime-linux-x64-tensorrt
 tar zxvf $ARTIFACT_DIR/onnxruntime-linux-x64-tensorrt-*.tgz -C onnxruntime-linux-x64-tensorrt
@@ -20,10 +20,10 @@ rm $ARTIFACT_DIR/onnxruntime-linux-x64-tensorrt-*.tgz
 
 # Rename cuda directory to gpu directory
 mkdir -p $ARTIFACT_DIR/onnxruntime-linux-x64-gpu
-tar zxvf $ARTIFACT_DIR/onnxruntime-linux-x64-cuda-*.tgz -C onnxruntime-linux-x64-gpu
-VERSION=`ls $ARTIFACT_DIR/onnxruntime-linux-x64-gpu | sed 's/onnxruntime-linux-x64-cuda-//'`
+tar zxvf $ARTIFACT_DIR/onnxruntime-linux-x64-cuda_*.tgz -C onnxruntime-linux-x64-gpu
+VERSION=`ls $ARTIFACT_DIR/onnxruntime-linux-x64-gpu | sed 's/onnxruntime-linux-x64-cuda_[0-9]*-//'`
 mv $ARTIFACT_DIR/onnxruntime-linux-x64-gpu/* $ARTIFACT_DIR/onnxruntime-linux-x64-gpu/onnxruntime-linux-x64-gpu-$VERSION
-rm $ARTIFACT_DIR/onnxruntime-linux-x64-cuda-*.tgz
+rm $ARTIFACT_DIR/onnxruntime-linux-x64-cuda_*.tgz
 
 cp onnxruntime-linux-x64-tensorrt/*/lib/libonnxruntime.so* onnxruntime-linux-x64-gpu/*/lib
 cp onnxruntime-linux-x64-tensorrt/*/lib/libonnxruntime_providers_tensorrt.so onnxruntime-linux-x64-gpu/*/lib
diff --git a/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 b/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1
index 29946dcb73f8a..371347e816af1 100644
--- a/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1
+++ b/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1
@@ -48,7 +48,7 @@ Foreach-Object {
 New-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo" -ItemType directory -ErrorAction SilentlyContinue
 
 # Copy CUDA libraries.
-Copy-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-cuda-*\lib\*" -Destination "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo"
+Copy-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-cuda_*\lib\*" -Destination "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo"
 
 # Install protoc via dotnet.
 $protocInstallDir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build"
@@ -68,13 +68,13 @@ else {
     Write-Error "Could not find protoc.exe in $protocInstallDir"
 }
 
-# Rename onnxruntime directories to a generic format.
+# Rename onnxruntime directories to a generic format. Name is of format <dir>-<cuda_version>-<ort_version>
 $ort_dirs = Get-ChildItem -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-*" -Directory
 foreach ($ort_dir in $ort_dirs) {
     $dirname = Split-Path -Path $ort_dir -Leaf
-    $lastHyphenIndex = $dirname.LastIndexOf('-')
-    if ($lastHyphenIndex -gt -1) {
-        $newName = $dirname.Substring(0, $lastHyphenIndex)
+    Write-Host "Processing directory: $dirname"
+    if( $dirname -match '^(.+)-cuda_\d+-[\d.]+$' ) {
+        $newName = $matches[1]
         $newPath = Join-Path -Path $ort_dir.Parent.FullName -ChildPath $newName
         Write-Output "Renaming '$($ort_dir.FullName)' to '$newPath'"
         Rename-Item -Path $ort_dir.FullName -NewName $newName
diff --git a/tools/ci_build/github/windows/extract_zip_files_gpu.ps1 b/tools/ci_build/github/windows/extract_zip_files_gpu.ps1
index 6671fecfbe072..fcec5bb4e87e2 100644
--- a/tools/ci_build/github/windows/extract_zip_files_gpu.ps1
+++ b/tools/ci_build/github/windows/extract_zip_files_gpu.ps1
@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # extract *-cuda-*.zip and *-tensorrt-*.zip
-Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\zip-artifacts -Filter *.zip | 
+Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\zip-artifacts -Filter *.zip |
 Foreach-Object {
  $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\zip-artifacts"
  Write-Output $cmd
@@ -10,14 +10,15 @@ Foreach-Object {
 }
 
 # Rename tensorrt directory for later use in bundle_dlls_gpu.bat
-Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\zip-artifacts | Where-Object { $_.Name -match 'onnxruntime-win-x64-tensorrt-\d{1,}\.\d{1,}\.\d{1,}$' } | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\zip-artifacts\onnxruntime-win-x64-tensorrt
+Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\zip-artifacts | Where-Object { $_.Name -match 'onnxruntime-win-x64-tensorrt-cuda_\d{2}-\d{1,}\.\d{1,}\.\d{1,}$' } | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\zip-artifacts\onnxruntime-win-x64-tensorrt
+
 Remove-Item $Env:BUILD_BINARIESDIRECTORY\zip-artifacts\*.zip
 
 # Rename cuda directory to gpu directory and re-compress it for later use in bundle_dlls_gpu.bat
 Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\zip-artifacts -Filter *cuda* |
 Foreach-Object {
- $($_.FullName) -match '.*onnxruntime-win-x64-cuda-(.*)'
- $version=$matches[1]
+ $($_.FullName) -match '.*onnxruntime-win-x64-cuda_(.*)-(.*)'
+ $version=$matches[2]
  Rename-Item -Path $($_.FullName) -NewName onnxruntime-win-x64-gpu-$version
  $cmd = "7z.exe a $Env:BUILD_BINARIESDIRECTORY\zip-artifacts\onnxruntime-win-x64-gpu-$version.zip $Env:BUILD_BINARIESDIRECTORY\zip-artifacts\onnxruntime-win-x64-gpu-$version"
  Write-Output $cmd