Migrate stage artifact deployment from Cloud SDK Pipeline-Lib (#1324)

Co-authored-by: Daniel Kurzynski <[email protected]>
Co-authored-by: Florian Geckeler <[email protected]>

Author: 3 people

cmd/nexusUpload.go

@@ -135,18 +135,12 @@ func uploadMTA(utils nexusUploadUtils, uploader nexus.Uploader, options *nexusUp
 		// This will fail anyway if the file doesn't exist
 		mtaPath = "mta.yml"
 	}
-	version, err := getVersionFromMtaFile(utils, mtaPath)
-	var artifactID = options.ArtifactID
-	if artifactID == "" {
-		artifactID = utils.getEnvParameter(".pipeline/commonPipelineEnvironment/configuration", "artifactId")
-		if artifactID == "" {
-			err = fmt.Errorf("the 'artifactId' parameter was not provided and could not be retrieved from the Common Pipeline Environment")
-		} else {
-			log.Entry().Debugf("mtar artifact id from CPE: '%s'", artifactID)
-		}
-	}
+	mtaInfo, err := getInfoFromMtaFile(utils, mtaPath)
 	if err == nil {
-		err = uploader.SetInfo(options.GroupID, artifactID, version)
+		if options.ArtifactID != "" {
+			mtaInfo.ID = options.ArtifactID
+		}
+		err = uploader.SetInfo(options.GroupID, mtaInfo.ID, mtaInfo.Version)
 		if err == nexus.ErrEmptyVersion {
 			err = fmt.Errorf("the project descriptor file 'mta.yaml' has an invalid version: %w", err)
 		}
@@ -170,25 +164,25 @@ type mtaYaml struct {
 	Version string `json:"version"`
 }
 
-func getVersionFromMtaFile(utils nexusUploadUtils, filePath string) (string, error) {
+func getInfoFromMtaFile(utils nexusUploadUtils, filePath string) (*mtaYaml, error) {
 	mtaYamlContent, err := utils.fileRead(filePath)
 	if err != nil {
-		return "", fmt.Errorf("could not read from required project descriptor file '%s'",
+		return nil, fmt.Errorf("could not read from required project descriptor file '%s'",
 			filePath)
 	}
-	return getVersionFromMtaYaml(mtaYamlContent, filePath)
+	return getInfoFromMtaYaml(mtaYamlContent, filePath)
 }
 
-func getVersionFromMtaYaml(mtaYamlContent []byte, filePath string) (string, error) {
+func getInfoFromMtaYaml(mtaYamlContent []byte, filePath string) (*mtaYaml, error) {
 	var mtaYaml mtaYaml
 	err := yaml.Unmarshal(mtaYamlContent, &mtaYaml)
 	if err != nil {
 		// Eat the original error as it is unhelpful and confusingly mentions JSON, while the
 		// user thinks it should parse YAML (it is transposed by the implementation).
-		return "", fmt.Errorf("failed to parse contents of the project descriptor file '%s'",
+		return nil, fmt.Errorf("failed to parse contents of the project descriptor file '%s'",
 			filePath)
 	}
-	return mtaYaml.Version, nil
+	return &mtaYaml, nil
 }
 
 func createMavenExecuteOptions(options *nexusUploadOptions) maven.ExecuteOptions {

cmd/nexusUpload_generated.go

@@ -166,18 +166,20 @@ func TestUploadMTAProjects(t *testing.T) {
 		assert.Equal(t, 0, len(uploader.GetArtifacts()))
 		assert.Equal(t, 0, len(uploader.uploadedArtifacts))
 	})
-	t.Run("Uploading MTA project without artifactId parameter fails", func(t *testing.T) {
+	t.Run("Uploading MTA project without artifactId parameter works", func(t *testing.T) {
 		utils := newMockUtilsBundle(true, false)
 		utils.files["mta.yaml"] = testMtaYml
+		utils.files["test.mtar"] = []byte("contentsOfMtar")
 		utils.cpe[".pipeline/commonPipelineEnvironment/mtarFilePath"] = "test.mtar"
 		uploader := mockUploader{}
 		options := createOptions()
 		options.ArtifactID = ""
 
 		err := runNexusUpload(&utils, &uploader, &options)
-		assert.EqualError(t, err, "the 'artifactId' parameter was not provided and could not be retrieved from the Common Pipeline Environment")
-		assert.Equal(t, 0, len(uploader.GetArtifacts()))
-		assert.Equal(t, 0, len(uploader.uploadedArtifacts))
+		if assert.NoError(t, err) {
+			assert.Equal(t, 2, len(uploader.uploadedArtifacts))
+			assert.Equal(t, "test", uploader.GetArtifactsID())
+		}
 	})
 	t.Run("Uploading MTA project fails due to missing yaml file", func(t *testing.T) {
 		utils := newMockUtilsBundle(true, false)
@@ -274,30 +276,6 @@ func TestUploadMTAProjects(t *testing.T) {
 		assert.Equal(t, "0.3.0", uploader.GetArtifactsVersion())
 		assert.Equal(t, "artifact.id", uploader.GetArtifactsID())
 
-		artifacts := uploader.uploadedArtifacts
-		if assert.Equal(t, 2, len(artifacts)) {
-			assert.Equal(t, "mta.yml", artifacts[0].File)
-			assert.Equal(t, "yaml", artifacts[0].Type)
-
-			assert.Equal(t, "test.mtar", artifacts[1].File)
-			assert.Equal(t, "mtar", artifacts[1].Type)
-		}
-	})
-	t.Run("Test uploading mta.yml project works with artifactID from CPE", func(t *testing.T) {
-		utils := newMockUtilsBundle(true, false)
-		utils.files["mta.yml"] = testMtaYml
-		utils.files["test.mtar"] = []byte("contentsOfMtar")
-		utils.cpe[".pipeline/commonPipelineEnvironment/mtarFilePath"] = "test.mtar"
-		utils.cpe[".pipeline/commonPipelineEnvironment/configuration/artifactId"] = "my-artifact-id"
-		uploader := mockUploader{}
-		options := createOptions()
-		// Clear artifact ID to trigger reading it from the CPE
-		options.ArtifactID = ""
-
-		err := runNexusUpload(&utils, &uploader, &options)
-		assert.NoError(t, err, "expected mta.yml project upload to work")
-		assert.Equal(t, "my-artifact-id", uploader.GetArtifactsID())
-
 		artifacts := uploader.uploadedArtifacts
 		if assert.Equal(t, 2, len(artifacts)) {
 			assert.Equal(t, "mta.yml", artifacts[0].File)

cmd/nexusUpload_test.go

@@ -48,6 +48,36 @@ func (nexusUpload *Upload) SetRepoURL(nexusURL, nexusVersion, repository string)
 	return nil
 }
 
+func getBaseURL(nexusURL, nexusVersion, repository string) (string, error) {
+	if nexusURL == "" {
+		return "", errors.New("nexusURL must not be empty")
+	}
+	nexusURL = strings.ToLower(nexusURL)
+	var protocols = []string{"http://", "https://"}
+	for _, protocol := range protocols {
+		if strings.HasPrefix(nexusURL, protocol) {
+			nexusURL = strings.TrimPrefix(nexusURL, protocol)
+			break
+		}
+	}
+	if repository == "" {
+		return "", errors.New("repository must not be empty")
+	}
+	baseURL := nexusURL
+	switch nexusVersion {
+	case "nexus2":
+		baseURL += "/content/repositories/"
+	case "nexus3":
+		baseURL += "/repository/"
+	default:
+		return "", fmt.Errorf("unsupported Nexus version '%s', must be 'nexus2' or 'nexus3'", nexusVersion)
+	}
+	baseURL += repository + "/"
+	// Replace any double slashes, as nexus does not like them
+	baseURL = strings.ReplaceAll(baseURL, "//", "/")
+	return baseURL, nil
+}
+
 // GetRepoURL returns the base URL for the nexus repository.
 func (nexusUpload *Upload) GetRepoURL() string {
 	return nexusUpload.repoURL
@@ -144,29 +174,3 @@ func (nexusUpload *Upload) GetArtifacts() []ArtifactDescription {
 func (nexusUpload *Upload) Clear() {
 	nexusUpload.artifacts = []ArtifactDescription{}
 }
-
-func getBaseURL(nexusURL, nexusVersion, repository string) (string, error) {
-	if nexusURL == "" {
-		return "", errors.New("nexusURL must not be empty")
-	}
-	nexusURL = strings.ToLower(nexusURL)
-	if strings.HasPrefix(nexusURL, "http://") || strings.HasPrefix(nexusURL, "https://") {
-		return "", errors.New("nexusURL must not start with 'http://' or 'https://'")
-	}
-	if repository == "" {
-		return "", errors.New("repository must not be empty")
-	}
-	baseURL := nexusURL
-	switch nexusVersion {
-	case "nexus2":
-		baseURL += "/content/repositories/"
-	case "nexus3":
-		baseURL += "/repository/"
-	default:
-		return "", fmt.Errorf("unsupported Nexus version '%s', must be 'nexus2' or 'nexus3'", nexusVersion)
-	}
-	baseURL += repository + "/"
-	// Replace any double slashes, as nexus does not like them
-	baseURL = strings.ReplaceAll(baseURL, "//", "/")
-	return baseURL, nil
-}

pkg/nexus/nexus.go

@@ -110,12 +110,16 @@ func TestSetBaseURL(t *testing.T) {
 	t.Run("Test host wrongly includes protocol http://", func(t *testing.T) {
 		nexusUpload := Upload{}
 		err := nexusUpload.SetRepoURL("htTp://localhost:8081", "nexus3", "maven-releases")
-		assert.Error(t, err, "Expected SetRepoURL() to fail (invalid host)")
+		if assert.NoError(t, err, "Expected SetRepoURL() to work") {
+			assert.Equal(t, "localhost:8081/repository/maven-releases/", nexusUpload.repoURL)
+		}
 	})
 	t.Run("Test host wrongly includes protocol https://", func(t *testing.T) {
 		nexusUpload := Upload{}
 		err := nexusUpload.SetRepoURL("htTpS://localhost:8081", "nexus3", "maven-releases")
-		assert.Error(t, err, "Expected SetRepoURL() to fail (invalid host)")
+		if assert.NoError(t, err, "Expected SetRepoURL() to work") {
+			assert.Equal(t, "localhost:8081/repository/maven-releases/", nexusUpload.repoURL)
+		}
 	})
 	t.Run("Test invalid version provided", func(t *testing.T) {
 		nexusUpload := Upload{}

pkg/nexus/nexus_test.go

@@ -12,6 +12,8 @@ spec:
       - name: nexusCredentialsId
         description: The technical username/password credential for accessing the nexus endpoint.
         type: jenkins
+        aliases:
+          - name: nexus/credentialsId
     params:
       - name: version
         type: string
@@ -22,6 +24,8 @@ spec:
         - STEPS
         mandatory: false
         default: nexus3
+        aliases:
+          - name: nexus/version
       - name: url
         type: string
         description: URL of the nexus. The scheme part of the URL will not be considered, because only http is supported.
@@ -30,6 +34,8 @@ spec:
         - STAGES
         - STEPS
         mandatory: true
+        aliases:
+          - name: nexus/url
       - name: repository
         type: string
         description: Name of the nexus repository.
@@ -38,22 +44,22 @@ spec:
         - STAGES
         - STEPS
         mandatory: true
-        default:
+        aliases:
+          - name: nexus/repository
       - name: groupId
         type: string
         description: Group ID of the artifacts. Only used in MTA projects, ignored for Maven.
         scope:
         - PARAMETERS
         - STAGES
         - STEPS
-        mandatory: false
+        aliases:
+          - name: nexus/groupId
       - name: artifactId
         type: string
         description: The artifact ID used for both the .mtar and mta.yaml files deployed for MTA projects, ignored for Maven.
         scope:
         - PARAMETERS
-        - STAGES
-        - STEPS
       - name: globalSettingsFile
         type: string
         description: Path to the mvn settings file that should be used as global settings file.
@@ -81,20 +87,18 @@ spec:
         - PARAMETERS
         - STAGES
         - STEPS
+        aliases:
+          - name: nexus/additionalClassifiers
       - name: user
         type: string
         description: User
         scope:
         - PARAMETERS
-        - STAGES
-        - STEPS
       - name: password
         type: string
         description: Password
         scope:
         - PARAMETERS
-        - STAGES
-        - STEPS
   containers:
     - name: mvn
       image: maven:3.6-jdk-8

resources/metadata/nexusUpload.yaml

@@ -133,6 +133,7 @@ public class CommonStepsTest extends BasePiperTest{
         'nexusUpload', //implementing new golang pattern without fields
         'mavenBuild', //implementing new golang pattern without fields
         'mavenExecuteStaticCodeChecks', //implementing new golang pattern without fields
+        'piperPipelineStageArtifactDeployment', //stage without step flags
     ]
 
     @Test

test/groovy/CommonStepsTest.groovy

@@ -1,11 +1,22 @@
+import com.sap.piper.DownloadCacheUtils
 import groovy.transform.Field
 
+import static groovy.json.JsonOutput.toJson
+
 @Field String STEP_NAME = getClass().getName()
 @Field String METADATA_FILE = 'metadata/nexusUpload.yaml'
 
 //Metadata maintained in file project://resources/metadata/nexusUpload.yaml
 
 void call(Map parameters = [:]) {
+    // Replace 'additionalClassifiers' List with JSON encoded String.
+    // This is currently necessary, since the go code doesn't support complex/arbitrary parameter types.
+    // TODO: Support complex/structured types of parameters in piper-go
+    if (parameters.additionalClassifiers) {
+        parameters.additionalClassifiers = "${toJson(parameters.additionalClassifiers as List)}"
+    }
+    parameters = DownloadCacheUtils.injectDownloadCacheInMavenParameters(parameters.script, parameters)
+
     List credentials = [[type: 'usernamePassword', id: 'nexusCredentialsId', env: ['PIPER_username', 'PIPER_password']]]
     piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
 }

vars/nexusUpload.groovy

@@ -68,6 +68,7 @@ static String getCustomDefaultConfigsArg() {
 
 static String getCustomConfigArg(def script) {
     if (script?.commonPipelineEnvironment?.configurationFile
+        && script.commonPipelineEnvironment.configurationFile != '.pipeline/config.yml'
         && script.commonPipelineEnvironment.configurationFile != '.pipeline/config.yaml') {
         return " --customConfig ${BashUtils.quoteAndEscape(script.commonPipelineEnvironment.configurationFile)}"
     }