diff --git a/backend/plugins/gitlab/e2e/pipelines_test.go b/backend/plugins/gitlab/e2e/pipelines_test.go index d313ed3b2ad..0f4bb38d626 100644 --- a/backend/plugins/gitlab/e2e/pipelines_test.go +++ b/backend/plugins/gitlab/e2e/pipelines_test.go @@ -23,6 +23,7 @@ import ( "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/impl" "github.com/apache/incubator-devlake/plugins/gitlab/models" "github.com/apache/incubator-devlake/plugins/gitlab/tasks" @@ -33,13 +34,19 @@ func TestGitlabPipelineDataFlow(t *testing.T) { var gitlab impl.Gitlab dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab) + regexEnricher := api.NewRegexEnricher() + _ = regexEnricher.TryAdd(devops.DEPLOYMENT, "EE-7121") + _ = regexEnricher.TryAdd(devops.PRODUCTION, "EE-7121") + taskData := &tasks.GitlabTaskData{ Options: &tasks.GitlabOptions{ ConnectionId: 1, ProjectId: 12345678, GitlabTransformationRule: new(models.GitlabTransformationRule), }, + RegexEnricher: regexEnricher, } + // import raw data table dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_pipeline.csv", "_raw_gitlab_api_pipeline") dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv", &models.GitlabProject{}) @@ -57,11 +64,15 @@ func TestGitlabPipelineDataFlow(t *testing.T) { "gitlab_created_at", "project_id", "status", + "ref", + "sha", "web_url", "duration", "started_at", "finished_at", "coverage", + "type", + "environment", ), ) diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv index 4256089736d..46bb9b3a2f8 100644 --- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv +++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv @@ -1,21 +1,21 @@ -connection_id,gitlab_id,gitlab_created_at,project_id,status,web_url,duration,started_at,finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,457474837,2022-01-27T10:07:09.429+00:00,12345678,running,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152, -1,457474996,2022-01-27T10:07:18.884+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153, -1,457475160,2022-01-27T10:07:26.435+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154, -1,457475337,2022-01-27T10:07:36.502+00:00,12345678,running,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155, -1,485811050,2022-03-07T06:26:42.109+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156, -1,485811059,2022-03-07T06:26:43.784+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157, -1,485813816,2022-03-07T06:33:56.824+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158, -1,485813830,2022-03-07T06:33:58.889+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159, -1,485814501,2022-03-07T06:35:28.111+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160, -1,485814516,2022-03-07T06:35:31.255+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161, -1,485814871,2022-03-07T06:36:50.020+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162, -1,485817670,2022-03-07T06:45:09.471+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163, -1,485837602,2022-03-07T07:20:45.859+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164, -1,485842553,2022-03-07T07:30:47.018+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165, -1,485845850,2022-03-07T07:38:58.611+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166, -1,485852752,2022-03-07T07:46:09.385+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167, -1,485865876,2022-03-07T08:04:56.406+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168, -1,485877118,2022-03-07T08:22:48.943+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169, -1,485905167,2022-03-07T09:02:09.994+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170, -1,485932863,2022-03-07T09:34:57.476+00:00,12345678,success,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171, +connection_id,gitlab_id,gitlab_created_at,project_id,status,ref,sha,web_url,duration,started_at,finished_at,coverage,type,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,457474837,2022-01-27T10:07:09.429+00:00,12345678,running,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152, +1,457474996,2022-01-27T10:07:18.884+00:00,12345678,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153, +1,457475160,2022-01-27T10:07:26.435+00:00,12345678,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154, +1,457475337,2022-01-27T10:07:36.502+00:00,12345678,running,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155, +1,485811050,2022-03-07T06:26:42.109+00:00,12345678,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156, +1,485811059,2022-03-07T06:26:43.784+00:00,12345678,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157, +1,485813816,2022-03-07T06:33:56.824+00:00,12345678,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158, +1,485813830,2022-03-07T06:33:58.889+00:00,12345678,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159, +1,485814501,2022-03-07T06:35:28.111+00:00,12345678,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,DEPLOYMENT,PRODUCTION,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160, +1,485814516,2022-03-07T06:35:31.255+00:00,12345678,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161, +1,485814871,2022-03-07T06:36:50.020+00:00,12345678,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162, +1,485817670,2022-03-07T06:45:09.471+00:00,12345678,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163, +1,485837602,2022-03-07T07:20:45.859+00:00,12345678,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164, +1,485842553,2022-03-07T07:30:47.018+00:00,12345678,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165, +1,485845850,2022-03-07T07:38:58.611+00:00,12345678,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166, +1,485852752,2022-03-07T07:46:09.385+00:00,12345678,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167, +1,485865876,2022-03-07T08:04:56.406+00:00,12345678,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168, +1,485877118,2022-03-07T08:22:48.943+00:00,12345678,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169, +1,485905167,2022-03-07T09:02:09.994+00:00,12345678,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170, +1,485932863,2022-03-07T09:34:57.476+00:00,12345678,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171, diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv index d73e46df023..b09f525ea8f 100644 --- a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv +++ b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv @@ -3,11 +3,11 @@ gitlab:GitlabPipeline:1:457474837,gitlab:GitlabProject:1:12345678,,IN_PROGRESS,, gitlab:GitlabPipeline:1:457474996,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-01-27T10:07:18.884+00:00,2022-01-27T10:07:19.043+00:00,gitlab:GitlabProject:1:12345678 gitlab:GitlabPipeline:1:457475160,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,gitlab:GitlabProject:1:12345678 gitlab:GitlabPipeline:1:457475337,gitlab:GitlabProject:1:12345678,,IN_PROGRESS,,0,,2022-01-27T10:07:36.502+00:00,,gitlab:GitlabProject:1:12345678 -gitlab:GitlabPipeline:1:485811050,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,gitlab:GitlabProject:1:12345678 +gitlab:GitlabPipeline:1:485811050,gitlab:GitlabProject:1:12345678,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,gitlab:GitlabProject:1:12345678 gitlab:GitlabPipeline:1:485811059,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,gitlab:GitlabProject:1:12345678 -gitlab:GitlabPipeline:1:485813816,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,gitlab:GitlabProject:1:12345678 +gitlab:GitlabPipeline:1:485813816,gitlab:GitlabProject:1:12345678,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,gitlab:GitlabProject:1:12345678 gitlab:GitlabPipeline:1:485813830,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:33:58.889+00:00,2022-03-07T06:33:58.889+00:00,gitlab:GitlabProject:1:12345678 -gitlab:GitlabPipeline:1:485814501,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,gitlab:GitlabProject:1:12345678 +gitlab:GitlabPipeline:1:485814501,gitlab:GitlabProject:1:12345678,FAILURE,DONE,DEPLOYMENT,0,PRODUCTION,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,gitlab:GitlabProject:1:12345678 gitlab:GitlabPipeline:1:485814516,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,,2022-03-07T06:35:31.255+00:00,2022-03-07T06:35:31.255+00:00,gitlab:GitlabProject:1:12345678 gitlab:GitlabPipeline:1:485814871,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,42,,2022-03-07T06:36:50.020+00:00,2022-03-07T06:37:32.103+00:00,gitlab:GitlabProject:1:12345678 gitlab:GitlabPipeline:1:485817670,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,1956,,2022-03-07T06:45:09.471+00:00,2022-03-07T07:17:46.305+00:00,gitlab:GitlabProject:1:12345678 diff --git a/backend/plugins/gitlab/gitlab.go b/backend/plugins/gitlab/gitlab.go index a45b0aa27c3..c44faa4da14 100644 --- a/backend/plugins/gitlab/gitlab.go +++ b/backend/plugins/gitlab/gitlab.go @@ -44,23 +44,26 @@ func main() { issueTypeBug := cmd.Flags().String("issueTypeBug", "^(bug|failure|error)$", "issue type bug") issueTypeIncident := cmd.Flags().String("issueTypeIncident", "", "issue type incident") issueTypeRequirement := cmd.Flags().String("issueTypeRequirement", "^(feat|feature|proposal|requirement)$", "issue type requirement") - deployTagPattern := cmd.Flags().String("deployTagPattern", "(?i)deploy", "deploy tag name") - + deploymentPattern := cmd.Flags().String("deploymentPattern", "(?i)deploy", "deploy Pattern for project name") + productionPattern := cmd.Flags().String("productionPattern", "product", "product Pattern for project name") cmd.Run = func(cmd *cobra.Command, args []string) { runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ - "projectId": *projectId, - "connectionId": *connectionId, - "timeAfter": *timeAfter, - "prType": *prType, - "prComponent": *prComponent, - "prBodyClosePattern": *prBodyClosePattern, - "issueSeverity": *issueSeverity, - "issuePriority": *issuePriority, - "issueComponent": *issueComponent, - "issueTypeBug": *issueTypeBug, - "issueTypeIncident": *issueTypeIncident, - "issueTypeRequirement": *issueTypeRequirement, - "deployTagPattern": *deployTagPattern, + "projectId": *projectId, + "connectionId": *connectionId, + "timeAfter": *timeAfter, + "transformationRules": map[string]interface{}{ + "prType": *prType, + "prComponent": *prComponent, + "prBodyClosePattern": *prBodyClosePattern, + "issueSeverity": *issueSeverity, + "issuePriority": *issuePriority, + "issueComponent": *issueComponent, + "issueTypeBug": *issueTypeBug, + "issueTypeIncident": *issueTypeIncident, + "issueTypeRequirement": *issueTypeRequirement, + "deploymentPattern": *deploymentPattern, + "productionPattern": *productionPattern, + }, }) } diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go index 561538ae5bd..fb04f6207a2 100644 --- a/backend/plugins/gitlab/impl/impl.go +++ b/backend/plugins/gitlab/impl/impl.go @@ -24,6 +24,7 @@ import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/api" @@ -209,9 +210,18 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i op.GitlabTransformationRule = &transformationRule } + regexEnricher := helper.NewRegexEnricher() + if err := regexEnricher.TryAdd(devops.DEPLOYMENT, op.DeploymentPattern); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid value for `deploymentPattern`") + } + if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ProductionPattern); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`") + } + taskData := tasks.GitlabTaskData{ - Options: op, - ApiClient: apiClient, + Options: op, + ApiClient: apiClient, + RegexEnricher: regexEnricher, } if !timeAfter.IsZero() { diff --git a/backend/plugins/gitlab/models/migrationscripts/20230419_add_type_env_to_pipeline.go b/backend/plugins/gitlab/models/migrationscripts/20230419_add_type_env_to_pipeline.go new file mode 100644 index 00000000000..fed04a02bd5 --- /dev/null +++ b/backend/plugins/gitlab/models/migrationscripts/20230419_add_type_env_to_pipeline.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addTypeEnvToPipeline struct{} + +type pipeline20230419 struct { + Type string `gorm:"type:varchar(255)"` + Environment string `gorm:"type:varchar(255)"` +} + +func (pipeline20230419) TableName() string { + return "_tool_gitlab_pipelines" +} + +func (u *addTypeEnvToPipeline) Up(baseRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(baseRes, &pipeline20230419{}) +} + +func (*addTypeEnvToPipeline) Version() uint64 { + return 20230322150357 +} + +func (*addTypeEnvToPipeline) Name() string { + return "add type/env to gitlab pipelines" +} diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go b/backend/plugins/gitlab/models/migrationscripts/register.go index 9612cf3f702..53823fae42c 100644 --- a/backend/plugins/gitlab/models/migrationscripts/register.go +++ b/backend/plugins/gitlab/models/migrationscripts/register.go @@ -33,5 +33,6 @@ func All() []plugin.MigrationScript { new(addStdTypeToIssue221230), new(addIsDetailRequired20230210), new(addConnectionIdToTransformationRule), + new(addTypeEnvToPipeline), } } diff --git a/backend/plugins/gitlab/models/pipeline.go b/backend/plugins/gitlab/models/pipeline.go index c66e78a248e..920f2d07771 100644 --- a/backend/plugins/gitlab/models/pipeline.go +++ b/backend/plugins/gitlab/models/pipeline.go @@ -40,6 +40,9 @@ type GitlabPipeline struct { FinishedAt *time.Time Coverage string + Type string `gorm:"type:varchar(255)"` + Environment string `gorm:"type:varchar(255)"` + IsDetailRequired bool common.NoPKModel diff --git a/backend/plugins/gitlab/tasks/pipeline_convertor.go b/backend/plugins/gitlab/tasks/pipeline_convertor.go index 810b84c9be0..30e6b979b79 100644 --- a/backend/plugins/gitlab/tasks/pipeline_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_convertor.go @@ -92,6 +92,8 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error { CreatedDate: createdAt, FinishedDate: gitlabPipeline.GitlabUpdatedAt, CicdScopeId: projectIdGen.Generate(data.Options.ConnectionId, gitlabPipeline.ProjectId), + Environment: gitlabPipeline.Environment, + Type: gitlabPipeline.Type, } // rebuild the FinishedDate and DurationSec by Status diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go index 6b1ad5875bc..39c84011926 100644 --- a/backend/plugins/gitlab/tasks/pipeline_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" @@ -80,16 +81,22 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error { } gitlabPipeline := &models.GitlabPipeline{ - GitlabId: gitlabApiPipeline.Id, - ProjectId: data.Options.ProjectId, - WebUrl: gitlabApiPipeline.WebUrl, - Status: gitlabApiPipeline.Status, - GitlabCreatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt), - GitlabUpdatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt), - StartedAt: api.Iso8601TimeToTime(gitlabApiPipeline.StartedAt), - FinishedAt: api.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt), - Duration: gitlabApiPipeline.Duration, - ConnectionId: data.Options.ConnectionId, + GitlabId: gitlabApiPipeline.Id, + ProjectId: data.Options.ProjectId, + Ref: gitlabApiPipeline.Ref, + Sha: gitlabApiPipeline.Sha, + WebUrl: gitlabApiPipeline.WebUrl, + Status: gitlabApiPipeline.Status, + GitlabCreatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.CreatedAt), + GitlabUpdatedAt: api.Iso8601TimeToTime(gitlabApiPipeline.UpdatedAt), + StartedAt: api.Iso8601TimeToTime(gitlabApiPipeline.StartedAt), + FinishedAt: api.Iso8601TimeToTime(gitlabApiPipeline.FinishedAt), + Duration: gitlabApiPipeline.Duration, + ConnectionId: data.Options.ConnectionId, + + Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, gitlabApiPipeline.Ref), + Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, gitlabApiPipeline.Ref), + IsDetailRequired: false, } diff --git a/backend/plugins/gitlab/tasks/task_data.go b/backend/plugins/gitlab/tasks/task_data.go index 9c0a5a67ab6..4170a454749 100644 --- a/backend/plugins/gitlab/tasks/task_data.go +++ b/backend/plugins/gitlab/tasks/task_data.go @@ -39,6 +39,7 @@ type GitlabTaskData struct { ApiClient *helper.ApiAsyncClient ProjectCommit *models.GitlabProjectCommit TimeAfter *time.Time + RegexEnricher *helper.RegexEnricher } func DecodeAndValidateTaskOptions(options map[string]interface{}) (*GitlabOptions, errors.Error) {