Skip to content

Commit 67ed27f

Browse files
daskuznetsovasumeetpatil
daskuznetsova
and
sumeetpatil
authored
added sed cmd to transform querySuite (#4945)
* added sed cmd to transform querySuite * changed sh to bash * added checking os * fixed tests by mocking utils * fixed tests * renamed param --------- Co-authored-by: sumeet patil <[email protected]>
1 parent e2a5b09 commit 67ed27f

4 files changed

+91
-19
lines changed

‎cmd/codeqlExecuteScan.go

+20-6
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,23 @@ func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry
5959
influx.step_data.fields.codeql = true
6060
}
6161

62-
func appendCodeqlQuery(cmd []string, codeqlQuery string) []string {
63-
if len(codeqlQuery) > 0 {
64-
cmd = append(cmd, codeqlQuery)
62+
func appendCodeqlQuerySuite(utils codeqlExecuteScanUtils, cmd []string, querySuite, transformString string) []string {
63+
if len(querySuite) > 0 {
64+
if len(transformString) > 0 {
65+
var bufferOut, bufferErr bytes.Buffer
66+
utils.Stdout(&bufferOut)
67+
defer utils.Stdout(log.Writer())
68+
utils.Stderr(&bufferErr)
69+
defer utils.Stderr(log.Writer())
70+
if err := utils.RunExecutable("sh", []string{"-c", fmt.Sprintf("echo %s | sed -E \"%s\"", querySuite, transformString)}...); err != nil {
71+
log.Entry().WithError(err).Error("failed to transform querySuite")
72+
e := bufferErr.String()
73+
log.Entry().Error(e)
74+
} else {
75+
querySuite = strings.TrimSpace(bufferOut.String())
76+
}
77+
}
78+
cmd = append(cmd, querySuite)
6579
}
6680

6781
return cmd
@@ -271,7 +285,7 @@ func runGithubUploadResults(config *codeqlExecuteScanOptions, repoInfo *codeql.R
271285
func executeAnalysis(format, reportName string, customFlags map[string]string, config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) ([]piperutils.Path, error) {
272286
moduleTargetPath := filepath.Join(config.ModulePath, "target")
273287
report := filepath.Join(moduleTargetPath, reportName)
274-
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, format, report)
288+
cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, format, report)
275289
if err != nil {
276290
log.Entry().Errorf("failed to prepare command for codeql database analyze (format=%s)", format)
277291
return nil, err
@@ -323,11 +337,11 @@ func prepareCmdForDatabaseCreate(customFlags map[string]string, config *codeqlEx
323337
return cmd, nil
324338
}
325339

326-
func prepareCmdForDatabaseAnalyze(customFlags map[string]string, config *codeqlExecuteScanOptions, format, reportName string) ([]string, error) {
340+
func prepareCmdForDatabaseAnalyze(utils codeqlExecuteScanUtils, customFlags map[string]string, config *codeqlExecuteScanOptions, format, reportName string) ([]string, error) {
327341
cmd := []string{"database", "analyze", "--format=" + format, "--output=" + reportName, config.Database}
328342
cmd = codeql.AppendThreadsAndRam(cmd, config.Threads, config.Ram, customFlags)
329343
cmd = codeql.AppendCustomFlags(cmd, customFlags)
330-
cmd = appendCodeqlQuery(cmd, config.QuerySuite)
344+
cmd = appendCodeqlQuerySuite(utils, cmd, config.QuerySuite, config.TransformQuerySuite)
331345
return cmd, nil
332346
}
333347

‎cmd/codeqlExecuteScan_generated.go

+11
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cmd/codeqlExecuteScan_test.go

+53-13
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
package cmd
55

66
import (
7+
"fmt"
8+
"io"
79
"os"
810
"strings"
911
"testing"
@@ -21,7 +23,11 @@ type codeqlExecuteScanMockUtils struct {
2123

2224
func newCodeqlExecuteScanTestsUtils() codeqlExecuteScanMockUtils {
2325
utils := codeqlExecuteScanMockUtils{
24-
ExecMockRunner: &mock.ExecMockRunner{},
26+
ExecMockRunner: &mock.ExecMockRunner{
27+
Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error {
28+
return nil
29+
},
30+
},
2531
FilesMock: &mock.FilesMock{},
2632
HttpClientMock: &mock.HttpClientMock{},
2733
}
@@ -406,12 +412,13 @@ func TestPrepareCmdForDatabaseCreate(t *testing.T) {
406412

407413
func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
408414
t.Parallel()
415+
utils := codeqlExecuteScanMockUtils{}
409416

410417
t.Run("No additional flags, no querySuite, sarif format", func(t *testing.T) {
411418
config := &codeqlExecuteScanOptions{
412419
Database: "codeqlDB",
413420
}
414-
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
421+
cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
415422
assert.NoError(t, err)
416423
assert.NotEmpty(t, cmd)
417424
assert.Equal(t, 5, len(cmd))
@@ -422,7 +429,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
422429
config := &codeqlExecuteScanOptions{
423430
Database: "codeqlDB",
424431
}
425-
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "csv", "target/codeqlReport.csv")
432+
cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "csv", "target/codeqlReport.csv")
426433
assert.NoError(t, err)
427434
assert.NotEmpty(t, cmd)
428435
assert.Equal(t, 5, len(cmd))
@@ -434,7 +441,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
434441
Database: "codeqlDB",
435442
QuerySuite: "security.ql",
436443
}
437-
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
444+
cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
438445
assert.NoError(t, err)
439446
assert.NotEmpty(t, cmd)
440447
assert.Equal(t, 6, len(cmd))
@@ -448,7 +455,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
448455
Threads: "1",
449456
Ram: "2000",
450457
}
451-
cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
458+
cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
452459
assert.NoError(t, err)
453460
assert.NotEmpty(t, cmd)
454461
assert.Equal(t, 8, len(cmd))
@@ -465,7 +472,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
465472
customFlags := map[string]string{
466473
"--threads": "--threads=2",
467474
}
468-
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
475+
cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
469476
assert.NoError(t, err)
470477
assert.NotEmpty(t, cmd)
471478
assert.Equal(t, 8, len(cmd))
@@ -482,7 +489,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
482489
customFlags := map[string]string{
483490
"-j": "-j=2",
484491
}
485-
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
492+
cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
486493
assert.NoError(t, err)
487494
assert.NotEmpty(t, cmd)
488495
assert.Equal(t, 8, len(cmd))
@@ -499,7 +506,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
499506
customFlags := map[string]string{
500507
"--no-download": "--no-download",
501508
}
502-
cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
509+
cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
503510
assert.NoError(t, err)
504511
assert.NotEmpty(t, cmd)
505512
assert.Equal(t, 9, len(cmd))
@@ -559,21 +566,54 @@ func TestPrepareCmdForUploadResults(t *testing.T) {
559566
})
560567
}
561568

562-
func TestAppendCodeqlQuery(t *testing.T) {
569+
func TestAppendCodeqlQuerySuite(t *testing.T) {
563570
t.Parallel()
564571

565572
t.Run("Empty query", func(t *testing.T) {
573+
utils := newCodeqlExecuteScanTestsUtils()
566574
cmd := []string{"database", "analyze"}
567-
query := ""
568-
cmd = appendCodeqlQuery(cmd, query)
575+
querySuite := ""
576+
cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, "")
569577
assert.Equal(t, 2, len(cmd))
570578
})
571579

572580
t.Run("Not empty query", func(t *testing.T) {
581+
utils := newCodeqlExecuteScanTestsUtils()
582+
cmd := []string{"database", "analyze"}
583+
querySuite := "java-extended.ql"
584+
cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, "")
585+
assert.Equal(t, 3, len(cmd))
586+
})
587+
588+
t.Run("Add prefix to querySuite", func(t *testing.T) {
589+
utils := codeqlExecuteScanMockUtils{
590+
ExecMockRunner: &mock.ExecMockRunner{
591+
Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error {
592+
stdout.Write([]byte("test-java-security-extended.qls"))
593+
return nil
594+
},
595+
},
596+
}
597+
cmd := []string{"database", "analyze"}
598+
querySuite := "java-security-extended.qls"
599+
cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, `s/^(java|python)-(security-extended\.qls|security-and-quality\.qls)/test-\1-\2/`)
600+
assert.Equal(t, 3, len(cmd))
601+
assert.Equal(t, "test-java-security-extended.qls", cmd[2])
602+
})
603+
604+
t.Run("Don't add prefix to querySuite", func(t *testing.T) {
605+
utils := codeqlExecuteScanMockUtils{
606+
ExecMockRunner: &mock.ExecMockRunner{
607+
Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error {
608+
return fmt.Errorf("error")
609+
},
610+
},
611+
}
573612
cmd := []string{"database", "analyze"}
574-
query := "java-extended.ql"
575-
cmd = appendCodeqlQuery(cmd, query)
613+
querySuite := "php-security-extended.qls"
614+
cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, `s/^(java|python)-(security-extended\.qls|security-and-quality\.qls)/test-\1-\2/`)
576615
assert.Equal(t, 3, len(cmd))
616+
assert.Equal(t, "php-security-extended.qls", cmd[2])
577617
})
578618
}
579619

‎resources/metadata/codeqlExecuteScan.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,13 @@ spec:
240240
- STEPS
241241
- STAGES
242242
- PARAMETERS
243+
- name: transformQuerySuite
244+
type: string
245+
description: "A transform string that will be applied to the querySuite using the sed command."
246+
scope:
247+
- STEPS
248+
- STAGES
249+
- PARAMETERS
243250
containers:
244251
- image: ""
245252
outputs:

0 commit comments

Comments
 (0)