Improve Scan (#1568)

* Add conf directory * Linters for external issues produce external* files * Improve issue headline * Better evaluation of local build * Fix alpine version * Quality pass * Quality pass
okorach · Jan 20, 2025 · d28eb22 · d28eb22
1 parent 30b728f
commit d28eb22
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 27 deletions.
diff --git a/conf/run_linters.sh b/conf/run_linters.sh
@@ -29,12 +29,12 @@ buildDir="$ROOTDIR/build"
 pylintReport="$buildDir/pylint-report.out"
 # banditReport="$buildDir/bandit-report.json"
 flake8Report="$buildDir/flake8-report.out"
-shellcheckReport="$buildDir/shellcheck.json"
-trivyReport="$buildDir/trivy.json"
+shellcheckReport="$buildDir/external-issues-shellcheck.json"
+trivyReport="$buildDir/external-issues-trivy.json"
 [ ! -d "$buildDir" ] && mkdir "$buildDir"
 # rm -rf -- ${buildDir:?"."}/* .coverage */__pycache__ */*.pyc # mediatools/__pycache__  tests/__pycache__
 
-echo "Running pylint"
+echo "===> Running pylint"
 rm -f "$pylintReport"
 pylint --rcfile "$CONFDIR"/pylintrc "$ROOTDIR"/*.py "$ROOTDIR"/*/*.py -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" | tee "$pylintReport"
 re=$?
@@ -43,19 +43,19 @@ if [ "$re" == "32" ]; then
     exit $re
 fi
 
-echo "Running flake8"
+echo "===> Running flake8"
 rm -f "$flake8Report"
 # See .flake8 file for settings
 flake8 --config "$CONFIG/.flake8" "$ROOTDIR" >"$flake8Report"
 
 if [ "$localbuild" = "true" ]; then
-    echo "Running shellcheck"
+    echo "===> Running shellcheck"
     shellcheck "$ROOTDIR"/*.sh "$ROOTDIR"/*/*.sh -s bash -f json | "$CONFDIR"/shellcheck2sonar.py >"$shellcheckReport"
 
-    echo "Running checkov"
+    echo "===> Running checkov"
     checkov -d . --framework dockerfile -o sarif --output-file-path "$buildDir"
 
-    echo "Running trivy"
+    echo "===> Running trivy"
     "$CONFDIR"/build.sh docker
     trivy image -f json -o "$buildDir"/trivy_results.json olivierkorach/sonar-tools:latest
     python3 "$CONFDIR"/trivy2sonar.py < "$buildDir"/trivy_results.json > "$trivyReport"

diff --git a/conf/run_tests.sh b/conf/run_tests.sh
@@ -19,12 +19,12 @@
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 
-ME="$( basename "${BASH_SOURCE[0]}" )"
+# ME="$( basename "${BASH_SOURCE[0]}" )"
 ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
 CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 buildDir="$ROOTDIR/build"
 
-[ ! -d $buildDir ] && mkdir $buildDir
+[ ! -d "$buildDir" ] && mkdir "$buildDir"
 
 echo "Running tests"
 

diff --git a/conf/scan.sh b/conf/scan.sh
@@ -19,13 +19,17 @@
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 
-ME="$( basename "${BASH_SOURCE[0]}" )"
+# ME="$( basename "${BASH_SOURCE[0]}" )"
 ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
 CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 dolint="true"
 dotest="false"
-localbuild="false"
+if [ "$CI" == "" ]; then
+  localbuild="true"
+else
+  localbuild="false"
+fi
 
 scanOpts=()
 
@@ -37,7 +41,6 @@ do
       ;;
     -test)
       dotest="true"
-      localbuild="true"
       ;;
     *)
       scanOpts=("${scanOpts[@]}" "$1")
@@ -48,14 +51,9 @@ done
 
 buildDir="build"
 pylintReport="$buildDir/pylint-report.out"
-banditReport="$buildDir/bandit-report.json"
 flake8Report="$buildDir/flake8-report.out"
-coverageReport="$buildDir/coverage.xml"
-shellcheckReport="$buildDir/shellcheck.json"
-trivyReport="$buildDir/trivy.json"
-utReport="$buildDir/xunit-results.xml"
 
-[ ! -d $buildDir ] && mkdir $buildDir
+[ ! -d "$buildDir" ] && mkdir "$buildDir"
 rm -rf -- ${buildDir:?"."}/* .coverage */__pycache__ */*.pyc # mediatools/__pycache__  testpytest/__pycache__ testunittest/__pycache__
 
 
@@ -67,12 +65,11 @@ if [ "$dotest" == "true" ]; then
   "$CONFDIR"/run_tests.sh
 fi
 
-version=$(grep PACKAGE_VERSION $ROOTDIR/sonar/version.py | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//")
+version=$(grep PACKAGE_VERSION "$ROOTDIR/sonar/version.py" | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//")
 
 cmd="sonar-scanner -Dsonar.projectVersion=$version \
   -Dsonar.python.flake8.reportPaths=$flake8Report \
   -Dsonar.python.pylint.reportPaths=$pylintReport \
-  -Dsonar.externalIssuesReportPaths=$shellcheckReport,$trivyReport \
   -Dsonar.login=$SONAR_TOKEN \
   -Dsonar.token=$SONAR_TOKEN \
   "${scanOpts[*]}""
@@ -82,14 +79,26 @@ if ls $buildDir/coverage*.xml >/dev/null 2>&1; then
 else
   echo "===> NO COVERAGE REPORT"
 fi
+
 if ls $buildDir/xunit-results*.xml >/dev/null 2>&1; then
   cmd="$cmd -Dsonar.python.xunit.reportPath=$buildDir/xunit-results*.xml"
 else
   echo "===> NO UNIT TESTS REPORT"
   cmd="$cmd -Dsonar.python.xunit.reportPath="
 fi
 
-echo "Running: $cmd"
+if ls $buildDir/external-issues*.json >/dev/null 2>&1; then
+  files=$(ls $buildDir/external-issues*.json | tr '\n' ' ' | sed -E -e 's/ +$//' -e 's/ +/,/g')
+  echo "EXTERNAL ISSUES FILES = $files"
+  cmd="$cmd -Dsonar.externalIssuesReportPaths=$files"
+else
+  echo "===> NO EXTERNAL ISSUES"
+fi
+
+
+echo
+echo "Running: $cmd" | sed "s/$SONAR_TOKEN/<SONAR_TOKEN>/g"
+echo
 
 $cmd
 

diff --git a/conf/snapshot.Dockerfile b/conf/snapshot.Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:3.20.3
+FROM alpine:3.21.2
 LABEL maintainer="[email protected]" 
 ENV IN_DOCKER="Yes"
 

diff --git a/conf/trivy2sonar.py b/conf/trivy2sonar.py
@@ -36,15 +36,15 @@ def main() -> None:
     text = "".join(sys.stdin)
 
     rules_dict = {}
-    issue_list = []
+    issue_list = {}
 
     for issue in json.loads(text)["Results"][0]["Vulnerabilities"]:
 
         sonar_issue = {
             "ruleId": f"{TOOLNAME}:{issue['VulnerabilityID']}",
             "effortMinutes": 30,
             "primaryLocation": {
-                "message": issue["Title"],
+                "message": f"{issue['VulnerabilityID']} - {issue['Title']}",
                 "filePath": "conf/snapshot.Dockerfile",
                 "textRange": {
                     "startLine": 1,
@@ -54,7 +54,7 @@ def main() -> None:
                 },
             },
         }
-        issue_list.append(sonar_issue)
+        issue_list[sonar_issue["primaryLocation"]["message"]] = sonar_issue
         # score = max([v["V3Score"] for v in issue['CVSS'].values()])
         # if score <= 4:
         #     sev = "LOW"
@@ -74,7 +74,7 @@ def main() -> None:
             "impacts": [{"softwareQuality": "SECURITY", "severity": sev_mqr}],
         }
 
-    external_issues = {"rules": list(rules_dict.values()), "issues": issue_list}
+    external_issues = {"rules": list(rules_dict.values()), "issues": list(issue_list.values())}
     print(json.dumps(external_issues, indent=3, separators=(",", ": ")))
 
 

diff --git a/sonar-project.properties b/sonar-project.properties
@@ -4,7 +4,7 @@ sonar.projectName=Sonar Tools
 sonar.python.version=3.9
 
 # Comma-separated paths to directories with sources (required)
-sonar.sources=sonar, cli, migration, setup.py, setup_migration.py
+sonar.sources=sonar, cli, migration, conf, setup.py, setup_migration.py
 
 # Encoding of the source files
 sonar.sourceEncoding=UTF-8