From 9c9c8d28ebd45568f28b62afbbfbe6cda95efa3b Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Wed, 24 Jul 2024 17:32:03 +0200 Subject: [PATCH 1/8] chore: Remove unnecessary file from .gitignore. --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9c61f10..d3b7fee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ gitleaks-report.json gitleaks-report-detailed.json -temp.json From 761fcd155fb30a44c918d0f050ae95c435ec457c Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Wed, 24 Jul 2024 17:35:07 +0200 Subject: [PATCH 2/8] docs: Update READMEs to add 2 new ENVs, add 1 new JSON field and fix some existing commands and descriptions mentioned. --- README.md | 29 ++++++++++++++++++++--------- docker/README.md | 29 ++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ee90cdc..d677f0c 100644 --- a/README.md +++ b/README.md @@ -15,21 +15,24 @@ Below you can find an example of the JSON report generated: ```json [ { - "Description": "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches.", - "File": "./code/main.py", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "File": "scripts/main.py", "Line No.": "11", - "Secret Type": "hashicorp-tf-password", + "Link": "https://gitlab.com/my-projects/my-repo/-/blob/master/scripts/main.py#L11", + "Secret Type": "generic-api-key", "Commit": "__REDACTED__", "Author": "__REDACTED__" }, { "Description": "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches.", - "File": "./code/main.conf", - "Line No.": "30", + "File": "configurations/main.tf", + "Line No.": "6", + "Link": "https://gitlab.com/my-projects/my-repo/-/blob/master/configurations/main.tf#L6", "Secret Type": "hashicorp-tf-password", "Commit": "__REDACTED__", "Author": "__REDACTED__" } + ... ] ``` @@ -82,8 +85,16 @@ Following are the prerequisites to be met once before you begin: ### Execution Instructions Once all the prerequisites are met, set the following environment variables: - - `PATH_TO_GIT_REPO` - - Description: To keep the size of the git repository to be cloned lower to make the job faster. + - `LOCAL_PATH_TO_GIT_REPO` + - Description: Local path to the Git repository. + - Example: `/Users/Abdullah.Khawer/Desktop/my-projects/my-repo` + - Requirement: REQUIRED + - `REMOTE_PATH_TO_GIT_REPO` + - Description: Remote path to the Git repository. + - Example: `https://gitlab.com/my-projects/my-repo` + - Requirement: REQUIRED + - `BRANCH_NAME` + - Description: Name of the branch in the Git repository against which secrets detection tool will be executed. - Example: `/Users/Abdullah.Khawer/Desktop/myrepo` - Requirement: REQUIRED - `CONFLUENCE_ENABLED` @@ -120,10 +131,10 @@ Once all the prerequisites are met, set the following environment variables: - Example: `[https://mydomain.atlassian.net](https://hooks.slack.com/services/__REDACTED__/__REDACTED__/__REDACTED__)` - Requirement: REQUIRED (if `SLACK_ENABLED` is set to `1`) -And then simply run the following 3 commands in the correct order: +And then simply run the following 2 commands: - `bash gitleaks.sh` - `python3 main.py TIME_ZONE REPOSITORY_NAME BRANCH_NAME [JSON_REPORT_URL]` - - Example: `python3 main.py Europe/Amsterdam myproj/myrepo master` + - Example: `python3 main.py Europe/Amsterdam my-projects/my-repo master` - Note: Details about supported time zones and their constant names can be found here: [pypi.org > project > pytz > Helpers](https://pypi.org/project/pytz/#:~:text=through%20multiple%20timezones.-,Helpers,-There%20are%20two) ## Automatically via CI/CD Pipeline diff --git a/docker/README.md b/docker/README.md index 48bc37e..552d0bf 100644 --- a/docker/README.md +++ b/docker/README.md @@ -15,21 +15,24 @@ Below you can find an example of the JSON report generated: ```json [ { - "Description": "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches.", - "File": "./code/main.py", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "File": "scripts/main.py", "Line No.": "11", - "Secret Type": "hashicorp-tf-password", + "Link": "https://gitlab.com/my-projects/my-repo/-/blob/master/scripts/main.py#L11", + "Secret Type": "generic-api-key", "Commit": "__REDACTED__", "Author": "__REDACTED__" }, { "Description": "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches.", - "File": "./code/main.conf", - "Line No.": "30", + "File": "configurations/main.tf", + "Line No.": "6", + "Link": "https://gitlab.com/my-projects/my-repo/-/blob/master/configurations/main.tf#L6", "Secret Type": "hashicorp-tf-password", "Commit": "__REDACTED__", "Author": "__REDACTED__" } + ... ] ``` @@ -46,8 +49,16 @@ Below you can find an example of the Slack notification: ### Execution Instructions Set the following environment variables: - - `PATH_TO_GIT_REPO` - - Description: To keep the size of the git repository to be cloned lower to make the job faster. + - `LOCAL_PATH_TO_GIT_REPO` + - Description: Local path to the Git repository. + - Example: `/Users/Abdullah.Khawer/Desktop/my-projects/my-repo` + - Requirement: REQUIRED + - `REMOTE_PATH_TO_GIT_REPO` + - Description: Remote path to the Git repository. + - Example: `https://gitlab.com/my-projects/my-repo` + - Requirement: REQUIRED + - `BRANCH_NAME` + - Description: Name of the branch in the Git repository against which secrets detection tool will be executed. - Example: `/Users/Abdullah.Khawer/Desktop/myrepo` - Requirement: REQUIRED - `CONFLUENCE_ENABLED` @@ -85,11 +96,11 @@ Set the following environment variables: - Requirement: REQUIRED (if `SLACK_ENABLED` is set to `1`) And then simply run the following 4 commands: -- `docker run --platform linux/amd64 -it -e PATH_TO_GIT_REPO=/git_repo -e CONFLUENCE_ENABLED=1 -e CONFLUENCE_SITE=$CONFLUENCE_SITE -e CONFLUENCE_USER_EMAIL_ID=$CONFLUENCE_USER_EMAIL_ID -e CONFLUENCE_USER_TOKEN=$CONFLUENCE_USER_TOKEN -e CONFLUENCE_PAGE_TITLE=$CONFLUENCE_PAGE_TITLE -e CONFLUENCE_PAGE_SPACE=$CONFLUENCE_PAGE_SPACE -e SLACK_ENABLED=1 -e SLACK_WEBHOOK_URL=$SLACK_WEBHOOK_URL -v $PATH_TO_GIT_REPO:/git_repo abdullahkhawer/find-and-report-secrets-in-code:latest` +- `docker run --platform linux/amd64 -it -e LOCAL_PATH_TO_GIT_REPO=$LOCAL_PATH_TO_GIT_REPO -e REMOTE_PATH_TO_GIT_REPO=$REMOTE_PATH_TO_GIT_REPO -e BRANCH_NAME=$BRANCH_NAME -e CONFLUENCE_ENABLED=$CONFLUENCE_ENABLED -e CONFLUENCE_SITE=$CONFLUENCE_SITE -e CONFLUENCE_USER_EMAIL_ID=$CONFLUENCE_USER_EMAIL_ID -e CONFLUENCE_USER_TOKEN=$CONFLUENCE_USER_TOKEN -e CONFLUENCE_PAGE_TITLE=$CONFLUENCE_PAGE_TITLE -e CONFLUENCE_PAGE_SPACE=$CONFLUENCE_PAGE_SPACE -e SLACK_ENABLED=$SLACK_ENABLED -e SLACK_WEBHOOK_URL=$SLACK_WEBHOOK_URL -v $LOCAL_PATH_TO_GIT_REPO:$LOCAL_PATH_TO_GIT_REPO abdullahkhawer/find-and-report-secrets-in-code:latest` - `export PATH=$PATH:/usr/local/gitleaks` - `bash /find-and-report-secrets-in-code/gitleaks.sh` - `python3 /find-and-report-secrets-in-code/main.py TIME_ZONE REPOSITORY_NAME BRANCH_NAME [JSON_REPORT_URL]` - - Example: `python3 /find-and-report-secrets-in-code/main.py Europe/Amsterdam myproj/myrepo master` + - Example: `python3 /find-and-report-secrets-in-code/main.py Europe/Amsterdam my-projects/my-repo master` - Note: Details about supported time zones and their constant names can be found here: [pypi.org > project > pytz > Helpers](https://pypi.org/project/pytz/#:~:text=through%20multiple%20timezones.-,Helpers,-There%20are%20two) ## Automatically via CI/CD Pipeline From f8cfcf9a605696b78481315b4bf286dc3f21ebdd Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Wed, 24 Jul 2024 17:37:20 +0200 Subject: [PATCH 3/8] ci: Add 2 new variables and use image 1.1.0 --- ci/.gitlab-ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml index 2198517..ffc4fe4 100644 --- a/ci/.gitlab-ci.yml +++ b/ci/.gitlab-ci.yml @@ -25,7 +25,7 @@ stage: scan extends: - .find-secrets:variables - image: abdullahkhawer/find-and-report-secrets-in-code:1.0.0 + image: abdullahkhawer/find-and-report-secrets-in-code:1.1.0 before_script: - | if [ -n "$CONFLUENCE_ENABLED" ] && [ "$CONFLUENCE_ENABLED" -eq 1 ]; then @@ -63,13 +63,15 @@ fi - git fetch origin $CI_COMMIT_BRANCH script: + - export LOCAL_PATH_TO_GIT_REPO=$(pwd) + - export REMOTE_PATH_TO_GIT_REPO=$CI_PROJECT_URL + - export BRANCH_NAME=$CI_COMMIT_BRANCH + - export REPO_NAME=$CI_PROJECT_PATH - export PATH=$PATH:/usr/local/gitleaks - - export PATH_TO_GIT_REPO=$(pwd) - - export REPO_NAME=$(echo "$CI_PROJECT_DIR" | sed 's|/builds/||') - cd /find-and-report-secrets-in-code/ - bash ./gitleaks.sh - - python3 main.py "Europe/Amsterdam" $REPO_NAME $CI_COMMIT_BRANCH $CI_JOB_URL/artifacts/raw/gitleaks-report.json - - cp ./gitleaks-report.json $PATH_TO_GIT_REPO/gitleaks-report.json + - python3 main.py "Europe/Amsterdam" $REPO_NAME $BRANCH_NAME $CI_JOB_URL/artifacts/raw/gitleaks-report.json + - cp ./gitleaks-report.json $LOCAL_PATH_TO_GIT_REPO/gitleaks-report.json artifacts: paths: - gitleaks-report.json From d60340427ef29b0d17ea475e0bab2c71fb86e274 Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Wed, 24 Jul 2024 17:39:14 +0200 Subject: [PATCH 4/8] feat: Update shell script to prepare and add URL for each finding in the JSON report. --- gitleaks.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/gitleaks.sh b/gitleaks.sh index 0b35043..4036511 100644 --- a/gitleaks.sh +++ b/gitleaks.sh @@ -4,25 +4,25 @@ echo "Script Execution Started!" # remove Gitleaks reports if they exist already echo "Removing Gitleaks reports if they exist already..." -rm -rf ${PATH_TO_GIT_REPO}/gitleaks-report-detailed.json +rm -rf ${LOCAL_PATH_TO_GIT_REPO}/gitleaks-report-detailed.json rm -rf ./gitleaks-report.json # run Gitleaks to find secrets and generate a detailed report in JSON for the secrets found echo "Running Gitleaks to find secrets and generating a detailed report in JSON for the secrets found..." -gitleaks detect -r ${PATH_TO_GIT_REPO}/gitleaks-report-detailed.json -f json -s ${PATH_TO_GIT_REPO} --redact --no-git +gitleaks detect -r ${LOCAL_PATH_TO_GIT_REPO}/gitleaks-report-detailed.json -f json -s ${LOCAL_PATH_TO_GIT_REPO} --redact --no-git # create a final report in JSON using the detailed report having relevant information only echo "Creating a final report in JSON using the detailed report having relevant information only..." echo "[" > ./gitleaks-report.json -cat ${PATH_TO_GIT_REPO}/gitleaks-report-detailed.json | jq -c '.[]' | while read -r line; do +cat ${LOCAL_PATH_TO_GIT_REPO}/gitleaks-report-detailed.json | jq -c '.[]' | while read -r line; do description=$(jq -r '.Description' <<< "$line") start_line=$(jq -r '.StartLine' <<< "$line") file=$(jq -r '.File' <<< "$line") - file=$(echo "$file" | sed "s|^${PATH_TO_GIT_REPO}|.|") + file=$(echo "$file" | sed "s|^${LOCAL_PATH_TO_GIT_REPO}/||") secret_type=$(jq -r '.RuleID' <<< "$line") # use 'git blame' to find the commit id and author for each finding - blame=$(cd ${PATH_TO_GIT_REPO} && git blame -L "$start_line","$start_line" "$file" --porcelain) + blame=$(cd ${LOCAL_PATH_TO_GIT_REPO} && git blame -L "$start_line","$start_line" ./"$file" --porcelain) commit_id=$(echo "$blame" | awk 'NR==1' | awk -F ' ' '{print $1}') author=$(echo "$blame" | awk 'NR==2' | awk -F 'author ' '{print $2}') @@ -31,10 +31,11 @@ cat ${PATH_TO_GIT_REPO}/gitleaks-report-detailed.json | jq -c '.[]' | while read --arg desc "$description" \ --arg file "$file" \ --arg line_no "$start_line" \ + --arg url "${REMOTE_PATH_TO_GIT_REPO}/-/blob/${BRANCH_NAME}/${file}#L${start_line}" \ --arg type "$secret_type" \ --arg commit "$commit_id" \ --arg author "$author" \ - '{"Description": $desc, "File": $file, "Line No.": $line_no, "Secret Type": $type, "Commit": $commit, "Author": $author}' >> ./gitleaks-report.json + '{"Description": $desc, "File": $file, "Line No.": $line_no, "Link": $url, "Secret Type": $type, "Commit": $commit, "Author": $author}' >> ./gitleaks-report.json echo "," >> ./gitleaks-report.json done From 2d905d622cb38cf4ba8a418ef36cfd07aa1358f7 Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Wed, 24 Jul 2024 17:44:33 +0200 Subject: [PATCH 5/8] feat: Update python script to improve logging, comments, pylint score from 1.44 to 9.25/10 by refactoring code, HTML content template to add link to the file reference where secret is detected and Slack notification message along with its format in case of both no secrets found and 1 or more secrets found. --- main.py | 423 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 256 insertions(+), 167 deletions(-) diff --git a/main.py b/main.py index cbd7a53..09998ff 100644 --- a/main.py +++ b/main.py @@ -1,26 +1,37 @@ +#!/usr/bin/python + +""" +Python script to update an Atlassian Confluence page with the secrets found +based on the report generated by the custom shell script 'gitleaks.sh' which +is using Gitleaks and to send a notification on Slack. +""" + +# required imports import json import os -import pytz import re -import requests import sys -from atlassian import Confluence from datetime import datetime +import pytz +import requests +from atlassian import Confluence + +print("Script Execution Started!") # get time zone, repository name and branch name from the arguments passed to the script if len(sys.argv) < 4 or len(sys.argv) > 5: - print("ERROR: Invalid arguments passed.") - print("Usage: python main.py TIME_ZONE REPOSITORY_NAME BRANCH_NAME [JSON_REPORT_URL]") - print("Example: python main.py Europe/Amsterdam myproj/myrepo master") - sys.exit(1) + print("ERROR: Invalid arguments passed.") + print("Usage: python main.py TIME_ZONE REPOSITORY_NAME BRANCH_NAME [JSON_REPORT_URL]") + print("Example: python main.py Europe/Amsterdam my-projects/my-repo master") + sys.exit(1) time_zone = sys.argv[1] repo_name = sys.argv[2] branch_name = sys.argv[3] json_report_url = "" if len(sys.argv) == 5: - json_report_url = sys.argv[4] + json_report_url = sys.argv[4] -# Get the current time in UTC and convert it into the desired time zone's time +# get the current time in UTC and convert it into the desired time zone's time time_now = datetime.now() target_timezone = pytz.timezone(time_zone) time_now = time_now.astimezone(target_timezone) @@ -29,195 +40,273 @@ # get environment variables related to Confluence confluence_enabled = os.getenv("CONFLUENCE_ENABLED") if confluence_enabled is None: - print("ERROR: CONFLUENCE_ENABLED environment variable is not set.") - sys.exit(1) + print("ERROR: CONFLUENCE_ENABLED environment variable is not set.") + sys.exit(1) elif confluence_enabled == "1": - confluence_site = os.getenv("CONFLUENCE_SITE") - confluence_user = os.getenv("CONFLUENCE_USER_EMAIL_ID") - confluence_pass = os.getenv("CONFLUENCE_USER_TOKEN") - page_title = os.getenv("CONFLUENCE_PAGE_TITLE") - page_space = os.getenv("CONFLUENCE_PAGE_SPACE") - if confluence_site is None: - print("ERROR: CONFLUENCE_SITE environment variable is not set.") - sys.exit(1) - if confluence_user is None: - print("ERROR: CONFLUENCE_USER_EMAIL_ID environment variable is not set.") - sys.exit(1) - if confluence_pass is None: - print("ERROR: CONFLUENCE_USER_TOKEN environment variable is not set.") - sys.exit(1) - if page_title is None: - print("ERROR: CONFLUENCE_PAGE_TITLE environment variable is not set.") - sys.exit(1) - if page_space is None: - print("ERROR: CONFLUENCE_PAGE_SPACE environment variable is not set.") - sys.exit(1) + confluence_site = os.getenv("CONFLUENCE_SITE") + confluence_user = os.getenv("CONFLUENCE_USER_EMAIL_ID") + confluence_pass = os.getenv("CONFLUENCE_USER_TOKEN") + page_title = os.getenv("CONFLUENCE_PAGE_TITLE") + page_space = os.getenv("CONFLUENCE_PAGE_SPACE") + if confluence_site is None: + print("ERROR: CONFLUENCE_SITE environment variable is not set.") + sys.exit(1) + if confluence_user is None: + print("ERROR: CONFLUENCE_USER_EMAIL_ID environment variable is not set.") + sys.exit(1) + if confluence_pass is None: + print("ERROR: CONFLUENCE_USER_TOKEN environment variable is not set.") + sys.exit(1) + if page_title is None: + print("ERROR: CONFLUENCE_PAGE_TITLE environment variable is not set.") + sys.exit(1) + if page_space is None: + print("ERROR: CONFLUENCE_PAGE_SPACE environment variable is not set.") + sys.exit(1) # get environment variables related to Slack slack_enabled = os.getenv("SLACK_ENABLED") if slack_enabled is None: - print("ERROR: SLACK_ENABLED environment variable is not set.") - sys.exit(1) + print("ERROR: SLACK_ENABLED environment variable is not set.") + sys.exit(1) elif slack_enabled == "1": - slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL") - if slack_webhook_url is None: - print("ERROR: SLACK_WEBHOOK_URL environment variable is not set.") - sys.exit(1) + slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL") + if slack_webhook_url is None: + print("ERROR: SLACK_WEBHOOK_URL environment variable is not set.") + sys.exit(1) # define HTML page template if confluence_enabled == "1": - html_template = """ -

Repository: {} - Branch: {}

-

Last Scan Time: {}

-

Secrets Found: {}

- - - - - - - - - - - {} - -
- -

Description

-
-

File

-
-

Line No.

-
-

Secret Type

-
-

Commit ID

-
-

Commit Author

-
- """ + html_template = """ +

Repository: {} - Branch: {}

+

Last Scan Time: {}

+

Secrets Found: {}

+ + + + + + + + + + {} + +
+ +

Description

+
+

File Reference

+
+

Secret Type

+
+

Commit ID

+
+

Commit Author

+
+ """ # define HTML row template if confluence_enabled == "1": - row_template = """ - - - {} - - -

{}

- - -

{}

- - -

{}

- - -

{}

- - -

{}

- - -

{}

- - - """ - -# connect to Confluence + row_template = """ + + + {} + + +

{}

+ + +

{}

+ + +

{}

+ + +

{}

+ + +

{}

+ + + """ + +# connect to Atlassian Confluence if confluence_enabled == "1": - confluence = Confluence(url=confluence_site, username=confluence_user, password=confluence_pass) + print("Connecting to Atlassian Confluence...") + confluence = Confluence( + url=confluence_site, + username=confluence_user, + password=confluence_pass + ) -# resolve page ID +# resolve the page ID on Confluence if confluence_enabled == "1": - page_id = confluence.get_page_id(page_space, page_title) + print("Resolving the page ID on Confluence...") + page_id = confluence.get_page_id( + page_space, + page_title + ) -# get current page content +# fetch current page content from the page on Confluence if confluence_enabled == "1": - page = confluence.get_page_by_id(page_id, expand='body.storage') - page_content = page['body']['storage']['value'] + print("Fetching current page content from the page on Confluence...") + page = confluence.get_page_by_id( + page_id, + expand='body.storage' + ) + page_content = page['body']['storage']['value'] -# read JSON from file -with open("./gitleaks-report.json", "r") as file: - data = json.load(file) +# read JSON from the report generated by the custom shell script 'gitleaks.sh' +print("Reading JSON from the report generated by the custom shell script 'gitleaks.sh'...") +with open("./gitleaks-report.json", "r", encoding='UTF-8') as file: + data = json.load(file) +secrets_count = len(data) -# update HTML page template and find unique commit authors from the data read from JSON file +# update HTML page template and find unique commit authors from the JSON data read +print("Updating HTML page template and finding unique commit authors from the JSON data read...") authors = [] rows = "" rows_count = 1 for entry in data: - author = entry["Author"] - if confluence_enabled == "1": - description = entry["Description"] - file = entry["File"] - line_no = entry["Line No."] - secret_type = entry["Secret Type"] - commit = entry["Commit"] - rows += row_template.format(rows_count, description, file, line_no, secret_type, commit, author) - if slack_enabled == "1": - authors.append(author) - rows_count = rows_count + 1 + author = entry["Author"] + if confluence_enabled == "1": + description = entry["Description"] + file_reference = f'{entry["File"]}:{entry["Line No."]}' + secret_type = entry["Secret Type"] + commit = entry["Commit"] + rows += row_template.format( + rows_count, + description, + file_reference, + secret_type, + commit, + author + ) + if slack_enabled == "1": + authors.append(author) + rows_count = rows_count + 1 if confluence_enabled == "1": - html_template = html_template.format(repo_name, branch_name, time_now, len(data), rows) + html_template = html_template.format( + repo_name, + branch_name, + time_now, + secrets_count, + rows + ) if slack_enabled == "1": - authors = list(set(authors)) - authors.sort() + authors = list(set(authors)) + authors.sort() -# define the pattern to replace the respective div +# define the pattern to replace the respective div using repository and branch name if confluence_enabled == "1": - pattern = r'

Repository: {} - Branch: {}.*?'.format(repo_name, branch_name) + print("Defining the pattern to replace the respective div using repository and branch name...") + pattern = f'

Repository: {repo_name} - Branch: {branch_name}.*?' -# Check if pattern is found or not and update the page content accordingly +# check if pattern is found or not and update the page content accordingly if confluence_enabled == "1": - new_page_content = page_content - if re.search(pattern, page_content, flags=re.DOTALL): - # pattern found; replace matching pattern in the existing content with the new HTML page template - new_page_content = re.sub(pattern, html_template, page_content, flags=re.DOTALL) - else: - # pattern not found; add the new HTML page template at the end of the existing content - new_page_content = new_page_content + "\n" + html_template + print("Checking if pattern is found or not and updating the page content accordingly...") + new_page_content = page_content + re_result = re.search( + pattern, + page_content, + flags=re.DOTALL + ) + if re_result: + # pattern found; replace the existing content with the new HTML page template + new_page_content = re.sub( + pattern, + html_template, + page_content, + flags=re.DOTALL + ) + else: + # pattern not found; add the new HTML page template at the end of the page + new_page_content = new_page_content + "\n" + html_template # update page with new content if confluence_enabled == "1": - try: - confluence.update_page(page_id, page_title, new_page_content, type='page', representation='storage', minor_edit=False, full_width=True) - print("Confluence page updated successfully.") - except Exception as err: - print("ERROR: Failed to update Confluence page.") - print(f'ERROR: {err}') - sys.exit(1) - -# send notification to Slack + try: + confluence.update_page( + page_id, page_title, + new_page_content, + type='page', + representation='storage', + minor_edit=False, + full_width=True + ) + print("Confluence page is updated successfully.") + except Exception as err: + print("ERROR: Failed to update Confluence page.") + print(f'ERROR: {err}') + sys.exit(1) + +# prepare a message to send it as a notification to Slack if slack_enabled == "1": - message = "*Secrets Detection Notification*" - message += f'\n>:file_folder: *Repository:* `{repo_name}`' - message += f'\n>:git: *Branch:* `{branch_name}`' - message += f'\n>:clock1: *Last Scan Time:* `{time_now}`' - message += f'\n>:warning: *Secrets Found:* `{len(data)}`' - message += f'\n>:technologist: *Commit Authors:* \n>ā€¢ *{'*\n>ā€¢ *'.join(authors)}*' - if confluence_enabled == "1": - message += f'\n:link: More details can be found here: <{confluence_site}/wiki/spaces/{page_space}/pages/{page_id}/{page_title}|Confluence Page>' - if json_report_url != "": - message += f'\n:link: JSON report can be found here: <{json_report_url}|JSON Report>' - slack_data = { - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": message + print("Preparing a message to send it as a notification to Slack...") + message_color = "#02cc38" # green + message_title = f':tada: No Secrets found in the :file_folder: "{repo_name}" repository' + message_title += f' on the :git: "{branch_name}" branch.' + message = f'\n:clock1: *Last Scan Time:* `{time_now}`' + slack_message = { + "attachments": [ + { + "pretext": "*Secrets Detection Notification*", + "title": message_title, + "text": message, + "color": message_color } - } - ] - } - headers = {'Content-Type': "application/json"} - try: - response = requests.post(slack_webhook_url, data=json.dumps(slack_data), headers=headers) - response.raise_for_status() - print("Notification sent to Slack successfully.") - except Exception as err: - print("ERROR: Failed to send notification to Slack.") - print(f'ERROR: {err}') - sys.exit(1) + ] + } + message_links = "" + if secrets_count != 0: + message_color = "#cc0202" # red + message_title = f':warning: {secrets_count} Secrets found in the :file_folder:' + message_title += f' "{repo_name}" repository on the :git: "{branch_name}" branch.' + message += f'\n:technologist: *Commit Authors:* \nā€¢ *{'*\nā€¢ *'.join(authors)}*' + slack_message = { + "attachments": [ + { + "pretext": "*Secrets Detection Notification*", + "title": message_title, + "text": message, + "color": message_color + } + ] + } + if confluence_enabled == "1": + message_links += f'\n:link: More details can be found here: <{confluence_site}/wiki/' + message_links += f'spaces/{page_space}/pages/{page_id}/{page_title}|Confluence Page>' + if json_report_url != "": + message_links += f'\n:link: JSON report can be found here: <{json_report_url}' + message_links += '|JSON Report>' + if message_links != "": + slack_message_links = { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": message_links + } + } + ] + } + slack_message["attachments"].append(slack_message_links) + + # send the prepared message as a notification to Slack + print("Sending the prepared message as a notification to Slack...") + try: + response = requests.post( + slack_webhook_url, + data=json.dumps(slack_message), + headers={'Content-Type': "application/json"}, + timeout=10) + response.raise_for_status() + print("Notification sent to Slack successfully.") + except Exception as err: + print("ERROR: Failed to send notification to Slack.") + print(f'ERROR: {err}') + sys.exit(1) + +print("Script Execution Completed!") From d734d63f226a268209fa948abb220a6388bbd535 Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Wed, 24 Jul 2024 17:45:43 +0200 Subject: [PATCH 6/8] chore: Update version to v1.1.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b18d465..795460f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.0.1 +v1.1.0 From e75f55226867ca232b19dcd7cea998488521c8db Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Wed, 24 Jul 2024 17:51:16 +0200 Subject: [PATCH 7/8] Update CHANGELOG.md --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9699603..239526e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. +## [1.1.0] - 2024-07-24 + +### šŸš€ Features + +- Update shell script to prepare and add URL for each finding in the JSON report. +- Update python script to improve logging, comments, pylint score from 1.44 to 9.25/10 by refactoring code, HTML content template to add link to the file reference where secret is detected and Slack notification message along with its format in case of both no secrets found and 1 or more secrets found. + +### šŸ“š Documentation + +- Update READMEs to add 2 new ENVs, add 1 new JSON field and fix some existing commands and descriptions mentioned. + +### āš™ļø Miscellaneous Tasks + +- Remove unnecessary file from .gitignore. +- Add 2 new variables and use image 1.1.0 +- Update version to v1.1.0 + ## [1.0.1] - 2024-07-03 [1.0.1]: https://github.com/abdullahkhawer/find-and-report-secrets-in-code/releases/tag/v1.0.1 From ae31a911153f40db4f2ac0717911afe0c095a94c Mon Sep 17 00:00:00 2001 From: Abdullah Khawer Date: Wed, 24 Jul 2024 17:52:54 +0200 Subject: [PATCH 8/8] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 239526e..9b5e085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. ## [1.1.0] - 2024-07-24 +[1.1.0]: https://github.com/abdullahkhawer/find-and-report-secrets-in-code/releases/tag/v1.1.0 + ### šŸš€ Features - Update shell script to prepare and add URL for each finding in the JSON report.