Merge branch 'apache:master' into optional_schema

Author: derrickaw

.github/workflows/beam_Infrastructure_PolicyEnforcer.yml

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This workflow works with the infrastructure policy enforcer to
+# generate a report of IAM and Service Account Policies violations
+
+name: Infrastructure Policy Enforcer
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Once a week at 9:00 AM on Monday
+    - cron: '0 9 * * 1'
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
+  cancel-in-progress: true
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  beam_Infrastructure_PolicyEnforcer:
+    name: Check and Report Infrastructure Policies Violations
+    runs-on: [self-hosted, ubuntu-20.04, main]
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.13'
+          
+      - name: Install Python dependencies
+        working-directory: ./infra/enforcement
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          
+      - name: Setup gcloud
+        uses: google-github-actions/setup-gcloud@v2
+          
+      - name: Run IAM Policy Enforcement
+        working-directory: ./infra/enforcement
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          SMTP_SERVER: smtp.gmail.com
+          SMTP_PORT: 465
+          EMAIL_ADDRESS: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
+          EMAIL_PASSWORD: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
+          EMAIL_RECIPIENT: "[email protected]"
+        run: python iam.py --action print
+
+      - name: Run Account Keys Policy Enforcement
+        working-directory: ./infra/enforcement
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          SMTP_SERVER: smtp.gmail.com
+          SMTP_PORT: 465
+          EMAIL_ADDRESS: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
+          EMAIL_PASSWORD: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
+          EMAIL_RECIPIENT: "[email protected]"
+        run: python account_keys.py --action print

.github/workflows/beam_UserRoles.yml renamed to .github/workflows/beam_Infrastructure_UsersPermissions.yml

@@ -19,7 +19,12 @@
 
 # Infrastructure rules enforcement
 
-This module is used to check that the infrastructure rules are being used.
+This module is used to check that the infrastructure rules are being used and provides automated notifications for compliance violations.
+
+The enforcement tools support multiple notification methods:
+- **GitHub Issues**: Automatically create GitHub issues with detailed compliance reports
+- **Email Notifications**: Send email alerts via SMTP for compliance violations
+- **Console Output**: Print detailed reports to console for manual review
 
 ## IAM Policies
 
@@ -34,8 +39,11 @@ You can specify the action either through the configuration file (`config.yml`)
 # Check compliance and report issues (default)
 python iam.py --action check
 
-# Create GitHub issue if compliance violations are found
-python iam.py --action issue
+# Create/update GitHub issue and send email if compliance violations are found
+python iam.py --action announce
+
+# Print announcement details for testing purposes (no actual issue created)
+python iam.py --action print
 
 # Generate new compliance file based on current IAM policy
 python iam.py --action generate
@@ -44,7 +52,8 @@ python iam.py --action generate
 ### Actions
 
 - **check**: Validates IAM policies against defined policies and reports any differences (default behavior)
-- **issue**: Creates a GitHub issue when IAM policies differ from the defined ones, including detailed permission discrepancies
+- **announce**: Creates or updates a GitHub issue and sends an email notification when IAM policies differ from the defined ones. If no open issue exists, creates a new one; if an open issue exists, updates the issue body with current violations
+- **print**: Prints announcement details for testing purposes without creating actual GitHub issues or sending emails
 - **generate**: Updates the compliance file to match the current GCP IAM policy, creating a new baseline from existing permissions
 
 ### Features
@@ -58,16 +67,31 @@ The IAM Policy enforcement tool provides the following capabilities:
 - **Sorted Output**: Provides consistent, sorted output for easy comparison and review
 - **Detailed Reporting**: Comprehensive reporting of permission differences with clear before/after comparisons
 - **GitHub Integration**: Automatic issue creation with detailed compliance violation reports
+- **Email Notifications**: Optional email notifications for compliance issues via SMTP
+- **Issue Management**: Smart issue handling - creates new issues when none exist, updates existing open issues with current violations
+- **Testing Support**: Print action allows testing notification content without actually sending
 
 ### Configuration
 
 The `config.yml` file supports the following parameters for IAM policies:
 
 - `project_id`: GCP project ID to check (default: `apache-beam-testing`)
 - `users_file`: Path to the YAML file containing expected IAM policies (default: `../iam/users.yml`)
-- `action`: Default action to perform (`check`, `issue`, or `generate`)
+- `action`: Default action to perform (`check`, `announce`, `print`, or `generate`)
 - `logging`: Logging configuration (level and format)
 
+### Environment Variables (for announce action)
+
+When using the `announce` action, the following environment variables are required:
+
+- `GITHUB_TOKEN`: GitHub personal access token for creating issues
+- `GITHUB_REPOSITORY`: Repository in format `owner/repo` (default: `apache/beam`)
+- `SMTP_SERVER`: SMTP server for email notifications
+- `SMTP_PORT`: SMTP port (default: 587)
+- `EMAIL_ADDRESS`: Email address for sending notifications
+- `EMAIL_PASSWORD`: Email password for authentication
+- `EMAIL_RECIPIENT`: Email address to receive notifications
+
 ### IAM Policy File Format
 
 The IAM policy file should follow this YAML structure:
@@ -98,9 +122,28 @@ Each user entry includes:
 3. **Role Processing**: Processes all roles while filtering out conditional bindings
 4. **Comparison**: Compares current permissions with expected permissions from the policy file
 5. **Reporting**: Generates detailed reports of any discrepancies found
+6. **Notification**: Sends notifications via GitHub issues and/or email when using announce action
+
+The `print` action can be used for testing notification content without actually creating GitHub issues or sending emails.
 
 Command-line arguments take precedence over configuration file settings.
 
+## GitHub Actions Integration
+
+The enforcement tools are integrated with GitHub Actions to provide automated compliance monitoring. The workflow is configured to run weekly and automatically create GitHub issues and send email notifications for any policy violations.
+
+### Workflow Configuration
+
+The GitHub Actions workflow (`.github/workflows/beam_Infrastructure_PolicyEnforcer.yml`) runs:
+- **Schedule**: Weekly on Mondays at 9:00 AM UTC
+- **Manual trigger**: Can be triggered manually via `workflow_dispatch`
+- **Actions**: Runs both IAM and Account Keys enforcement with the `announce` action
+
+**Note**:
+- The email service is configured to use gmail
+- The recipient email is set to `[email protected]` for Apache Beam project notifications
+- The `GITHUB_TOKEN` is automatically provided by GitHub Actions and doesn't need to be configured manually
+
 ## Account Keys
 
 The enforcement is also done by validating service account keys and their access permissions against the defined policies.
@@ -114,8 +157,11 @@ You can specify the action either through the configuration file (`config.yml`)
 # Check compliance and report issues (default)
 python account_keys.py --action check
 
-# Create GitHub issue if compliance violations are found
-python account_keys.py --action issue
+# Create/update GitHub issue and send email if compliance violations are found
+python account_keys.py --action announce
+
+# Print announcement details for testing purposes (no actual issue created)
+python account_keys.py --action print
 
 # Generate new compliance file based on current service account keys policy
 python account_keys.py --action generate
@@ -124,7 +170,8 @@ python account_keys.py --action generate
 ### Actions
 
 - **check**: Validates service account keys and their permissions against defined policies and reports any differences (default behavior)
-- **issue**: Creates a GitHub issue when service account keys policies differ from the defined ones
+- **announce**: Creates or updates a GitHub issue and sends an email notification when service account keys policies differ from the defined ones. If no open issue exists, creates a new one; if an open issue exists, updates the issue body with current violations
+- **print**: Prints announcement details for testing purposes without creating actual GitHub issues or sending emails
 - **generate**: Updates the compliance file to match the current GCP service account keys and Secret Manager permissions
 
 ### Features
@@ -143,9 +190,21 @@ The `config.yml` file supports the following parameters for account keys:
 
 - `project_id`: GCP project ID to check
 - `service_account_keys_file`: Path to the YAML file containing expected service account keys policies (default: `../keys/keys.yaml`)
-- `action`: Default action to perform (`check`, `issue`, or `generate`)
+- `action`: Default action to perform (`check`, `announce`, `print`, or `generate`)
 - `logging`: Logging configuration (level and format)
 
+### Environment Variables (for announce action)
+
+When using the `announce` action, the following environment variables are required:
+
+- `GITHUB_TOKEN`: GitHub personal access token for creating issues
+- `GITHUB_REPOSITORY`: Repository in format `owner/repo` (default: `apache/beam`)
+- `SMTP_SERVER`: SMTP server for email notifications
+- `SMTP_PORT`: SMTP port (default: 587)
+- `EMAIL_ADDRESS`: Email address for sending notifications
+- `EMAIL_PASSWORD`: Email password for authentication
+- `EMAIL_RECIPIENT`: Email address to receive notifications
+
 ### Service Account Keys File Format
 
 The service account keys file should follow this YAML structure:

infra/enforcement/README.md

@@ -32,6 +32,7 @@ service_account_keys_file: ../keys/keys.yaml
 # Action to perform when running the script
 # Options:
 #   - check: Check compliance and report issues (default)
-#   - issue: Create GitHub issue if compliance violations are found
+#   - announce: Create/update GitHub issue and send email if compliance violations are found
+#   - print: Print announcement details for testing purposes
 #   - generate: Generate new compliance file based on current IAM policy
-action: check
+action: announce

infra/enforcement/config.yml

@@ -21,3 +21,4 @@ google-cloud-iam==2.19.0
 google-cloud-resource-manager==1.14.1
 google-cloud-secret-manager==2.24.0
 google-crc32c==1.7.1
+requests==2.32.4