Skip to content

Commit 799748e

Browse files
tarun-googletarun-google
committed
Merge branch 'improve_iceberg_blueprint' of github.com:tarun-google/beam into improve_iceberg_blueprint
2 parents 85270f4 + b200c66 commit 799748e

File tree

118 files changed

+7384
-1120
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

118 files changed

+7384
-1120
lines changed

‎.editorconfig

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Licensed under the Apache License, Version 2.0 (the "License");
2+
# you may not use this file except in compliance with the License.
3+
# You may obtain a copy of the License at
4+
#
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
#
7+
# Unless required by applicable law or agreed to in writing, software
8+
# distributed under the License is distributed on an "AS IS" BASIS,
9+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
# See the License for the specific language governing permissions and
11+
# limitations under the License.
12+
# EditorConfig is awesome: https://EditorConfig.org
13+
14+
# top-most EditorConfig file
15+
root = true
16+
17+
[*]
18+
indent_style = space
19+
indent_size = 2
20+
end_of_line = lf
21+
charset = utf-8
22+
trim_trailing_whitespace = true
23+
insert_final_newline = true
24+
25+
[*.{go,mod,sum}]
26+
indent_style = tab
27+
indent_size = unset
28+
29+
[Dockerfile]
30+
indent_size = 4

‎.github/trigger_files/beam_PostCommit_Python.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run.",
3-
"modification": 33
3+
"modification": 27
44
}
55

‎.github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Dataflow.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"https://github.com/apache/beam/pull/35951": "triggering sideinput test"
3+
}

‎.github/trigger_files/beam_PostCommit_SQL.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run ",
3-
"modification": 3
3+
"modification": 4
44
}

‎.github/trigger_files/beam_PostCommit_XVR_Direct.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"modification": 4
2+
"modification": 5
33
}

‎.github/trigger_files/beam_PostCommit_XVR_Flink.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"modification": 1
2+
"modification": 2
33
}

‎.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"comment": "Modify this file in a trivial way to cause this test suite to run",
3+
"modification": 1
4+
}

‎.github/workflows/beam_Infrastructure_PolicyEnforcer.yml

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# This workflow works with the infrastructure policy enforcer to
19+
# generate a report of IAM and Service Account Policies violations
20+
21+
name: Infrastructure Policy Enforcer
22+
23+
on:
24+
workflow_dispatch:
25+
schedule:
26+
# Once a week at 9:00 AM on Monday
27+
- cron: '0 9 * * 1'
28+
29+
# This allows a subsequently queued workflow run to interrupt previous runs
30+
concurrency:
31+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
32+
cancel-in-progress: true
33+
34+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
35+
permissions:
36+
contents: read
37+
issues: write
38+
39+
jobs:
40+
beam_Infrastructure_PolicyEnforcer:
41+
name: Check and Report Infrastructure Policies Violations
42+
runs-on: [self-hosted, ubuntu-20.04, main]
43+
timeout-minutes: 30
44+
steps:
45+
- uses: actions/checkout@v4
46+
47+
- name: Setup Python
48+
uses: actions/setup-python@v4
49+
with:
50+
python-version: '3.13'
51+
52+
- name: Install Python dependencies
53+
working-directory: ./infra/enforcement
54+
run: |
55+
python -m pip install --upgrade pip
56+
pip install -r requirements.txt
57+
58+
- name: Setup gcloud
59+
uses: google-github-actions/setup-gcloud@v2
60+
61+
- name: Run IAM Policy Enforcement
62+
working-directory: ./infra/enforcement
63+
env:
64+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
65+
GITHUB_REPOSITORY: ${{ github.repository }}
66+
SMTP_SERVER: smtp.gmail.com
67+
SMTP_PORT: 465
68+
EMAIL_ADDRESS: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
69+
EMAIL_PASSWORD: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
70+
EMAIL_RECIPIENT: "[email protected]"
71+
run: python iam.py --action print
72+
73+
- name: Run Account Keys Policy Enforcement
74+
working-directory: ./infra/enforcement
75+
env:
76+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
77+
GITHUB_REPOSITORY: ${{ github.repository }}
78+
SMTP_SERVER: smtp.gmail.com
79+
SMTP_PORT: 465
80+
EMAIL_ADDRESS: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
81+
EMAIL_PASSWORD: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
82+
EMAIL_RECIPIENT: "[email protected]"
83+
run: python account_keys.py --action print

‎.github/workflows/beam_Infrastructure_SecurityLogging.yml

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# This workflow works with the GCP security log analyzer to
19+
# generate weekly security reports and initialize log sinks
20+
21+
name: GCP Security Log Analyzer
22+
23+
on:
24+
workflow_dispatch:
25+
schedule:
26+
# Once a week at 9:00 AM on Monday
27+
- cron: '0 9 * * 1'
28+
push:
29+
paths:
30+
- 'infra/security/config.yml'
31+
32+
# This allows a subsequently queued workflow run to interrupt previous runs
33+
concurrency:
34+
group: '${{ github.workflow }} @ ${{ github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.sender.login }}'
35+
cancel-in-progress: true
36+
37+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
38+
permissions:
39+
contents: read
40+
41+
jobs:
42+
beam_GCP_Security_LogAnalyzer:
43+
name: GCP Security Log Analysis
44+
runs-on: [self-hosted, ubuntu-20.04, main]
45+
timeout-minutes: 30
46+
steps:
47+
- uses: actions/checkout@v4
48+
49+
- name: Setup Python
50+
uses: actions/setup-python@v4
51+
with:
52+
python-version: '3.13'
53+
54+
- name: Install Python dependencies
55+
working-directory: ./infra/security
56+
run: |
57+
python -m pip install --upgrade pip
58+
pip install -r requirements.txt
59+
60+
- name: Setup gcloud
61+
uses: google-github-actions/setup-gcloud@v2
62+
63+
- name: Initialize Log Sinks
64+
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
65+
working-directory: ./infra/security
66+
run: python log_analyzer.py --config config.yml initialize
67+
68+
- name: Generate Weekly Security Report
69+
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
70+
working-directory: ./infra/security
71+
env:
72+
SMTP_SERVER: smtp.gmail.com
73+
SMTP_PORT: 465
74+
EMAIL_ADDRESS: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_ADDRESS }}
75+
EMAIL_PASSWORD: ${{ secrets.ISSUE_REPORT_SENDER_EMAIL_PASSWORD }}
76+
EMAIL_RECIPIENT: "[email protected]"
77+
run: python log_analyzer.py --config config.yml generate-report --dry-run

‎.github/workflows/beam_UserRoles.yml renamed to ‎.github/workflows/beam_Infrastructure_UsersPermissions.yml

File renamed without changes.

0 commit comments

Comments
 (0)