Skip to content

Commit 31677c5

Browse files
seemetherefacebook-github-bot
seemethere
authored and
facebook-github-bot
committed
[reland] .github: Add initial linux CI workflow (pytorch#56280)
Summary: This reverts commit 6b5ed5e. There'll also probably be fixes here, see diff from original PR: pytorch/pytorch@f2abce0...ci-all/add-initial-linux-ci-gha Pull Request resolved: pytorch#56280 Reviewed By: walterddr Differential Revision: D27826012 Pulled By: seemethere fbshipit-source-id: 71cad1d7f840ede5025b1bb4a33d628aa74686d1
1 parent 0917061 commit 31677c5

File tree

13 files changed

+676
-14
lines changed

13 files changed

+676
-14
lines changed

‎.github/scripts/generate_linux_ci_workflows.py‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
#!/usr/bin/env python
2+
3+
from pathlib import Path
4+
5+
import jinja2
6+
7+
DOCKER_REGISTRY = "308535385114.dkr.ecr.us-east-1.amazonaws.com"
8+
9+
GITHUB_DIR = Path(__file__).parent.parent
10+
11+
CPU_TEST_RUNNER = "linux.2xlarge"
12+
CUDA_TEST_RUNNER = "linux.8xlarge.nvidia.gpu"
13+
14+
15+
class PyTorchLinuxWorkflow:
16+
def __init__(self, build_environment: str, docker_image_base: str):
17+
self.build_environment = build_environment
18+
self.docker_image_base = docker_image_base
19+
self.test_runner_type = CPU_TEST_RUNNER
20+
if "cuda" in build_environment:
21+
self.test_runner_type = CUDA_TEST_RUNNER
22+
23+
def generate_workflow_file(
24+
self, workflow_template: jinja2.Template, jinja_env: jinja2.Environment
25+
) -> Path:
26+
output_file_path = GITHUB_DIR.joinpath(
27+
f"workflows/{self.build_environment}.yml"
28+
)
29+
with open(output_file_path, "w") as output_file:
30+
output_file.write(
31+
workflow_template.render(
32+
build_environment=self.build_environment,
33+
docker_image_base=self.docker_image_base,
34+
test_runner_type=self.test_runner_type
35+
)
36+
)
37+
output_file.write('\n')
38+
return output_file_path
39+
40+
41+
WORKFLOWS = [
42+
PyTorchLinuxWorkflow(
43+
build_environment="pytorch-linux-xenial-py3.6-gcc5.4",
44+
docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
45+
),
46+
# PyTorchLinuxWorkflow(
47+
# build_environment="pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4",
48+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
49+
# ),
50+
# PyTorchLinuxWorkflow(
51+
# build_environment="pytorch-parallelnative-linux-xenial-py3.6-gcc5.4",
52+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
53+
# ),
54+
# PyTorchLinuxWorkflow(
55+
# build_environment="pytorch-pure_torch-linux-xenial-py3.6-gcc5.4",
56+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
57+
# ),
58+
# PyTorchLinuxWorkflow(
59+
# build_environment="pytorch-linux-xenial-py3.6-gcc7",
60+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc7",
61+
# ),
62+
# PyTorchLinuxWorkflow(
63+
# build_environment="pytorch-linux-xenial-py3-clang5-asan",
64+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-asan",
65+
# ),
66+
# PyTorchLinuxWorkflow(
67+
# build_environment="pytorch-linux-xenial-py3-clang7-onnx",
68+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang7-onnx",
69+
# ),
70+
# PyTorchLinuxWorkflow(
71+
# build_environment="pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
72+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
73+
# ),
74+
# PyTorchLinuxWorkflow(
75+
# build_environment="pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
76+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
77+
# ),
78+
# PyTorchLinuxWorkflow(
79+
# build_environment="pytorch-libtorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
80+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
81+
# ),
82+
# PyTorchLinuxWorkflow(
83+
# build_environment="pytorch-linux-bionic-py3.6-clang9-noarch",
84+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9",
85+
# ),
86+
# PyTorchLinuxWorkflow(
87+
# build_environment="pytorch-xla-linux-bionic-py3.6-clang9",
88+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9",
89+
# ),
90+
# PyTorchLinuxWorkflow(
91+
# build_environment="pytorch-vulkan-linux-bionic-py3.6-clang9",
92+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9",
93+
# ),
94+
# PyTorchLinuxWorkflow(
95+
# build_environment="pytorch-linux-bionic-py3.8-gcc9-coverage",
96+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.8-gcc9",
97+
# ),
98+
# PyTorchLinuxWorkflow(
99+
# build_environment="pytorch-linux-bionic-rocm3.9-py3.6",
100+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-rocm3.9-py3.6",
101+
# ),
102+
# PyTorchLinuxWorkflow(
103+
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32",
104+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
105+
# ),
106+
# PyTorchLinuxWorkflow(
107+
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64",
108+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
109+
# ),
110+
# PyTorchLinuxWorkflow(
111+
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a",
112+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
113+
# ),
114+
# PyTorchLinuxWorkflow(
115+
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a",
116+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
117+
# ),
118+
# PyTorchLinuxWorkflow(
119+
# build_environment="pytorch-linux-xenial-py3-clang5-mobile",
120+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-asan",
121+
# ),
122+
# PyTorchLinuxWorkflow(
123+
# build_environment="pytorch-linux-xenial-py3-clang5-mobile-custom-dynamic",
124+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
125+
# ),
126+
# PyTorchLinuxWorkflow(
127+
# build_environment="pytorch-linux-xenial-py3-clang5-mobile-custom-static",
128+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
129+
# ),
130+
# PyTorchLinuxWorkflow(
131+
# build_environment="pytorch-linux-xenial-py3-clang5-mobile-code-analysis",
132+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
133+
# ),
134+
# PyTorchLinuxWorkflow(
135+
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32",
136+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
137+
# ),
138+
# PyTorchLinuxWorkflow(
139+
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64",
140+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
141+
# ),
142+
# PyTorchLinuxWorkflow(
143+
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a",
144+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
145+
# ),
146+
# PyTorchLinuxWorkflow(
147+
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a",
148+
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
149+
# ),
150+
]
151+
152+
if __name__ == "__main__":
153+
jinja_env = jinja2.Environment(
154+
variable_start_string="!{{",
155+
loader=jinja2.FileSystemLoader(str(GITHUB_DIR.joinpath("templates"))),
156+
)
157+
workflow_template = jinja_env.get_template("linux_ci_workflow.yml.in")
158+
for workflow in WORKFLOWS:
159+
print(
160+
workflow.generate_workflow_file(
161+
workflow_template=workflow_template,
162+
jinja_env=jinja_env
163+
)
164+
)

‎.github/scripts/install_nvidia_utils_linux.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/usr/bin/env bash
2+
3+
set -eou pipefail
4+
5+
DISTRIBUTION=$(. /etc/os-release;echo $ID$VERSION_ID) \
6+
DRIVER_FN="NVIDIA-Linux-x86_64-460.39.run"
7+
YUM_REPO_URL="https://nvidia.github.io/nvidia-docker/${DISTRIBUTION}/nvidia-docker.repo"
8+
9+
install_nvidia_docker2_amzn2() {
10+
(
11+
set -x
12+
# Needed for yum-config-manager
13+
sudo yum install -y yum-utils
14+
sudo yum-config-manager --add-repo "${YUM_REPO_URL}"
15+
sudo yum install -y nvidia-docker2
16+
sudo systemctl restart docker
17+
)
18+
}
19+
20+
install_nvidia_driver() {
21+
(
22+
set -x
23+
sudo yum groupinstall -y "Development Tools"
24+
curl -fsL -o nvidia_driver "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
25+
sudo /bin/bash nvidia_driver -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
26+
nvidia-smi
27+
)
28+
}
29+
30+
# Install container toolkit based on distribution
31+
echo "== Installing nvidia container toolkit for ${DISTRIBUTION} =="
32+
case "${DISTRIBUTION}" in
33+
amzn*)
34+
install_nvidia_docker2_amzn2
35+
;;
36+
*)
37+
echo "ERROR: Unknown distribution ${DISTRIBUTION}"
38+
exit 1
39+
;;
40+
esac
41+
42+
echo "== Installing nvidia driver ${DRIVER_FN} =="
43+
install_nvidia_driver

‎.github/scripts/report_git_status.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
CHANGES=$(git status --porcelain)
3+
echo "$CHANGES"
4+
git diff
5+
[ -z "$CHANGES" ]

0 commit comments

Comments
 (0)