Skip to content
This repository was archived by the owner on Aug 30, 2024. It is now read-only.

Commit 21a52c7

Browse files
airMengVincyZhang
airMeng
authored and
VincyZhang
committed
reorg directory
migrate CI Signed-off-by: Hengyu Meng <[email protected]> refine CI for neuralspeed Signed-off-by: Wenxin Zhang <[email protected]> add more CI scripts Signed-off-by: Wenxin Zhang <[email protected]> minor fix Signed-off-by: Wenxin Zhang <[email protected]> remove runner.name when running on ubuntu-latest Signed-off-by: Wenxin Zhang <[email protected]> update CI to share system Signed-off-by: Wenxin Zhang <[email protected]> rename jblas tp bestla directory reorg Signed-off-by: Hengyu Meng <[email protected]> remove itrex dependency Signed-off-by: Hengyu Meng <[email protected]> fix script path\n remove python dependency Signed-off-by: Hengyu Meng <[email protected]> -s remove python tests disable percentage disable monitor Signed-off-by: Hengyu Meng <[email protected]> fix naming fix threadpool conflict Signed-off-by: Hengyu Meng <[email protected]> restore percentage Signed-off-by: Hengyu Meng <[email protected]>
1 parent ace447c commit 21a52c7

File tree

1,171 files changed

+35337
-158290
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,171 files changed

+35337
-158290
lines changed

‎.github/pull_request_template.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
## Type of Change
2+
3+
feature or bug fix or documentation or others
4+
API changed or not
5+
6+
## Description
7+
8+
detail description
9+
Issues: xxx
10+
11+
## Expected Behavior & Potential Risk
12+
13+
the expected behavior that triggered by this PR
14+
15+
## How has this PR been tested?
16+
17+
how to reproduce the test (including hardware information)
18+
19+
## Dependency Change?
20+
21+
any library dependency introduced or removed

‎.github/workflows/copyright_check.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
name: Copyright Check
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
paths:
7+
- neural_speed/**
8+
- setup.py
9+
- .github/workflows/format_scan.yml
10+
workflow_dispatch:
11+
12+
# If there is a new commit, the previous jobs will be canceled
13+
concurrency:
14+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
15+
cancel-in-progress: true
16+
17+
env:
18+
CODE_SCAN_LOG_PATH: "${{ github.workspace }}/log"
19+
CONTAINER_NAME: "codeScan"
20+
21+
jobs:
22+
format-scan:
23+
runs-on: ubuntu-latest
24+
strategy:
25+
matrix:
26+
job_name: ["copyright"]
27+
fail-fast: false
28+
steps:
29+
30+
- name: Checkout out Repo
31+
uses: actions/checkout@v3
32+
33+
- name: CopyRight check
34+
run: |
35+
source ${{ github.workspace }}/.github/workflows/scripts/change_color.sh
36+
set -e
37+
mkdir -p ${{ env.CODE_SCAN_LOG_PATH }}
38+
supported_extensions=(py, sh, yaml)
39+
git fetch
40+
git --no-pager diff --name-only remotes/origin/${{ github.base_ref }} ${{ github.workspace }}/neural_speed> ${{ env.CODE_SCAN_LOG_PATH }}/diff.log
41+
files=$(cat ${{ env.CODE_SCAN_LOG_PATH }}/diff.log | awk '!a[$0]++')
42+
$LIGHT_PURPLE && echo " ----------------- checking ... --------------------------" && $RESET
43+
if [[ -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log ]]; then
44+
rm -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
45+
fi
46+
for file in ${files}
47+
do
48+
if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then
49+
if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then
50+
echo ${file} >> ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
51+
$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------"
52+
cat ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
53+
$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET
54+
$BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET
55+
fi
56+
else
57+
$LIGHT_PURPLE && echo "Skipping ${file}" && $RESET
58+
fi
59+
done
60+
if [[ -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log ]]; then
61+
$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------"
62+
cat ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
63+
$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET
64+
$BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET && exit 1
65+
fi
66+
67+
- name: Publish pipeline artifact
68+
if: ${{ failure() }}
69+
uses: actions/upload-artifact@v3
70+
with:
71+
name: ${{ matrix.job_name }}
72+
path: ${{ env.CODE_SCAN_LOG_PATH }}.*

‎.github/workflows/cpp-graph-test.yml‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
name: CPP Graph Test
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
paths:
7+
- '.github/workflows/cpp-graph-test.yml'
8+
- '.github/workflows/scripts/models/cpp_graph_inference.sh'
9+
- 'neural_speed/**'
10+
- 'bestla/**'
11+
workflow_dispatch:
12+
inputs:
13+
compiler_version:
14+
description: 'compiler_version'
15+
required: false
16+
type: string
17+
default: '13.1.0'
18+
models:
19+
description: 'models (in json)'
20+
required: false
21+
type: string
22+
default: '["llama-2-7b-chat", "gptj-6b"]'
23+
runner:
24+
description: 'runner'
25+
required: false
26+
type: string
27+
default: 'spr'
28+
29+
# If there is a new commit, the previous jobs will be canceled
30+
concurrency:
31+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
32+
cancel-in-progress: true
33+
34+
env:
35+
OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/scripts/models
36+
SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/scripts
37+
WORKING_DIR: ${{ github.workspace }}
38+
CONTAINER_NAME: "codeScan"
39+
INPUT_COMPILER_VERSION: ${{ inputs.compiler_version || '13.1.0' }}
40+
41+
jobs:
42+
CPP-Graph-Workflow:
43+
runs-on: ${{inputs.runner || 'spr'}}
44+
strategy:
45+
matrix:
46+
modelName: ${{fromJson(inputs.models || '["llama-2-7b-chat", "gptj-6b"]')}}
47+
steps:
48+
- name: Checkout out Repo
49+
uses: actions/checkout@v3
50+
with:
51+
submodules: "recursive"
52+
fetch-tags: true
53+
54+
- name: Env build
55+
run: |
56+
bash ${{ github.workspace }}/.github/workflows/scripts/prepare_env_with_conda.sh "cpp-graph-test" "3.8"
57+
58+
- name: Binary build
59+
if: 0 == 1
60+
run: |
61+
cd ${{ github.workspace }}
62+
conda activate cpp-graph-test || source activate cpp-graph-test
63+
pip install build --upgrade
64+
pip install -r requirements.txt
65+
python setup.py sdist bdist_wheel
66+
pip install dist/neuralspeed*.whl
67+
pip list
68+
69+
- name: BF16 Benchmark
70+
run: |
71+
cd ${{ github.workspace }}/.github/workflows/scripts/models
72+
bash cpp_graph_inference.sh cpp-graph-test ${{ matrix.modelName }} ${{ env.INPUT_COMPILER_VERSION }}
73+
74+
- name: Rename summary
75+
run: |
76+
cd ${{ github.workspace }}
77+
cp cpp_graph_summary.log cpp_graph_summary_${{matrix.modelName}}.log
78+
79+
- name: Publish pipeline artifact
80+
uses: actions/upload-artifact@v3
81+
if: ${{ !cancelled() }}
82+
with:
83+
name: cpp_graph
84+
path: ${{ github.workspace }}/cpp_graph_summary_${{matrix.modelName}}.log
85+
if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
86+
retention-days: 60 # 1 <= retention-days <= 90
87+
88+
Genreate-Report:
89+
runs-on: ubuntu-latest
90+
needs: [CPP-Graph-Workflow]
91+
steps:
92+
- name: Docker Clean Up
93+
run: |
94+
docker ps -a
95+
if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then
96+
docker start ${{ env.CONTAINER_NAME }}-${{ runner.name }}
97+
echo "remove left files through container ..."
98+
docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /neural-speed && rm -fr /neural-speed/* && rm -fr /neural-speed/.* || true"
99+
fi
100+
- name: Checkout out Repo
101+
uses: actions/checkout@v3
102+
103+
- name: Download Summary Log
104+
uses: actions/download-artifact@v3
105+
with:
106+
path: ${{ env.OUT_SCRIPT_PATH }}/generated/log
107+
108+
- name: Merge CPP Graph Summary Log
109+
run: |
110+
cd ${{ env.OUT_SCRIPT_PATH }}/generated/log/cpp_graph
111+
for summary in $(find . -name "cpp_graph_summary_*.log"); do cat $summary >> cpp_graph_summary.log; done
112+
113+
- name: Download Reference Artifact
114+
id: download-artifact
115+
uses: dawidd6/action-download-artifact@v2
116+
with:
117+
workflow: cpp-graph-test.yml
118+
name: FinalReport
119+
run_id: ${{ vars.GRAPH_REF_ID }}
120+
path: ${{ env.OUT_SCRIPT_PATH }}
121+
name_is_regexp: true
122+
repo: ${{ github.repository }}
123+
check_artifacts: false
124+
search_artifacts: false
125+
skip_unpack: false
126+
if_no_artifact_found: warn
127+
128+
- name: Display structure of downloaded files
129+
run: cd ${{ env.OUT_SCRIPT_PATH }} && ls -R
130+
131+
- name: Generate report
132+
run: |
133+
echo "------ Generating final report.html ------"
134+
cd ${{ env.OUT_SCRIPT_PATH }}
135+
/usr/bin/bash generate_report.sh --workflow=deploy
136+
sed -n '/<body>/,/<\/body>/p' generated/report.html | sed -r '/^$/d' | sed -r 's/^ +//g' >> $GITHUB_STEP_SUMMARY
137+
env:
138+
RUN_DISPLAY_URL: https://github.com/neural-speed/actions/runs/${{ github.run_id }}
139+
BUILD_NUMBER: ${{ github.run_id }}
140+
JOB_STATUS: succeed
141+
MR_source_branch: ${{ github.head_ref }}
142+
ghprbActualCommit: ${{ github.event.pull_request.head.sha }}
143+
144+
- name: Publish Report
145+
uses: actions/upload-artifact@v3
146+
if: ${{ !cancelled() }}
147+
with:
148+
name: FinalReport
149+
path: ${{ env.OUT_SCRIPT_PATH }}/generated
150+
151+
- name: Specify performance regression
152+
run: |
153+
if [ $(is_perf_reg) == 'true' ]; then
154+
echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports."
155+
exit 1
156+
fi

deprecated/test/kernels/benchmark/dynamic_quant_matmul/CMakeLists.txt renamed to .github/workflows/docker/codeScan.dockerfile

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#
12
# Copyright (c) 2022 Intel Corporation
23
#
34
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +13,29 @@
1213
# See the License for the specific language governing permissions and
1314
# limitations under the License.
1415

15-
file(GLOB DYNAMIC_QUANT_MATMUL ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
16-
set(SRCS ${SRCS} ${DYNAMIC_QUANT_MATMUL} PARENT_SCOPE)
16+
ARG UBUNTU_VER=22.04
17+
FROM ubuntu:${UBUNTU_VER} as devel
18+
19+
# See http://bugs.python.org/issue19846
20+
ENV LANG C.UTF-8
21+
22+
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
23+
aspell \
24+
aspell-en \
25+
python3 \
26+
python3-pip \
27+
python3-dev \
28+
python3-distutils \
29+
build-essential \
30+
cloc \
31+
python3.10-venv \
32+
git
33+
34+
RUN ln -sf $(which python3) /usr/bin/python
35+
36+
RUN python -m pip install --no-cache-dir pylint==2.17.5\
37+
bandit==1.7.4\
38+
pyspelling\
39+
pydocstyle
1740

41+
WORKDIR /

‎.github/workflows/docker/devel.dockerfile‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#
2+
# Copyright (c) 2022 Intel Corporation
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
ARG UBUNTU_VER=22.04
16+
FROM ubuntu:${UBUNTU_VER} as devel
17+
18+
# See http://bugs.python.org/issue19846
19+
ENV LANG C.UTF-8
20+
21+
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
22+
python3 \
23+
python3-pip \
24+
python3-dev \
25+
python3-distutils \
26+
autoconf \
27+
build-essential \
28+
git \
29+
libgl1-mesa-glx \
30+
libglib2.0-0 \
31+
numactl \
32+
time \
33+
wget \
34+
bc \
35+
gawk \
36+
jq \
37+
python3.10-venv \
38+
vim
39+
40+
RUN ln -sf $(which python3) /usr/bin/python
41+
42+
RUN python -m pip --no-cache-dir install --upgrade pip
43+
RUN python -m pip install --no-cache-dir setuptools
44+
45+
RUN pip list
46+
47+
WORKDIR /
48+

0 commit comments

Comments
 (0)