[ci] [build] Containerize Windows CPU build and test (#4933)

* [ci] [build] Containerize Windows CPU build and test * Disable ninja * Avoid pybind11_add_module() * Force reinstall * Find pybind11 * Include pybind11 dir * Update include dir * Remove trailing whitespace * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Use correct pybind11 * Add path * Enable no extras for pybind11_add_module * Add no_extra * Clone in the container * Use github job container * Add runs-on * Revert back to docker based jobs * Install instead of develop * [ci] [build] Containerize Windows CPU build and test * Disable ninja * Avoid pybind11_add_module() * Force reinstall * Find pybind11 * Include pybind11 dir * Update include dir * Remove trailing whitespace * Use correct pybind11 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add path * Enable no extras for pybind11_add_module * Add no_extra * Clone in the container * Use github job container * Add runs-on * Revert back to docker based jobs * Install instead of develop * Use tar in jobs * Update cmake * Skip clone * Manual fixing white space * Remove comments Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
taichi-dev · May 17, 2022 · 31ded06 · 31ded06
1 parent 4078f25
commit 31ded06
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 2 deletions.
diff --git a/.github/workflows/scripts/win_build_test_cpu.ps1 b/.github/workflows/scripts/win_build_test_cpu.ps1
@@ -0,0 +1,72 @@
+# Build script for windows CPU
+# TODO unify this with the other Win scripts
+
+param (
+    [switch]$clone = $false,
+    [switch]$install = $false,
+    [string]$libsDir = "C:\"
+)
+
+$ErrorActionPreference = "Stop"
+
+$RepoURL = 'https://github.com/taichi-dev/taichi'
+
+function WriteInfo($text) {
+    Write-Host -ForegroundColor Green "[BUILD] $text"
+}
+
+$libsDir = (Resolve-Path $libsDir).Path
+if (-not (Test-Path $libsDir)) {
+    New-Item -ItemType Directory -Path $libsDir
+}
+Set-Location $libsDir
+
+if (-not (Test-Path "taichi_llvm")) {
+    WriteInfo("Download and extract LLVM")
+    curl.exe --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
+    7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
+}
+if (-not (Test-Path "taichi_clang")) {
+    WriteInfo("Download and extract Clang")
+    curl.exe --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
+    7z x clang-10.0.0-win.zip -otaichi_clang
+}
+
+WriteInfo("Setting the env vars")
+$env:LLVM_DIR = "C://taichi_llvm"
+
+#TODO enable build test
+$env:TAICHI_CMAKE_ARGS = "-DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_BUILD_TESTS:BOOL=OFF"
+
+#TODO: For now we need to hard code the compiler path from build tools 2019
+$env:TAICHI_CMAKE_ARGS +=' -DCMAKE_CXX_COMPILER=C:/Program\ Files\ (x86)/Microsoft\ Visual\ Studio/2019/BuildTools/vc/Tools/Llvm/x64/bin/clang++.exe -DCMAKE_C_COMPILER=C:/Program\ Files\ (x86)/Microsoft\ Visual\ Studio/2019/BuildTools/vc/Tools/Llvm/x64/bin/clang.exe'
+$env:TAICHI_CMAKE_ARGS += " -DCLANG_EXECUTABLE=C:\\taichi_clang\\bin\\clang++.exe"
+$env:TAICHI_CMAKE_ARGS += " -DLLVM_AS_EXECUTABLE=C:\\taichi_llvm\\bin\\llvm-as.exe -DTI_WITH_VULKAN:BOOL=OFF"
+
+WriteInfo("Checking clang compiler")
+clang --version
+
+WriteInfo("Enter the repository")
+Set-Location .\taichi
+
+WriteInfo("Setting up Python environment")
+conda activate py37
+python -m pip install -r requirements_dev.txt
+python -m pip install -r requirements_test.txt
+
+# These have to be re-installed to avoid strange certificate issue
+# on CPU docker environment
+python -m pip install --upgrade --force-reinstall numpy
+python -m pip install --upgrade --force-reinstall cmake
+python -m pip install --upgrade --force-reinstall wheel
+if (-not $?) { exit 1 }
+
+WriteInfo("Building Taichi")
+python setup.py install
+if (-not $?) { exit 1 }
+WriteInfo("Build finished")
+
+$env:TI_ENABLE_PADDLE = "0"
+WriteInfo("Testing Taichi")
+python tests/run_tests.py -vr2 -t4 -k "not torch and not paddle" -a cpu
+WriteInfo("Test finished")
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -239,6 +239,48 @@ jobs:
           TI_WANTED_ARCHS: ${{ matrix.wanted_archs }}
           TI_CI: 1
 
+  build_and_test_cpu_windows:
+    name: Build and Test Windows (CPU)
+    needs: check_files
+    timeout-minutes: 90
+    runs-on: windows-2019
+    permissions:
+      packages: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+
+      - name: Get docker images
+        shell: bash
+        run: |
+          if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then
+            exit 0
+          fi
+          echo $CR_PAT | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+          docker pull ghcr.io/taichi-dev/taichidev-cpu-windows:v0.0.1
+        env:
+          CR_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and Test
+        shell: bash
+        run: |
+          if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then
+            exit 0
+          fi
+          docker create --name taichi_build_test \
+            ghcr.io/taichi-dev/taichidev-cpu-windows:v0.0.1 \
+            C:/taichi/.github/workflows/scripts/win_build_test_cpu.ps1
+          tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx | docker cp - taichi_build_test:C:/
+          docker start -a taichi_build_test
+
+      - name: clean docker container
+        shell: bash
+        if: always()
+        run: |
+          docker rm taichi_build_test -f
+
   build_and_test_gpu_linux:
     name: Build and Test (GPU)
     needs: check_files

diff --git a/cmake/PythonNumpyPybind11.cmake b/cmake/PythonNumpyPybind11.cmake
@@ -10,7 +10,6 @@ message("    include: ${PYTHON_INCLUDE_DIR}")
 message("    library: ${PYTHON_LIBRARY}")
 message("    numpy include: ${NUMPY_INCLUDE_DIR}")
 
-
 include_directories(${NUMPY_INCLUDE_DIR})
 
 find_package(pybind11 CONFIG REQUIRED)
diff --git a/cmake/TaichiCore.cmake b/cmake/TaichiCore.cmake
@@ -465,7 +465,8 @@ if(NOT TI_EMSCRIPTENED)
     # Cannot compile Python source code with Android, but TI_EXPORT_CORE should be set and
     # Android should only use the isolated library ignoring those source code.
     if (NOT ANDROID)
-        pybind11_add_module(${CORE_WITH_PYBIND_LIBRARY_NAME} ${TAICHI_PYBIND_SOURCE})
+	# NO_EXTRAS is required here to avoid llvm symbol error during build
+	pybind11_add_module(${CORE_WITH_PYBIND_LIBRARY_NAME} NO_EXTRAS ${TAICHI_PYBIND_SOURCE})
     else()
         add_library(${CORE_WITH_PYBIND_LIBRARY_NAME} SHARED)
     endif ()