From 2efe1220d0128d3f2d371c9ed5234c4978d73a77 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Thu, 13 May 2021 01:19:37 -0700 Subject: [PATCH] Create linux-ci.yml (#13) * Create linux-ci.yml * Update linux-ci.yml * cancelling and python-package workflows + fixes to linux-ci workflow * Update linux-ci.yml * Update linux-ci.yml * Update linux-ci.yml * Update linux-ci.yml * Update linux-ci.yml * add_feature for Python3_EXECUTABLE * Update linux-ci.yml * rpath - enable CMAKE_INSTALL_RPATH_USE_LINK_PATH - fix to Kokkos_INSTALL_LIBDIR * Update linux-ci.yml * print Python3_SITEARCH * Update CMakeLists.txt * Python3_EXE and PYTHON_EXE sync * Fix to cmake-build install * Update KokkosPythonPackages.cmake * Update CMakeLists.txt * Update linux-ci.yml * Update linux-ci.yml * Update linux-ci.yml * removed some stuff from pyproject.toml --- .github/workflows/cancelling.yml | 20 +++ .github/workflows/linux-ci.yml | 173 +++++++++++++++++++++++ .github/workflows/python-package.yml | 39 +++++ CMakeLists.txt | 17 ++- README.md | 7 +- cmake/Modules/KokkosPythonOptions.cmake | 2 + cmake/Modules/KokkosPythonPackages.cmake | 15 ++ pyproject.toml | 2 - 8 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/cancelling.yml create mode 100644 .github/workflows/linux-ci.yml create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/cancelling.yml b/.github/workflows/cancelling.yml new file mode 100644 index 0000000..d83e27a --- /dev/null +++ b/.github/workflows/cancelling.yml @@ -0,0 +1,20 @@ +name: cancel-builds-on-update +on: + workflow_run: + workflows: ['linux-ci', 'python-package'] + types: ['requested'] + +jobs: + cancel-duplicate-workflow-runs: + name: "Cancel duplicate workflow runs" + runs-on: ubuntu-latest + steps: + - uses: potiuk/cancel-workflow-runs@master + name: "Cancel duplicate workflow runs" + with: + cancelMode: duplicates + cancelFutureDuplicates: true + token: ${{ secrets.GITHUB_TOKEN }} + sourceRunId: ${{ github.event.workflow_run.id }} + notifyPRCancel: true + skipEventTypes: '["schedule"]' diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml new file mode 100644 index 0000000..237984e --- /dev/null +++ b/.github/workflows/linux-ci.yml @@ -0,0 +1,173 @@ +name: linux-ci + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +env: + BUILD_TYPE: Release + +jobs: + cmake-build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update container + run: + sudo apt-get update && + sudo apt-get install -y build-essential gcc g++ && + python -m pip install --upgrade pip + + - name: Install Kokkos + run: + python -m pip install 'cmake==3.18.4' && + git clone https://github.com/kokkos/kokkos.git /tmp/kokkos-source && + cmake -B /tmp/kokkos-build + -DKokkos_ENABLE_SERIAL=ON + -DKokkos_ENABLE_OPENMP=ON + -DBUILD_SHARED_LIBS=ON + /tmp/kokkos-source && + cmake --build /tmp/kokkos-build --target all --parallel 2 && + sudo cmake --build /tmp/kokkos-build --target install --parallel 2 + + - name: Configure CMake + run: + python -m pip install -r requirements.txt && + cmake -B ${{github.workspace}}/build + -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + -DENABLE_LAYOUTS=OFF + -DENABLE_EXAMPLES=ON + -DENABLE_WERROR=ON + -DPython3_EXECUTABLE=$(which python) + + - name: Build + run: + cmake --build ${{github.workspace}}/build --target all --parallel 2 && + sudo cmake --build ${{github.workspace}}/build --target install --parallel 2 + + - name: Import Test + working-directory: ${{github.workspace}}/build + run: + mkdir scratch && + cd scratch && + export PYTHONPATH=/usr/local/lib/python${{ matrix.python-version }}/site-packages:${PYTHONPATH} && + python -c "import kokkos; print(kokkos.__file__)" + + - name: Test + working-directory: ${{github.workspace}}/build + run: + export PYTHONPATH=${PWD}:/usr/local/lib/python${{ matrix.python-version }}/site-packages:${PYTHONPATH} && + ls && + python ./ex-numpy.py + + python-build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update container + run: + sudo apt-get update && + sudo apt-get install -y build-essential gcc g++ && + python -m pip install --upgrade pip + + - name: Install Kokkos + run: + python -m pip install 'cmake==3.18.4' && + git clone https://github.com/kokkos/kokkos.git /tmp/kokkos-source && + cmake -B /tmp/kokkos-build + -DKokkos_ENABLE_SERIAL=ON + -DKokkos_ENABLE_OPENMP=ON + -DCMAKE_CXX_STANDARD=17 + -DBUILD_SHARED_LIBS=ON + /tmp/kokkos-source && + cmake --build /tmp/kokkos-build --target all --parallel 2 && + sudo cmake --build /tmp/kokkos-build --target install --parallel 2 + + - name: Build + run: + python -m pip install -r requirements.txt && + python setup.py install + --enable-layouts + --disable-memory-traits + --enable-werror + -- -DENABLE_EXAMPLES=ON + + - name: Import Test + run: + mkdir scratch && + cd scratch && + python -c "import kokkos; print(kokkos.__file__)" + + - name: Test + working-directory: ${{github.workspace}}/_skbuild + run: + cd ./*/cmake-build && + ls && + python ./ex-numpy.py + + pip-build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update container + run: + sudo apt-get update && + sudo apt-get install -y build-essential gcc g++ && + python -m pip install --upgrade pip + + - name: Install Kokkos + run: + python -m pip install 'cmake==3.18.4' && + git clone https://github.com/kokkos/kokkos.git /tmp/kokkos-source && + cmake -B /tmp/kokkos-build + -DKokkos_ENABLE_SERIAL=ON + -DKokkos_ENABLE_PTHREAD=ON + -DCMAKE_CXX_STANDARD=17 + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + /tmp/kokkos-source && + cmake --build /tmp/kokkos-build --target all --parallel 2 && + sudo cmake --build /tmp/kokkos-build --target install --parallel 2 + + - name: Build + run: + python -m pip install -r requirements.txt && + PYKOKKOS_BASE_SETUP_ARGS="-DENABLE_WERROR=ON -DENABLE_MEMORY_TRAITS=OFF -DCMAKE_CXX_STANDARD=17" + python -m pip install -v --user --no-deps -e . + + - name: Import Test + run: + mkdir scratch && + cd scratch && + python -c "import kokkos; print(kokkos.__file__)" diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..8e35c0c --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,39 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: python-package + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=90 --statistics + #- name: Test with pytest + # run: | + # pytest diff --git a/CMakeLists.txt b/CMakeLists.txt index d21971b..09e482f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,27 +118,34 @@ SET_TARGET_PROPERTIES(libpykokkos PROPERTIES IF(SKBUILD) SET(Kokkos_INSTALL_PYTHONDIR ${CMAKE_INSTALL_PREFIX}) - SET(Kokkos_INSTALL_LIBDIR ${CMAKE_INSTALL_PREFIX}/kokkos) + SET(Kokkos_INSTALL_LIBDIR ${CMAKE_INSTALL_PREFIX}/kokkos) ELSE() SET(Kokkos_INSTALL_PYTHONDIR ${Python3_SITEARCH}/kokkos) - SET(Kokkos_INSTALL_LIBDIR ${Kokkos_INSTALL_PYTHONDIR}) + SET(Kokkos_INSTALL_LIBDIR ${Python3_SITEARCH}/kokkos) ENDIF() # figure out if we can install to Python3_SITEARCH EXECUTE_PROCESS( - COMMAND ${CMAKE_COMMAND} -E touch ${Python3_SITEARCH}/kokkos/__init__.py + COMMAND ${CMAKE_COMMAND} -E touch ${Python3_SITEARCH}/.__kokkos__init__.py WORKING_DIRECTORY ${PROJECT_BINARY_DIR} ERROR_VARIABLE ERR_MSG RESULT_VARIABLE ERR_CODE) +ADD_FEATURE(Python3_SITEARCH "Python site-packages directory") + IF(ERR_CODE AND NOT SKBUILD) # get the python directory name, e.g. 'python3.6' from # '/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6' get_filename_component(PYDIR "${Python3_STDLIB}" NAME) # Should not be CMAKE_INSTALL_LIBDIR! Python won't look in a lib64 folder set(Kokkos_INSTALL_PYTHONDIR lib/${PYDIR}/site-packages/kokkos) + set(Kokkos_INSTALL_LIBDIR lib/${PYDIR}/site-packages/kokkos) ENDIF() +EXECUTE_PROCESS( + COMMAND ${CMAKE_COMMAND} -E rm ${Python3_SITEARCH}/.__kokkos__init__.py + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + INSTALL(TARGETS libpykokkos DESTINATION ${Kokkos_INSTALL_LIBDIR}) @@ -158,8 +165,10 @@ FOREACH(_FILE ${PYPACKAGE_FILES}) SET(_OUT_FILE ${PROJECT_BINARY_DIR}/${_OUT_PATH}/${_OUT_NAME}.py) # put version, python interpreter, etc. in the file for reference CONFIGURE_FILE(${_FILE} ${_OUT_FILE} @ONLY) + # patch duplicated subfolder + STRING(REPLACE "kokkos/kokkos" "kokkos" _OUT_PATH "${Kokkos_INSTALL_PYTHONDIR}/${_OUT_PATH}") # install to the correct folder structure - INSTALL(FILES ${_OUT_FILE} DESTINATION ${Kokkos_INSTALL_PYTHONDIR}/${_OUT_PATH}) + INSTALL(FILES ${_OUT_FILE} DESTINATION ${_OUT_PATH}) ENDFOREACH() # build the examples, not designed to be built stand-alone diff --git a/README.md b/README.md index 7d78f1c..804d7b2 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,10 @@ pip install pykokkos-base --install-option={--,-DENABLE_LAYOUTS=ON,-DENABLE_MEMO ### Overview This example is designed to emulate a work-flow where the user has code using Kokkos in C++ and writes python bindings to those functions. A python script is used as the `"main"`: - - `ex-numpy.py` imports the kokkos bindings - - Calls a routine in the "users" python bindings to a C++ function which returns a `Kokkos::View` - - This view is then converted to a numpy array in python and printed via the numpy capabilities. + +- `ex-numpy.py` imports the kokkos bindings +- Calls a routine in the "users" python bindings to a C++ function which returns a `Kokkos::View` +- This view is then converted to a numpy array in python and printed via the numpy capabilities. ### Files diff --git a/cmake/Modules/KokkosPythonOptions.cmake b/cmake/Modules/KokkosPythonOptions.cmake index 3a82c6e..efa4739 100644 --- a/cmake/Modules/KokkosPythonOptions.cmake +++ b/cmake/Modules/KokkosPythonOptions.cmake @@ -11,6 +11,8 @@ IF("${CMAKE_BUILD_TYPE}" STREQUAL "") SET(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) ENDIF() +SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON CACHE BOOL "Install with rpath") + # backwards compat IF(NOT DEFINED ENABLE_EXAMPLES) SET(BUILD_EXAMPLES OFF CACHE BOOL "(deprecated) Use ENABLE_EXAMPLES") diff --git a/cmake/Modules/KokkosPythonPackages.cmake b/cmake/Modules/KokkosPythonPackages.cmake index 0eba1e4..2c1148b 100644 --- a/cmake/Modules/KokkosPythonPackages.cmake +++ b/cmake/Modules/KokkosPythonPackages.cmake @@ -3,8 +3,22 @@ # INCLUDE(KokkosPythonUtilities) +# synchronize Python3_EXECUTABLE and PYTHON_EXECUTABLE +IF(Python3_EXECUTABLE) + SET(PYTHON_EXECUTABLE ${Python3_EXECUTABLE}) +ELSEIF(PYTHON_EXECUTABLE) + SET(Python3_EXECUTABLE ${PYTHON_EXECUTABLE}) +ENDIF() + # basically just used to get Python3_SITEARCH for installation FIND_PACKAGE(Python3 REQUIRED COMPONENTS Interpreter Development) + +FOREACH(_VAR MAJOR MINOR PATCH STRING) + IF(Python3_VERSION_${_VAR}) + SET(PYTHON_VERSION_${_VAR} ${Python3_VERSION_${_VAR}}) + ENDIF() +ENDFOREACH() + IF(NOT PYTHON_VERSION_STRING) IF(PYTHON_VERSION_MAJOR AND PYTHON_VERSION_MINOR AND PYTHON_VERSION_PATCH) SET(PYTHON_VERSION_STRING "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}.${PYTHON_VERSION_PATCH}" CACHE STRING "Python version" FORCE) @@ -15,6 +29,7 @@ ENDIF() SET(PYBIND11_PYTHON_VERSION "${PYTHON_VERSION_STRING}" CACHE STRING "Python version" FORCE) ADD_FEATURE(PYBIND11_PYTHON_VERSION "Python version used by PyBind11") ADD_FEATURE(PYTHON_VERSION_STRING "Python version found") +ADD_FEATURE(Python3_EXECUTABLE "Python interpreter") # python binding library IF(ENABLE_INTERNAL_PYBIND11) diff --git a/pyproject.toml b/pyproject.toml index 45d4eb5..9dd7ff2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,5 @@ [build-system] requires = [ - "setuptools>=42", - "wheel", "scikit-build", "cmake>=3.16", ]