Skip to content

Commit

Permalink
Added kokkos as a submodule (#19)
Browse files Browse the repository at this point in the history
* Added kokkos as a submodule

- updated README

* Update linux-ci.yml

* Update linux-ci.yml

* Update KokkosPythonKokkos.cmake

* Update KokkosPythonKokkos.cmake

* Install kokkoscore and kokkoscontainers to python lib directory when submodule enabled

* submodule + python import fixes

* user example initialize fix + header reorganization

- fixed initialize in example
- reorganized headers

* Update setup.cfg

* Update CMakeLists.txt

* formatting

* re-defined uniform type + unmanaged support in kokkos.array

- eliminated LayoutRight as implicit
- eliminated mirror_view_type

* Update user.hpp
  • Loading branch information
jrmadsen authored Jun 1, 2021
1 parent d44ac66 commit 550b668
Show file tree
Hide file tree
Showing 39 changed files with 1,171 additions and 726 deletions.
17 changes: 10 additions & 7 deletions .github/workflows/linux-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,12 @@ jobs:
run:
sudo apt-get update &&
sudo apt-get install -y build-essential gcc g++ ninja-build &&
python -m pip install --upgrade pip
python -m pip install --upgrade pip &&
python -m pip install 'cmake==3.18.4'

- name: Install Kokkos
if: ${{ matrix.python-version != 'pypy-3.6' && matrix.python-version != 'pypy-3.7' }}
run:
python -m pip install 'cmake==3.18.4' &&
git clone -b ${{ matrix.kokkos-branch }} https://github.com/kokkos/kokkos.git /tmp/kokkos-source &&
cmake -B /tmp/kokkos-build
-DKokkos_ENABLE_SERIAL=ON
Expand Down Expand Up @@ -94,11 +95,12 @@ jobs:
run:
sudo apt-get update &&
sudo apt-get install -y build-essential gcc g++ ninja-build &&
python -m pip install --upgrade pip
python -m pip install --upgrade pip &&
python -m pip install 'cmake==3.18.4'

- name: Install Kokkos
if: ${{ matrix.python-version != '3.8' && matrix.python-version != '3.9' }}
run:
python -m pip install 'cmake==3.18.4' &&
git clone -b ${{ matrix.kokkos-branch }} https://github.com/kokkos/kokkos.git /tmp/kokkos-source &&
cmake -B /tmp/kokkos-build
-DKokkos_ENABLE_SERIAL=ON
Expand Down Expand Up @@ -153,11 +155,12 @@ jobs:
run:
sudo apt-get update &&
sudo apt-get install -y build-essential gcc g++ ninja-build &&
python -m pip install --upgrade pip
python -m pip install --upgrade pip &&
python -m pip install 'cmake==3.20.2'

- name: Install Kokkos
if: ${{ matrix.python-version != '3.6' && matrix.python-version != '3.7' }}
run:
python -m pip install 'cmake==3.18.4' &&
git clone -b ${{ matrix.kokkos-branch }} https://github.com/kokkos/kokkos.git /tmp/kokkos-source &&
cmake -B /tmp/kokkos-build
-DKokkos_ENABLE_SERIAL=ON
Expand All @@ -172,7 +175,7 @@ jobs:
run:
python -m pip install -r requirements.txt &&
python -m pip install pytest &&
PYKOKKOS_BASE_SETUP_ARGS="-DENABLE_WERROR=ON -DENABLE_MEMORY_TRAITS=ON -DENABLE_LAYOUTS=ON -DCMAKE_CXX_STANDARD=17"
PYKOKKOS_BASE_SETUP_ARGS="-DENABLE_WERROR=ON -DENABLE_MEMORY_TRAITS=ON -DENABLE_LAYOUTS=ON -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_PTHREAD=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=OFF"
python -m pip install -v --user --no-deps -e .

- name: Import Test
Expand Down
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
[submodule "pybind11"]
path = pybind11
url = https://github.com/pybind/pybind11.git
[submodule "external/kokkos"]
path = external/kokkos
url = https://github.com/kokkos/kokkos.git
69 changes: 50 additions & 19 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,10 +1,5 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.11 FATAL_ERROR)

# set this to a known value to we can override the cache if user doesn't specify install dir
IF(NOT SKBUILD)
SET(CMAKE_INSTALL_PREFIX "/kokkos-install-dir" CACHE PATH "Installation directory")
ENDIF()

SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake/Modules ${CMAKE_MODULE_PATH})

INCLUDE(KokkosPythonSetup)
Expand All @@ -14,11 +9,17 @@ PROJECT(
LANGUAGES C CXX
VERSION ${pykokkos-base_VERSION})

FIND_PACKAGE(Kokkos REQUIRED)
INCLUDE(KokkosPythonUtilities) # miscellaneous macros and functions

ADD_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON)
# force to release if not specified
IF("${CMAKE_BUILD_TYPE}" STREQUAL "")
SET(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
ENDIF()

INCLUDE(KokkosPythonKokkos) # find external Kokkos or add submodule
INCLUDE(KokkosPythonFormat) # format target
INCLUDE(KokkosPythonCompilers) # compiler identification
INCLUDE(KokkosPythonUtilities) # miscellaneous macros and functions
INCLUDE(KokkosPythonOptions) # cache options and various variable settings
INCLUDE(KokkosPythonPackages) # Python Interp

Expand All @@ -36,8 +37,9 @@ SET(libpykokkos_SOURCES

SET(libpykokkos_HEADERS
${CMAKE_CURRENT_LIST_DIR}/include/libpykokkos.hpp
# ${CMAKE_CURRENT_LIST_DIR}/include/type_caster.hpp
${CMAKE_CURRENT_LIST_DIR}/include/deep_copy.hpp
${CMAKE_CURRENT_LIST_DIR}/include/concepts.hpp
${CMAKE_CURRENT_LIST_DIR}/include/defines.hpp
${CMAKE_CURRENT_LIST_DIR}/include/common.hpp
${CMAKE_CURRENT_LIST_DIR}/include/traits.hpp
${CMAKE_CURRENT_LIST_DIR}/include/views.hpp
Expand All @@ -47,7 +49,7 @@ PYBIND11_ADD_MODULE(libpykokkos MODULE ${PYBIND11_EXTRAS}
${libpykokkos_SOURCES}
${libpykokkos_HEADERS})

add_subdirectory(src)
ADD_SUBDIRECTORY(src)

# add extra build properties to this target
ADD_LIBRARY(libpykokkos-build-options INTERFACE)
Expand Down Expand Up @@ -113,16 +115,6 @@ ENDIF()
# link to kokkos and the custom build properties
TARGET_LINK_LIBRARIES(libpykokkos PRIVATE Kokkos::kokkos libpykokkos-build-options)

# set the output path to <BINARY_DIR>/kokkos so one
# can test the python import from the build directory
# Really, only LIBRARY_* is needed for Unix but Windows
# builds are weird so just setting all of them
SET_TARGET_PROPERTIES(libpykokkos PROPERTIES
LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos
ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos
RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos
PDB_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos)

IF(SKBUILD)
SET(Kokkos_INSTALL_PYTHONDIR ${CMAKE_INSTALL_PREFIX})
SET(Kokkos_INSTALL_LIBDIR ${CMAKE_INSTALL_PREFIX}/kokkos)
Expand Down Expand Up @@ -153,9 +145,48 @@ EXECUTE_PROCESS(
COMMAND ${CMAKE_COMMAND} -E rm -f ${Python3_SITEARCH}/.__kokkos__init__.py
WORKING_DIRECTORY ${PROJECT_BINARY_DIR})

# location where kokkos libraries are/will be installed
IF(ENABLE_INTERNAL_KOKKOS OR NOT Kokkos_DIR)
SET(_Kokkos_LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
ELSEIF(Kokkos_DIR)
STRING(REGEX REPLACE "/cmake/.*" "" _Kokkos_LIBDIR "${Kokkos_DIR}")
ENDIF()

# absolute path to libpykokkos install
SET(Kokkos_INSTALL_FULL_PYTHONDIR ${Kokkos_INSTALL_LIBDIR})
IF(NOT IS_ABSOLUTE "${Kokkos_INSTALL_FULL_PYTHONDIR}")
SET(Kokkos_INSTALL_FULL_PYTHONDIR ${CMAKE_INSTALL_PREFIX}/${Kokkos_INSTALL_FULL_PYTHONDIR})
ENDIF()

# relative path from libpykokkos install directory to library install directory
FILE(RELATIVE_PATH LIB_RELPATH "${Kokkos_INSTALL_FULL_PYTHONDIR}" "${_Kokkos_LIBDIR}")

# configure the rpath: <RELATIVE>:<CWD>:<FULL>
IF(APPLE)
SET(_INSTALL_RPATH MACOSX_RPATH "@loader_path/${LIB_RELPATH}:@loader_path:${_Kokkos_LIBDIR}:${CMAKE_INSTALL_RPATH}")
ELSEIF(UNIX)
SET(_INSTALL_RPATH INSTALL_RPATH "\$ORIGIN/${LIB_RELPATH}:\$ORIGIN:${_Kokkos_LIBDIR}:${CMAKE_INSTALL_RPATH}")
ENDIF()

# set the output path to <BINARY_DIR>/kokkos so one
# can test the python import from the build directory
# Really, only LIBRARY_* is needed for Unix but Windows
# builds are weird so just setting all of them
SET_TARGET_PROPERTIES(libpykokkos PROPERTIES
LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos
ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos
RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos
PDB_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos
${_INSTALL_RPATH})

CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/pytest.ini
${PROJECT_BINARY_DIR}/pytest.ini COPYONLY)

IF(NOT SKBUILD)
CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/setup.cfg
${PROJECT_BINARY_DIR}/setup.cfg COPYONLY)
ENDIF()

INSTALL(TARGETS libpykokkos
DESTINATION ${Kokkos_INSTALL_LIBDIR})

Expand Down
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ include pytest.ini
recursive-include cmake *
recursive-include kokkos *
recursive-include examples *
recursive-include external *
recursive-include include *
recursive-include src *
recursive-include pybind11 *
Expand Down
77 changes: 69 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,32 @@
# pykokkos-base

> Extended Documentation can be found in [Wiki](https://github.com/kokkos/pykokkos-base/wiki)
This package contains the minimal set of bindings for [Kokkos](https://github.com/kokkos/kokkos)
interoperability with Python:

- `Kokkos::initialize(...)`
- `Kokkos::finalize()`
- `Kokkos::View<...>`
- `Kokkos::DynRankView<...>`
- Free-standing function bindings
- `Kokkos::initialize(...)`
- `Kokkos::finalize()`
- `Kokkos::is_initialized()`
- `Kokkos::deep_copy(...)`
- `Kokkos::create_mirror(...)`
- `Kokkos::create_mirror_view(...)`
- `Kokkos::Tools::profileLibraryLoaded()`
- `Kokkos::Tools::pushRegion(...)`
- `Kokkos::Tools::popRegion()`
- `Kokkos::Tools::createProfileSection(...)`
- `Kokkos::Tools::destroyProfileSection(...)`
- `Kokkos::Tools::startSection(...)`
- `Kokkos::Tools::stopSection(...)`
- `Kokkos::Tools::markEvent(...)`
- `Kokkos::Tools::declareMetadata(...)`
- `Kokkos::Tools::Experimental::set_<...>_callback(...)`
- Data structures
- `Kokkos::View<...>`
- `Kokkos::DynRankView<...>`
- `Kokkos_Profiling_KokkosPDeviceInfo`
- `Kokkos_Profiling_SpaceHandle`

By importing this package in Python, you can pass the supported Kokkos Views and DynRankViews
from C++ to Python and vice-versa. Furthermore, in Python, these bindings provide interoperability
Expand All @@ -31,14 +51,55 @@ In order to write native Kokkos in Python, see [pykokkos](https://github.com/kok

## Installation

You can install this package via CMake or Python's `setup.py`. There are two important cmake options:
You can install this package via CMake or Python's `setup.py`. The important cmake options are:

- `ENABLE_LAYOUTS`
- `ENABLE_MEMORY_TRAITS`
- `ENABLE_VIEW_RANKS` (integer)
- `ENABLE_LAYOUTS` (bool)
- `ENABLE_MEMORY_TRAITS` (bool)
- `ENABLE_INTERNAL_KOKKOS` (bool)

By default, CMake will enable these options if the Kokkos installation was not built with CUDA support.
By default, CMake will enable the layouts and memory traits options if the Kokkos installation was not
built with CUDA support.
If Kokkos was built with CUDA support, these options will be disabled by default due to unreasonable
compilation times (> 1 hour).
The `ENABLE_VIEW_RANKS` option (defaults to a value of 4) is the max number of ranks for
`Kokkos::View<...>` that can be returned to Python. For example, value of 4 means that
views of data type `T*`, `T**`, `T***`, and `T****` can be returned to python but
`T*****` and higher cannot. Increasing this value up to 7 can dramatically increase the length
of time required to compile the bindings.

### Kokkos Installation

If the `ENABLE_INTERNAL_KOKKOS` option is not specified the first time CMake is run, CMake will try to
find an existing Kokkos installation. If no existing installation is found, it will build and install
Kokkos from a submodule. When Kokkos is added as a submodule, you can configure the submodule
as you would normally configure Kokkos. However, due to some general awkwardness configuring cmake
from `setup.py` (especially via `pip install`), CMake tries to "automatically" configure
reasonable default CMake settings for the Kokkos submodule.

Here are the steps when Kokkos is added as a submodule:

- Does `external/kokkos/CMakeLists.txt` exists?
- **YES**: assumes the submodule is already checked out
- > _If compute node does not have internet access, checkout submodule before installing!_
- **NO**: does `.gitmodules` exist?
- **YES**: `git submodule update --init external/kokkos`
- **NO**: `git clone -b master https://github.com/kokkos/kokkos.git external/kokkos`
- Set `BUILD_SHARED_LIBS=ON`
- Set `Kokkos_ENABLE_SERIAL=ON`
- `find_package(OpenMP)`
- Was OpenMP found?
- **YES**: set `Kokkos_ENABLE_OPENMP=ON`
- **NO**: `find_package(Threads)`
- Was Threads found?
- **YES**: set `Kokkos_ENABLE_PTHREADS=ON` (if not Windows)
- `find_package(CUDA)`
- Was CUDA found?
- **YES**: set:
- `Kokkos_ENABLE_CUDA=ON`
- `Kokkos_ENABLE_CUDA_UVM=ON`
- `Kokkos_ENABLE_CUDA_LAMBDA=ON`
- `Kokkos_ENABLE_CUDA_CONSTEXPR=ON`

### Configuring Options via CMake

Expand Down
112 changes: 112 additions & 0 deletions cmake/Modules/KokkosPythonKokkos.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@

#----------------------------------------------------------------------------------------#
# Kokkos submodule
#----------------------------------------------------------------------------------------#

INCLUDE(KokkosPythonUtilities) # miscellaneous macros and functions

# if first time cmake is run and no external/internal preference is specified,
# try to find already installed kokkos
IF(NOT DEFINED ENABLE_INTERNAL_KOKKOS)
FIND_PACKAGE(Kokkos QUIET)
# set the default cache value
IF(Kokkos_FOUND)
SET(_INTERNAL_KOKKOS OFF)
ELSE()
SET(_INTERNAL_KOKKOS ON)
ENDIF()
ELSE()
# make sure ADD_OPTION in KokkosPythonOptions has a value
SET(_INTERNAL_KOKKOS ${ENABLE_INTERNAL_KOKKOS})
ENDIF()

# force an error
IF(NOT _INTERNAL_KOKKOS)
UNSET(FIND_PACKAGE_MESSAGE_DETAILS_Kokkos)
FIND_PACKAGE(Kokkos REQUIRED)
ENDIF()

#
IF(_INTERNAL_KOKKOS)

# try to find some packages quietly in order to set some defaults
SET(OpenMP_FOUND OFF)
SET(Threads_FOUND OFF)
SET(CUDA_FOUND OFF)

IF(NOT Kokkos_ENABLE_PTHREAD)
FIND_PACKAGE(OpenMP QUIET)
ENDIF()

IF(NOT DEFINED Kokkos_ENABLE_PTHREAD AND NOT OpenMP_FOUND)
FIND_PACKAGE(Threads QUIET)
ENDIF()

IF(NOT DEFINED Kokkos_ENABLE_CUDA)
FIND_PACKAGE(CUDA QUIET)
ENDIF()

ADD_OPTION(ENABLE_SERIAL "Enable Serial backend when building Kokkos submodule" ON)
ADD_OPTION(ENABLE_OPENMP "Enable OpenMP when building Kokkos submodule" ${OpenMP_FOUND})
ADD_OPTION(ENABLE_THREADS "Enable Pthreads when building Kokkos submodule" ${Threads_FOUND})
ADD_OPTION(ENABLE_CUDA "Enable CUDA when building Kokkos submodule" ${CUDA_FOUND})

# if OpenMP defaulted to ON but Kokkos_ENABLE_PTHREAD was explicitly set,
# disable OpenMP defaulting to ON
IF(ENABLE_OPENMP AND Kokkos_ENABLE_PTHREAD)
SET(ENABLE_OPENMP OFF)
SET(Kokkos_ENABLE_OPENMP OFF)
ENDIF()

# always disable pthread backend since pthreads are not supported on Windows
IF(WIN32)
SET(ENABLE_THREADS OFF)
SET(Kokkos_ENABLE_PTHREAD OFF)
ENDIF()

# make sure this pykokkos-base option is synced to Kokkos option
IF(DEFINED Kokkos_ENABLE_SERIAL)
SET(ENABLE_SERIAL ${Kokkos_ENABLE_SERIAL})
ENDIF()

# make sure this pykokkos-base option is synced to Kokkos option
IF(DEFINED Kokkos_ENABLE_OPENMP)
SET(ENABLE_OPENMP ${Kokkos_ENABLE_OPENMP})
ENDIF()

# make sure this pykokkos-base option is synced to Kokkos option
IF(DEFINED Kokkos_ENABLE_PTHREAD)
SET(ENABLE_THREADS ${Kokkos_ENABLE_PTHREAD})
ENDIF()

# make sure this pykokkos-base option is synced to Kokkos option
IF(DEFINED Kokkos_ENABLE_CUDA)
SET(ENABLE_CUDA ${Kokkos_ENABLE_CUDA})
ENDIF()

# define the kokkos option as default and/or get it to display
IF(ENABLE_SERIAL)
ADD_OPTION(Kokkos_ENABLE_SERIAL "Build Kokkos submodule with serial support" ON)
ENDIF()

# define the kokkos option as default and/or get it to display
IF(ENABLE_OPENMP)
ADD_OPTION(Kokkos_ENABLE_OPENMP "Build Kokkos submodule with OpenMP support" ON)
ENDIF()

# define the kokkos option as default and/or get it to display
IF(ENABLE_THREADS)
ADD_OPTION(Kokkos_ENABLE_PTHREAD "Build Kokkos submodule with Pthread support" ON)
ENDIF()

# define the kokkos option as default and/or get it to display
IF(ENABLE_CUDA)
ADD_OPTION(Kokkos_ENABLE_CUDA "Build Kokkos submodule with CUDA support" ON)
ADD_OPTION(Kokkos_ENABLE_CUDA_UVM "Build Kokkos submodule with CUDA UVM support" ON)
ADD_OPTION(Kokkos_ENABLE_CUDA_LAMBDA "Build Kokkos submodule with CUDA lambda support" ON)
ADD_OPTION(Kokkos_ENABLE_CUDA_CONSTEXPR "Build Kokkos submodule with CUDA constexpr support" ON)
ENDIF()

ADD_SUBDIRECTORY(external)

ENDIF()
Loading

0 comments on commit 550b668

Please sign in to comment.