Python bindings using scikit build core

.github/workflows/python-cibuildwheel-pybind.yml

@@ -0,0 +1,171 @@
+name: Python Pybind cibuildwheel
+
+on:
+  push:
+    branches: [master, release-*]
+  pull_request:
+    branches: [master]
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: pybind11-cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+        # separate archs, so they use individual caches
+        arch: ["x86_64", "arm64"]
+        flavor: ["cpython", "pypy"]
+        # separate musl and many on linux, for mac we just skip one of those
+        target: ["many", "musl"]
+        exclude:
+          - os: ubuntu-22.04
+            target: musl
+            flavor: pypy
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }}
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python
+
+      - name: Sets env for x86_64
+        run: |
+          echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV
+          echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV
+        if: matrix.arch == 'x86_64'
+
+      - name: Sets env for arm64
+        run: |
+          echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV
+          echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV
+        if: matrix.arch == 'arm64'
+
+      - name: Skip manylinux for musllinux target
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }}
+        run: |
+          echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV
+
+      - name: Skip musllinux for manylinux target
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }}
+        run: |
+          echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV
+
+      - name: Skip pypy for cpython
+        if: ${{ matrix.flavor == 'cpython' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV
+
+      - name: Skip cpython for pypy
+        if: ${{ matrix.flavor == 'pypy' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV
+
+      - name: install mac dependencies
+        if: ${{ runner.os == 'macOS' }}
+        # 2nd command: workaround https://github.com/actions/setup-python/issues/577
+        run: |
+          brew update && \
+          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \
+          brew install ccache
+
+      - name: install mac dependencies X86_64
+        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }}
+        run: |
+          brew update && \
+          brew install zlib snappy boost
+
+      - name: install mac dependencies arm64
+        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }}
+        run: |
+          set -e
+          echo "MACOSX_DEPLOYMENT_TARGET=12.3.0" >> $GITHUB_ENV
+          echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV
+          echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV
+          mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew
+          PACKAGES=( icu4c xz lz4 zstd zlib snappy boost )
+          for PACKAGE in "${PACKAGES[@]}"
+          do
+            response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_sonoma $PACKAGE | grep Downloaded )
+            download_path=$(echo $response | xargs -n 1 | tail -1)
+            arm64-homebrew/bin/brew reinstall -vd $download_path
+          done
+          arm64-homebrew/bin/brew config
+          ls /Users/runner/work/keyvi/keyvi/arm64-homebrew
+
+      - name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }}
+        uses: pypa/[email protected]
+        env:
+          # Skip CPython 3.6 and CPython 3.7
+          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* pp37-*
+
+          # skip testing all python versions on linux arm, only test 3.12
+          # skip tests on pypy, currently fails for indexer tests
+          CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*"
+
+          # (many)linux custom docker images
+          CIBW_MANYLINUX_X86_64_IMAGE: "keyvidev/manylinux-builder-x86_64"
+          CIBW_MANYLINUX_AARCH64_IMAGE: "keyvidev/manylinux-builder-aarch64"
+          CIBW_MUSLLINUX_X86_64_IMAGE: "keyvidev/musllinux-builder-x86_64"
+          CIBW_MUSLLINUX_AARCH64_IMAGE: "keyvidev/musllinux-builder-aarch64"
+
+          # ccache using path
+          CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH
+          CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf
+
+          # for debugging set this to 1,2 or 3
+          # CIBW_BUILD_VERBOSITY: 2
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.flavor }}-${{ matrix.target }}
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: sdist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: install Linux deps
+        run: |
+          sudo apt-get update && \
+          sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache ninja-build pipx
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ matrix.os }}-sdist-python
+
+      - name: Build SDist
+        run: |
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+          pipx run build --sdist
+          python -m pip install dist/*.tar.gz -v && \
+          python -m pip install pytest && \
+          python -m pytest python-pybind/tests && \
+          python -m pip uninstall -y keyvi_pybind11
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-sdist
+          path: dist/*.tar.gz
+
+  upload_all:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: artifact-*
+          merge-multiple: true
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.pypi_password }}

.gitignore

@@ -31,7 +31,7 @@
 *.orig
 
 # cmake build dir
-build/*
+/*build*
 */cmake-build-debug/*
 build_dir_debug/
 cmake-build-debug/
@@ -45,3 +45,11 @@ cmake-build-debug/
 
 # vim swap files
 *.swp
+
+# python
+*.egg-info
+
+# pybind build folder
+python*/*build*
+python*/dist
+python*/.cache/

CMakeLists.txt

@@ -1,19 +1,30 @@
-cmake_minimum_required(VERSION 3.9)
+cmake_minimum_required(VERSION 3.21)
 project(keyvi)
 
 #### Build Type
 if (CMAKE_BUILD_TYPE)
   string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER)
 endif()
 
+#### Options
+
+option(KEYVI_C_BINDINGS "Keyvi: Build C binding" ${PROJECT_IS_TOP_LEVEL})
+option(KEYVI_PYTHON_BINDINGS "Keyvi: Build Python module" OFF)
+option(KEYVI_TESTS "Keyvi: Build unit tests" ${PROJECT_IS_TOP_LEVEL})
+option(KEYVI_BINARIES "Keyvi: Build Python module" ${PROJECT_IS_TOP_LEVEL})
+option(KEYVI_CLANG_TIDY "Keyvi: Build with clang tidy" ${PROJECT_IS_TOP_LEVEL})
+option(KEYVI_DOCS "Keyvi: Build docs" ${PROJECT_IS_TOP_LEVEL})
+
 #### Linting
-find_program(CLANGTIDY clang-tidy)
-if(CLANGTIDY)
-  message ("-- Found clang-tidy")
-  set(CMAKE_CXX_CLANG_TIDY clang-tidy; --extra-arg-before=-std=c++17)
-else()
-  message ("-- clang-tidy not found")
-endif()
+if(KEYVI_CLANG_TIDY)
+  find_program(CLANGTIDY clang-tidy)
+  if(CLANGTIDY)
+    message ("-- Found clang-tidy")
+    set(CMAKE_CXX_CLANG_TIDY clang-tidy; --extra-arg-before=-std=c++17)
+  else()
+    message ("-- clang-tidy not found")
+  endif()
+endif(KEYVI_CLANG_TIDY)
 
 #### Cmake modules
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/")
@@ -141,67 +152,73 @@ string(REPLACE " " ";" _KEYVI_COMPILE_DEFINITIONS_LIST "${_KEYVI_COMPILE_DEFINIT
 
 #### Targets ####
 
-# keyvicompiler
-add_executable(keyvicompiler keyvi/bin/keyvicompiler/keyvicompiler.cpp)
-target_link_libraries(keyvicompiler ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(keyvicompiler PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(keyvicompiler PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(keyvicompiler PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+if(KEYVI_BINARIES)
+  # keyvicompiler
+  add_executable(keyvicompiler keyvi/bin/keyvicompiler/keyvicompiler.cpp)
+  target_link_libraries(keyvicompiler ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(keyvicompiler PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(keyvicompiler PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(keyvicompiler PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
 
-install (TARGETS keyvicompiler DESTINATION bin COMPONENT applications OPTIONAL)
+  install (TARGETS keyvicompiler DESTINATION bin COMPONENT applications OPTIONAL)
 
-# keyviinspector
-add_executable(keyviinspector keyvi/bin/keyviinspector/keyviinspector.cpp)
-target_link_libraries(keyviinspector ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(keyviinspector PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(keyviinspector PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(keyviinspector PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+  # keyviinspector
+  add_executable(keyviinspector keyvi/bin/keyviinspector/keyviinspector.cpp)
+  target_link_libraries(keyviinspector ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(keyviinspector PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(keyviinspector PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(keyviinspector PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
 
-install (TARGETS keyviinspector DESTINATION bin COMPONENT applications OPTIONAL)
+  install (TARGETS keyviinspector DESTINATION bin COMPONENT applications OPTIONAL)
 
-# keyvimerger
-add_executable(keyvimerger keyvi/bin/keyvimerger/keyvimerger.cpp)
-target_link_libraries(keyvimerger ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(keyvimerger PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(keyvimerger PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(keyvimerger PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+  # keyvimerger
+  add_executable(keyvimerger keyvi/bin/keyvimerger/keyvimerger.cpp)
+  target_link_libraries(keyvimerger ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(keyvimerger PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(keyvimerger PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(keyvimerger PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
 
-install (TARGETS keyvimerger DESTINATION bin COMPONENT applications)
+  install (TARGETS keyvimerger DESTINATION bin COMPONENT applications)
+endif(KEYVI_BINARIES)
 
 # keyvi_c
-add_library(keyvi_c SHARED keyvi/bin/keyvi_c/c_api.cpp)
-target_link_libraries(keyvi_c ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(keyvi_c PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(keyvi_c PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(keyvi_c PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+if(KEYVI_C_BINDINGS)
+  add_library(keyvi_c SHARED keyvi/bin/keyvi_c/c_api.cpp)
+  target_link_libraries(keyvi_c ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(keyvi_c PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(keyvi_c PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(keyvi_c PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+endif(KEYVI_C_BINDINGS)
 
 # unit tests
-FILE(GLOB_RECURSE UNIT_TEST_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} keyvi/tests/keyvi/*.cpp)
-add_executable(unit_test_all ${UNIT_TEST_SOURCES})
-target_link_libraries(unit_test_all ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(unit_test_all PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(unit_test_all PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(unit_test_all PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
-add_dependencies(unit_test_all keyvimerger)
-
-if (WIN32)
-  message(STATUS "zlib: ${ZLIB_LIBRARY_RELEASE}")
-  # copies the dlls required to run to the build folder
-  foreach(LIB ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY_RELEASE} ${Boost_FILESYSTEM_LIBRARY_RELEASE} ${ZLIB_LIBRARY_RELEASE})
-    get_filename_component(UTF_BASE_NAME ${LIB} NAME_WE)
-    get_filename_component(UTF_PATH ${LIB} PATH)
-	if(EXISTS "${UTF_PATH}/${UTF_BASE_NAME}.dll")
-      add_custom_command(TARGET unit_test_all POST_BUILD
-        COMMAND ${CMAKE_COMMAND} -E copy "${UTF_PATH}/${UTF_BASE_NAME}.dll" ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}
-        )
-	# zlib might be stored in a different folder
-	elseif(EXISTS "${UTF_PATH}/../bin/${UTF_BASE_NAME}.dll")
-      add_custom_command(TARGET unit_test_all POST_BUILD
-        COMMAND ${CMAKE_COMMAND} -E copy "${UTF_PATH}/../bin/${UTF_BASE_NAME}.dll" ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}
-        )	
-	endif()
-  endforeach()
-endif (WIN32)
+if(KEYVI_TESTS)
+  FILE(GLOB_RECURSE UNIT_TEST_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} keyvi/tests/keyvi/*.cpp)
+  add_executable(unit_test_all ${UNIT_TEST_SOURCES})
+  target_link_libraries(unit_test_all ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(unit_test_all PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(unit_test_all PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(unit_test_all PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+  add_dependencies(unit_test_all keyvimerger)
+
+  if (WIN32)
+    message(STATUS "zlib: ${ZLIB_LIBRARY_RELEASE}")
+    # copies the dlls required to run to the build folder
+    foreach(LIB ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY_RELEASE} ${Boost_FILESYSTEM_LIBRARY_RELEASE} ${ZLIB_LIBRARY_RELEASE})
+      get_filename_component(UTF_BASE_NAME ${LIB} NAME_WE)
+      get_filename_component(UTF_PATH ${LIB} PATH)
+    if(EXISTS "${UTF_PATH}/${UTF_BASE_NAME}.dll")
+        add_custom_command(TARGET unit_test_all POST_BUILD
+          COMMAND ${CMAKE_COMMAND} -E copy "${UTF_PATH}/${UTF_BASE_NAME}.dll" ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}
+          )
+    # zlib might be stored in a different folder
+    elseif(EXISTS "${UTF_PATH}/../bin/${UTF_BASE_NAME}.dll")
+        add_custom_command(TARGET unit_test_all POST_BUILD
+          COMMAND ${CMAKE_COMMAND} -E copy "${UTF_PATH}/../bin/${UTF_BASE_NAME}.dll" ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}
+          )	
+    endif()
+    endforeach()
+  endif (WIN32)
+endif(KEYVI_TESTS)
 
 # bindings
 add_custom_target(bindings
@@ -226,10 +243,14 @@ target_include_directories(keyvi INTERFACE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>
 target_compile_definitions(keyvi INTERFACE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
 target_link_libraries(keyvi INTERFACE ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
 
+if (KEYVI_PYTHON_BINDINGS)
+    add_subdirectory(python-pybind)
+endif ()
+
 ### docs
 
 # don't run it as part of a non-toplevel build, e.g. python
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/sphinx-docs)
+if(KEYVI_DOCS)
     find_package(Doxygen)
     find_package(Sphinx COMPONENTS breathe)
 
@@ -255,4 +276,4 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/sphinx-docs)
     else()
         message ("-- Skip doc target, doxygen/sphinx not found")
     endif()
-endif()
+endif(KEYVI_DOCS)

keyvi/check-style.sh

@@ -10,7 +10,7 @@ else
 commit_range="upstream/master...HEAD"
 fi
 
-infiles=`git diff --name-only --diff-filter=ACMRT $(echo ${commit_range} | sed 's/\.//') | grep -v "3rdparty" | grep -E "\.(cpp|h)$"`
+infiles=`git diff --name-only --diff-filter=ACMRT $(echo ${commit_range} | sed 's/\.//') | grep -v "3rdparty" | grep -v "pybind11" | grep -E "\.(cpp|h)$"`
 
 clang_format_files=()
 cpplint_files=()

keyvi/include/keyvi/dictionary/match.h

@@ -58,7 +58,7 @@ namespace dictionary {
 #ifdef Py_PYTHON_H
 class attributes_visitor : public boost::static_visitor<PyObject*> {
  public:
-  PyObject* operator()(int i) const { return PyInt_FromLong(i); }
+  PyObject* operator()(int i) const { return PyLong_FromLong(i); }
 
   PyObject* operator()(double i) const { return PyFloat_FromDouble(i); }