Skip to content

Commit

Permalink
ZAL: ZK Accel Layer (#308)
Browse files Browse the repository at this point in the history 
* zal: export MSM and threadpool to C DLL

* fix LTO spurious warnings when building library

* fix appType staticLib vs staticlib

* fix threadpool

* shorter EC FFI

* zal: now compiles with halo2curves and can call Nim

* zal: Pass tests

* zal: add benches Halo2curves vs Constantine

* CI: add constantine-rust / ZAL to CI

* CI: can't use variables for shell

* CI: display rust version

* Rust: force clang for building (Rust regression following #309 removal of rustBuild in .nimble)

* LLVM (hence llvm-config) isn't installed in Github Actions CI

* documentation of compile options

* reduce rust verbosity

* cargo in CI, spray and pray

* don't update Rust twice

* if using clang, Nim expects llvm-ar for static libraries archiver

* Expose full LLVM on MacOS, no Rust for i386, CPU features

* fix HW info

* the light at the end of the CI tunnel

* ensure autoloading kernel32.dll threading API

* Keep stack guard pages on Windows

* MacOS CI: some agents don't support ADX instructions and SIGILL
  • Loading branch information
@mratsim
mratsim authored Dec 4, 2023
1 parent a810179 commit 78159b5
Show file tree
Hide file tree
Showing 53 changed files with 2,011 additions and 1,011 deletions.
4 changes: 2 additions & 2 deletions .cargo/config.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,6 @@
# if the base LLVM versions differ.
# For example before LLVM 15 opaque pointers in IR triggered an error.

[build]
# https://doc.rust-lang.org/rustc/linker-plugin-lto.html
# [build]
# # https://doc.rust-lang.org/rustc/linker-plugin-lto.html
# rustflags="-Clinker-plugin-lto -Clinker=clang -Clink-arg=-fuse-ld=lld"
121 changes: 92 additions & 29 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ jobs:
max-parallel: 20
matrix:
nim_version: [version-1-6] # [version-1-4, devel]
rust_toolchain: [stable] # [beta, nightly]
target:
- os: linux
cpu: i386
Expand Down Expand Up @@ -157,10 +158,12 @@ jobs:
run: |
sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
--no-install-recommends -yq \
libgmp-dev
libgmp-dev \
llvm
- name: Install test dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
# We don't install LLVM as the Rust libraries that call Constantine are 64-bit only.
run: |
sudo dpkg --add-architecture i386
sudo apt-fast update -qq
Expand Down Expand Up @@ -198,18 +201,20 @@ jobs:

# Furthermore, Apple Clang can delete symbols when building a static library
# in particular the hasAdxImpl bool for CPU runtime detection.
run: |
mkdir -p external/bin
cat << EOF > external/bin/clang
#!/bin/bash
exec $(brew --prefix llvm@15)/bin/clang "\$@"
EOF
cat << EOF > external/bin/clang++
#!/bin/bash
exec $(brew --prefix llvm@15)/bin/clang++ "\$@"
EOF
chmod 755 external/bin/{clang,clang++}
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH

# run: |
# mkdir -p external/bin
# cat << EOF > external/bin/clang
# #!/bin/bash
# exec $(brew --prefix llvm@15)/bin/clang "\$@"
# EOF
# cat << EOF > external/bin/clang++
# #!/bin/bash
# exec $(brew --prefix llvm@15)/bin/clang++ "\$@"
# EOF
# chmod 755 external/bin/{clang,clang++}
# echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
run: echo "$(brew --prefix llvm@15)/bin" >> $GITHUB_PATH

- name: Setup MSYS2 (Windows)
if: runner.os == 'Windows'
Expand All @@ -223,7 +228,7 @@ jobs:
if: runner.os == 'Windows'
shell: msys2 {0}
run: |
pacman -S --needed --noconfirm mingw-w64-x86_64-gmp
pacman -S --needed --noconfirm mingw-w64-x86_64-gmp mingw-w64-x86_64-llvm
nimble refresh --verbose -y
nimble install --verbose -y gmp jsony asynctools [email protected]
Expand All @@ -234,50 +239,108 @@ jobs:
nimble refresh --verbose -y
nimble install --verbose -y gmp jsony asynctools [email protected]
- name: Print Nim & compiler versions
- name: Install Rust
shell: bash
# gcc is an alias to Clang on MacOS
run: rustup update ${{ matrix.rust_toolchain }} && rustup default ${{ matrix.rust_toolchain }}

- name: Print Nim, Rust, LLVM versions and CPU specs.
shell: bash
# gcc is an alias to Apple Clang on MacOS
run: |
nim -v
gcc -v
clang -v
rustup --version
if [[ '${{ matrix.target.cpu }}' != 'i386' && '${{ runner.os }}' != 'Windows' ]]; then
llvm-config --version
fi
if [[ '${{ runner.os }}' == 'Linux' ]]; then
cat /proc/cpuinfo
fi
if [[ '${{ runner.os }}' == 'macOS' ]]; then
sysctl -a | grep machdep.cpu
sysctl -a | grep hw | grep cpu
sysctl -a | grep hw.optional
fi
- name: Run Constantine tests (UNIX with Assembly)
- name: Run Constantine as C library tests (UNIX with Assembly)
if: runner.os != 'Windows' && matrix.target.BACKEND == 'ASM'
shell: bash
run: |
cd constantine
nimble make_lib --verbose
nimble make_headers --verbose
nimble test_lib --verbose
- name: Run Constantine as C library tests (UNIX no Assembly)
if: runner.os != 'Windows' && matrix.target.BACKEND == 'NO_ASM'
shell: bash
run: |
cd constantine
CTT_ASM=0 nimble make_lib --verbose
nimble make_headers --verbose
nimble test_lib --verbose
- name: Run Constantine as C library tests (Windows with Assembly)
# So "test_bindings" uses C and can find GMP
# but nim-gmp cannot find GMP on Windows CI
if: runner.os == 'Windows' && matrix.target.BACKEND == 'ASM'
shell: msys2 {0}
run: |
cd constantine
nimble make_lib --verbose
nimble make_headers --verbose
nimble test_lib --verbose
- name: Run Constantine as C library tests (Windows no Assembly)
# So "test_bindings" uses C and can find GMP
# but nim-gmp cannot find GMP on Windows CI
if: runner.os == 'Windows' && matrix.target.BACKEND == 'NO_ASM'
shell: msys2 {0}
run: |
cd constantine
CTT_ASM=0 nimble make_lib --verbose
nimble make_headers --verbose
nimble test_lib --verbose
- name: Run Constantine as Rust library tests (with Assembly)
if: matrix.target.BACKEND == 'ASM' && matrix.target.cpu != 'i386'
shell: bash
# We need to deactivate the assembly (used by default for benches)
run: |
cd constantine
cargo test --no-default-features halo2curves --features halo2curves/multicore
- name: Run Constantine as Rust library tests (NO Assembly)
if: matrix.target.BACKEND == 'NO_ASM' && matrix.target.cpu != 'i386'
shell: bash
# We need to deactivate the assembly (used by default for benches)
run: |
cd constantine
CTT_ASM=0 cargo test --no-default-features halo2curves --features halo2curves/multicore
- name: Run Constantine in-depth tests (Unix - with GMP, with Assembly)
if: runner.os != 'Windows' && matrix.target.BACKEND == 'ASM'
shell: bash
run: |
cd constantine
nimble test_parallel --verbose
- name: Run Constantine tests (UNIX no Assembly)
- name: Run Constantine in-depth tests (Unix - with GMP, no Assembly)
if: runner.os != 'Windows' && matrix.target.BACKEND == 'NO_ASM'
shell: bash
run: |
cd constantine
CTT_ASM=0 nimble make_lib --verbose
nimble make_headers --verbose
nimble test_lib --verbose
CTT_ASM=0 nimble test_parallel --verbose
- name: Run Constantine tests (Windows with Assembly)
- name: Run Constantine in-depth tests (Windows - no GMP, with Assembly)
# So "test_bindings" uses C and can find GMP
# but nim-gmp cannot find GMP on Windows CI
if: runner.os == 'Windows' && matrix.target.BACKEND == 'ASM'
shell: msys2 {0}
run: |
cd constantine
nimble make_lib --verbose
nimble make_headers --verbose
nimble test_lib --verbose
nimble test_parallel_no_gmp --verbose
- name: Run Constantine tests (Windows no Assembly)
- name: Run Constantine in-depth tests (Windows - no GMP, no Assembly)
# So "test_bindings" uses C and can find GMP
# but nim-gmp cannot find GMP on Windows CI
if: runner.os == 'Windows' && matrix.target.BACKEND == 'NO_ASM'
shell: msys2 {0}
run: |
cd constantine
CTT_ASM=0 nimble make_lib --verbose
nimble make_headers --verbose
nimble test_lib --verbose
CTT_ASM=0 nimble test_parallel_no_gmp --verbose
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ build/
*.so.*
*.dylib
*.a
*.lib
*.la
*.dll
*.exe
Expand Down
14 changes: 9 additions & 5 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@
resolver = "2"
members = [
"constantine-rust/constantine-sys",
"constantine-rust/ctt-curve-bls12-381",
"constantine-rust/ctt-curve-bn254-snarks",
"constantine-rust/ctt-curve-pasta",
"constantine-rust/ctt-proto-ethereum-bls-signatures",
"constantine-rust/constantine-zal-halo2kzg",
]

# If Nim static library is compiled with Clang ThinLTO, enable it on Rust side
Expand All @@ -26,4 +23,11 @@ members = [
# lto = "thin"
#
# [profile.bench]
# lto = "thin"
# lto = "thin"

[profile.bench]
opt-level = 3
debug = false
debug-assertions = false
overflow-checks = false
codegen-units = 1
2 changes: 1 addition & 1 deletion bindings/c_curve_decls.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ template genBindings_EC_ShortW_Affine*(ECP, Field: untyped) =

{.pop.}

template genBindings_EC_ShortW_NonAffine*(ECP, ECP_Aff, Field: untyped) =
template genBindings_EC_ShortW_NonAffine*(ECP, ECP_Aff: untyped) =
when appType == "lib":
{.push noconv, dynlib, exportc, raises: [].} # No exceptions allowed
else:
Expand Down
55 changes: 55 additions & 0 deletions bindings/c_curve_decls_parallel.nim
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Constantine
# Copyright (c) 2018-2019 Status Research & Development GmbH
# Copyright (c) 2020-Present Mamy André-Ratsimbazafy
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.

import
../constantine/math/config/curves,
../constantine/curves_primitives_parallel,
../constantine/platforms/allocs,
../constantine/threadpool

export curves_primitives_parallel

template genParallelBindings_EC_ShortW_NonAffine*(ECP, ECP_Aff, ScalarField: untyped) =
# For some unknown reason {.push noconv.}
# would overwrite the threadpool {.nimcall.}
# in the parallel for-loop `generateClosure`
#
# Similarly, exportc breaks the threadpool generated closure
# hence instead of push/pop we create a pragma alias

when appType == "lib":
{.pragma: libExport, dynlib, exportc, raises: [].} # No exceptions allowed
else:
{.pragma: libExport, exportc, raises: [].} # No exceptions allowed

# --------------------------------------------------------------------------------------
proc `ctt _ ECP _ multi_scalar_mul_big_coefs_vartime_parallel`(
tp: Threadpool,
r: var ECP,
coefs: ptr UncheckedArray[BigInt[ECP.F.C.getCurveOrderBitwidth()]],
points: ptr UncheckedArray[ECP_Aff],
len: csize_t) {.libExport.} =
tp.multiScalarMul_vartime_parallel(r.addr, coefs, points, cast[int](len))

proc `ctt _ ECP _ multi_scalar_mul_fr_coefs_vartime_parallel`(
tp: Threadpool,
r: var ECP,
coefs: ptr UncheckedArray[ScalarField],
points: ptr UncheckedArray[ECP_Aff],
len: csize_t) {.libExport.} =

let n = cast[int](len)
let coefs_fr = allocHeapArrayAligned(matchingOrderBigInt(ECP.F.C), n, alignment = 64)

syncScope:
tp.parallelFor i in 0 ..< n:
captures: {coefs, coefs_fr}
coefs_fr[i].fromField(coefs[i])
tp.multiScalarMul_vartime_parallel(r.addr, coefs_fr, points, n)

freeHeapAligned(coefs_fr)
Loading

0 comments on commit 78159b5

Please sign in to comment.