Merge branch 'master' into ci/tidier29

erigontech · Jan 2, 2025 · 962e7ed · 962e7ed
2 parents 962a393 + 90a6c09
commit 962e7ed
Show file tree

Hide file tree

Showing 280 changed files with 6,406 additions and 3,904 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -85,8 +85,8 @@ commands:
       - run:
           name: "Install Conan"
           command: |
-            pip3 install --user --no-warn-script-location conan==1.64.1
-            conan_path="$(python -m site --user-base)/bin"
+            pip3 install --user --no-warn-script-location conan==2.10.2
+            conan_path="$(python3 -m site --user-base)/bin"
             echo "export \"PATH=$conan_path:\$PATH\"" >> "$BASH_ENV"
             "$conan_path/conan" --version
 
@@ -109,7 +109,7 @@ commands:
           command: |
             if [[ "<<parameters.compiler_id>>" == "clang" ]]
             then
-                conan_profile=linux_x64_clang_13_release
+                conan_profile=linux_x64_clang_16_release
             else
                 conan_profile=linux_x64_gcc_11_release
             fi
@@ -128,7 +128,7 @@ commands:
           name: "Save Conan cache"
           key: *conan-cache-key
           paths:
-            - ~/.conan
+            - ~/.conan2
 
   build_fuzzer:
     steps:
@@ -146,7 +146,7 @@ commands:
       - run:
           name: "CMake Fuzzer"
           working_directory: ~/build
-          command: cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_13_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON
+          command: cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON
       - run:
           name: "Build Fuzzer"
           command: |
@@ -155,6 +155,9 @@ commands:
 
   test:
     parameters:
+      sanitizer:
+        type: string
+        default: ""
       ethereum_tests:
         type: boolean
         default: true
@@ -164,7 +167,7 @@ commands:
           command: make "SILKWORM_BUILD_DIR=$HOME/build" run_smoke_tests
       - run:
           name: "Unit tests"
-          command: make "SILKWORM_BUILD_DIR=$HOME/build" run_unit_tests
+          command: make "SILKWORM_BUILD_DIR=$HOME/build" SILKWORM_CLANG_COVERAGE=OFF SILKWORM_SANITIZE=<<parameters.sanitizer>> "SILKWORM_PROJECT_DIR=$HOME/project" run_unit_tests
       - when:
           condition: <<parameters.ethereum_tests>>
           steps:
@@ -195,7 +198,7 @@ commands:
 jobs:
   lint:
     machine:
-      image: ubuntu-2204:2023.04.2    
+      image: ubuntu-2404:2024.05.1
     steps:
       - add_ssh_keys:
           fingerprints:
@@ -204,33 +207,11 @@ jobs:
       - run:
           name: "Format"
           working_directory: ~/project
-          command: |
-            sudo pip3 install cmake-format==0.6.13
-            make fmt
-            if ! git diff --exit-code
-            then
-                commit_message="make fmt"
-                head_commit_message="$(git log -1 --pretty=%B)"
-
-                if [[ "$head_commit_message" == "$commit_message" ]]
-                then
-                    echo "The formatting style is not compliant, although it was formatted. Try to run 'make fmt' locally and push the changes."
-                    exit 1
-                else
-                    git config user.name GitHub
-                    git config user.email [email protected]
-                    git commit --all --message="$commit_message"
-                    git config push.autoSetupRemote true
-                    git push
-
-                    echo "The formatting style was not compliant, but it is fixed now. A new workflow will start soon, wait for it..."
-                    exit 2
-                fi
-            fi
+          command: ./tools/lint/ci_format.sh
       - run:
-          name: "Copyright"
+          name: "Lint"
           working_directory: ~/project
-          command: cmake -P cmake/copyright.cmake
+          command: make lint
 
   linux-gcc-thread-sanitizer:
     environment:
@@ -248,7 +229,8 @@ jobs:
           compiler_id: gcc
           compiler_version: <<pipeline.parameters.gcc_version_latest>>
           build_type: Debug
-      - test
+      - test:
+          sanitizer: thread
 
   linux-release:
     parameters:
@@ -288,7 +270,8 @@ jobs:
           build_type: Debug
           compiler_id: clang
           compiler_version: 17
-      - test
+      - test:
+          sanitizer: address,undefined
 
   linux-clang-coverage:
     environment:
@@ -394,7 +377,7 @@ jobs:
     environment:
       WASI_SDK_VERSION: 20
     machine:
-      image: ubuntu-2204:2023.04.2
+      image: ubuntu-2404:2024.05.1
     steps:
       - checkout_with_submodules:
           ethereum_tests: false
@@ -409,7 +392,10 @@ jobs:
           name: "Install dependencies"
           command: |
             sudo apt-get update
-            sudo apt-get install -y texinfo libtinfo5
+            sudo apt-get install -y texinfo
+            # https://askubuntu.com/questions/1531760/how-to-install-libtinfo5-on-ubuntu24-04
+            wget https://security.ubuntu.com/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
+            sudo apt-get install ./libtinfo5_6.3-2ubuntu0.1_amd64.deb
       - install_conan
       - run:
           name: "Install Wasmer"

diff --git a/.github/actions/fuzzer-common-steps/action.yml b/.github/actions/fuzzer-common-steps/action.yml
@@ -23,17 +23,12 @@ runs:
 
     - name: Temporary step - conan cache cleanup - to be executed only once per runner
       shell: bash
-      run: conan remove "*" --force
-
-    - name: Preinstall Conan packages
-      shell: bash
-      working-directory: ${{runner.workspace}}/silkworm
-      run: CC=clang-16 CXX=clang++-16 conan install --install-folder=build/conan --build=missing --profile=cmake/profiles/experimental/linux_x64_clang_16_release .
+      run: /opt/conan2/bin/conan remove --confirm "*"
 
     - name: Configure CMake
       shell: bash
       working-directory: ${{runner.workspace}}/silkworm/build
-      run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=experimental/linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}}
+      run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}}
 
     - name: Build Silkworm Fuzzer Test
       shell: bash

diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml
@@ -47,8 +47,8 @@ jobs:
 
     - name: Install Prerequisites
       run: |
-        pip3 install --user --break-system-packages --no-warn-script-location conan==1.64.1 chardet
-        conan_path="$(python -m site --user-base)/bin"
+        pip3 install --user --break-system-packages --no-warn-script-location conan==2.10.2 chardet
+        conan_path="$(python3 -m site --user-base)/bin"
         echo "$conan_path" >> $GITHUB_PATH
         "$conan_path/conan" --version
 

diff --git a/.github/workflows/rpc-integration-tests.yml b/.github/workflows/rpc-integration-tests.yml
@@ -33,7 +33,7 @@ jobs:
       - name: Checkout RPC Tests Repository & Install Requirements
         run: |
           rm -rf ${{runner.workspace}}/rpc-tests
-          git -c advice.detachedHead=false clone --depth 1 --branch v1.24.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
+          git -c advice.detachedHead=false clone --depth 1 --branch v1.33.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
           cd ${{runner.workspace}}/rpc-tests
           pip3 install -r requirements.txt --break-system-packages
 

diff --git a/.github/workflows/run_integration_tests.sh b/.github/workflows/run_integration_tests.sh
@@ -28,8 +28,7 @@ debug_traceBlockByNumber/test_09,\
 debug_traceBlockByNumber/test_10,\
 debug_traceBlockByNumber/test_11,\
 debug_traceBlockByNumber/test_12,\
-debug_traceCallMany/test_07,\
-debug_traceCallMany/test_09,\
+debug_traceBlockByNumber/test_29,\
 debug_traceTransaction/test_25.json,\
 debug_traceTransaction/test_36.json,\
 debug_traceTransaction/test_43.json,\
@@ -42,9 +41,7 @@ debug_traceTransaction/test_91.tar,\
 debug_traceTransaction/test_92.tar,\
 debug_traceTransaction/test_96.json,\
 trace_replayBlockTransactions/test_29,\
-engine_,\
-erigon_getLatestLogs,\
-eth_getLogs
+engine_
 
 failed_test=$?
 

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -47,7 +47,7 @@ jobs:
       id: conan
       uses: turtlebrowser/get-conan@main
       with:
-        version: 1.64.1
+        version: 2.10.2
 
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory

diff --git a/AUTHORS b/AUTHORS
@@ -1,9 +1,3 @@
 # This is the official list of Silkworm authors for copyright purposes.
 
-Andrey Ashikhmin
-Tullio Canepa
-Greg Colvin
-Andrea Lanfranchi
-Daniel Lazarenko
-Giulio Rebuffo
-Michelangelo Riccobene
+https://github.com/erigontech/silkworm/graphs/contributors
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -14,25 +14,38 @@
    limitations under the License.
 ]]
 
-cmake_minimum_required(VERSION 3.19.0)
+cmake_minimum_required(VERSION 3.24.0)
 
 if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/evmone/evmone/evmc/.git)
   message(FATAL_ERROR "Git submodules not initialized, execute:\n  git submodule update --init --recursive")
 endif()
 
 get_directory_property(SILKWORM_HAS_PARENT PARENT_DIRECTORY)
 if(NOT SILKWORM_HAS_PARENT)
+  # reduce the log verbosity of evmone/cmake/cable
+  if(NOT CMAKE_MESSAGE_LOG_LEVEL)
+    set(CMAKE_MESSAGE_LOG_LEVEL_EMPTY YES)
+    set(CMAKE_MESSAGE_LOG_LEVEL NOTICE)
+  endif()
+
   include(third_party/evmone/evmone/cmake/cable/bootstrap.cmake)
   include(CableBuildType)
   cable_set_build_type(DEFAULT Release CONFIGURATION_TYPES Release Debug)
 
+  # restore the log verbosity
+  if(CMAKE_MESSAGE_LOG_LEVEL_EMPTY)
+    unset(CMAKE_MESSAGE_LOG_LEVEL)
+  endif()
+
   if(NOT CMAKE_TOOLCHAIN_FILE)
     set(CMAKE_TOOLCHAIN_FILE
         ${CMAKE_CURRENT_SOURCE_DIR}/cmake/toolchain/cxx20.cmake
         CACHE FILEPATH "" FORCE
     )
     include("${CMAKE_TOOLCHAIN_FILE}")
   endif()
+
+  include(cmake/conan.cmake)
 endif()
 
 project(silkworm)
@@ -70,10 +83,6 @@ option(SILKWORM_FUZZER_SANITIZERS "CLang sanitizer options for fuzzers" OFF)
 option(SILKWORM_USE_MIMALLOC "Enable using mimalloc for dynamic memory management" ON)
 option(SILKWORM_ALLOW_UNUSED_VAR_WARNINGS "Turn unused variable errors into warnings" OFF)
 
-if(NOT SILKWORM_HAS_PARENT)
-  include(cmake/conan.cmake)
-endif()
-
 set_property(
   DIRECTORY
   APPEND
@@ -125,5 +134,5 @@ if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
   # avoid fatal error C1002: compiler is out of heap space
   list(REMOVE_ITEM UNIT_TEST_TARGETS silkworm_rpcdaemon_test)
 endif()
-message(STATUS "UNIT_TEST_TARGETS: ${UNIT_TEST_TARGETS}")
+message(VERBOSE "UNIT_TEST_TARGETS: ${UNIT_TEST_TARGETS}")
 add_custom_target(all_unit_tests DEPENDS ${UNIT_TEST_TARGETS})
diff --git a/Makefile b/Makefile
@@ -27,6 +27,6 @@ run_smoke_tests:
 	@cmake/run_smoke_tests.sh $(SILKWORM_BUILD_DIR)
 
 run_unit_tests:
-	@cmake/run_unit_tests.sh $(SILKWORM_BUILD_DIR) $(SILKWORM_CLANG_COVERAGE)
+	@cmake/run_unit_tests.sh $(SILKWORM_BUILD_DIR) $(SILKWORM_CLANG_COVERAGE) $(SILKWORM_SANITIZE) $(SILKWORM_PROJECT_DIR)
 
 test: run_smoke_tests run_unit_tests
diff --git a/README.md b/README.md
@@ -98,7 +98,7 @@ or AppleClang ([Xcode](https://developer.apple.com/xcode/) >= 15)
 
 Conan requires Python, and can be installed using:
 
-    pip3 install --user conan==1.64.1 chardet
+    pip3 install --user conan==2.10.2 chardet
 
 On Linux the conan binary gets installed into `$HOME/.local/bin` which is typically in PATH already.
 On macOS need to add the binary to PATH manually:
@@ -122,9 +122,7 @@ will use "debug" configuration builds of dependencies.
 
 See available profiles in [cmake/profiles](cmake/profiles).
 
-The conan packages could also be pre-installed using [conan install](https://docs.conan.io/1/reference/commands/consumer/install.html):
-
-    conan install --install-folder=build/conan --build=missing --profile=cmake/profiles/macos_arm64_clang_13_debug .
+During the cmake configuration step `conan_provider.cmake` runs a [conan install](https://docs.conan.io/2/reference/commands/install.html) command that downloads packages and builds some of them from source if needed. The exact arguments to this command are printed in the build log.
 
 
 Then run the build itself

diff --git a/cmake/cmake_format.cmake b/cmake/cmake_format.cmake
@@ -38,4 +38,7 @@ file(
 list(PREPEND SRC "${CMAKE_CURRENT_LIST_DIR}/../CMakeLists.txt")
 list(FILTER SRC EXCLUDE REGEX "third_party/.+/(.+/)+CMakeLists.txt$")
 
-execute_process(COMMAND "${CMAKE_FORMAT}" --in-place "--config-file=${CMAKE_CURRENT_LIST_DIR}/cmake_format.yaml" ${SRC})
+execute_process(
+  COMMAND "${CMAKE_FORMAT}" --in-place "--config-file=${CMAKE_CURRENT_LIST_DIR}/cmake_format.yaml" ${SRC}
+  COMMAND_ERROR_IS_FATAL ANY
+)
diff --git a/cmake/cmake_format.yaml b/cmake/cmake_format.yaml
@@ -15,6 +15,13 @@ parse:
         PUBLIC: '*'
         PRIVATE: '*'
         TYPE: '*'
+    execute_process:
+      flags:
+        - OUTPUT_STRIP_TRAILING_WHITESPACE
+      kwargs:
+        COMMAND_ERROR_IS_FATAL: '*'
+        RESULT_VARIABLE: '*'
+        OUTPUT_VARIABLE: '*'
 
 format:
   line_width: 120

diff --git a/cmake/common/targets.cmake b/cmake/common/targets.cmake
@@ -46,6 +46,11 @@ function(silkworm_library TARGET)
   get_directory_property(SUB_LIBS SUBDIRECTORIES)
   list_filter_dirs(SRC SUB_LIBS)
 
+  # cli subdirectories with CMakeLists.txt belong only to silkworm_*_cli libraries
+  if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" MATCHES "/cli$")
+    list(FILTER SRC EXCLUDE REGEX "\/cli\/")
+  endif()
+
   set(TEST_REGEX "_test\\.cpp$")
   # test_util subdirectories without CMakeLists.txt belong to TEST_SRC
   if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" MATCHES "/test_util$")
@@ -57,6 +62,7 @@ function(silkworm_library TARGET)
 
   list(FILTER SRC EXCLUDE REGEX "${TEST_REGEX}")
   list(FILTER SRC EXCLUDE REGEX "_benchmark\\.cpp$")
+
   add_library(${TARGET} ${ARG_TYPE} ${SRC})
 
   target_include_directories(${TARGET} PUBLIC "${SILKWORM_MAIN_DIR}")

diff --git a/cmake/compiler_settings.cmake b/cmake/compiler_settings.cmake
@@ -16,7 +16,7 @@
 
 include(${CMAKE_CURRENT_LIST_DIR}/compiler_settings_sanitize.cmake)
 
-if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
+if(MSVC)
 
   message("MSVC_VERSION = ${MSVC_VERSION}")
   message("MSVC_CXX_ARCHITECTURE_ID = ${MSVC_CXX_ARCHITECTURE_ID}")
@@ -89,16 +89,6 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$")
     link_libraries(c++abi)
   endif()
 
-  # cmake-format: off
-  # abseil in conan is prebuilt with clang 13 (see profiles),
-  # linking absl::log with clang 18+ produces an error due to an ABI change:
-  # https://github.com/abseil/abseil-cpp/issues/1747
-  # https://github.com/llvm/llvm-project/issues/102443
-  # cmake-format: on
-  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 18)
-    add_compile_options(-fclang-abi-compat=17)
-  endif()
-
 else()
   message(WARNING "${CMAKE_CXX_COMPILER_ID} is not a supported compiler. Use at your own risk.")
 endif()