Enable VCPKG in CI build

.github/workflows/mac.yml

@@ -109,101 +109,6 @@ jobs:
           --apple_sysroot=iphonesimulator \
           --osx_arch=${{ matrix.target_arch }}
 
-  Vcpkg:
-    runs-on: macos-13
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: ${{ env.python_version }}
-
-    - name: "Run Homebrew"
-      run: brew install autoconf automake autoconf-archive
-
-    - name: "Run vcpkg(x64-osx)"
-      uses: lukka/run-vcpkg@v11
-      with:
-        vcpkgDirectory: "${{ runner.temp }}/vcpkg"
-        vcpkgGitCommitId: "b322364f06308bdd24823f9d8f03fe0cc86fd46f" # 2024.12.16
-        runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
-        vcpkgJsonGlob: "cmake/vcpkg.json"
-        vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
-      env:
-        VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
-        VCPKG_DEFAULT_TRIPLET: "x64-osx"
-        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
-        # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
-
-    - name: "Run compile_schema.py"
-      run: |
-        set -e -x
-        # Runner's host triplet should be x64-osx or arm64-osx
-        export FLATC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-osx/tools/flatbuffers"
-        export PATH="$FLATC_DIR:$PATH"
-        flatc --version
-        python onnxruntime/core/flatbuffers/schema/compile_schema.py --flatc "$(which flatc)"
-        python onnxruntime/lora/adapter_format/compile_schema.py --flatc "$(which flatc)"
-
-    - name: "Detect protoc"
-      id: protoc-detect
-      run: |
-        export PROTOC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-osx/tools/protobuf"
-        export PATH="$PROTOC_DIR:$PATH"
-        protoc --version
-        echo "protoc_path=$(which protoc)" >> "$GITHUB_OUTPUT"
-
-    - name: "Run build.py(x64-osx)"
-      run: |
-        python ./tools/ci_build/build.py \
-          --build_dir "build/x64-osx" \
-          --skip_submodule_sync \
-          --skip_tests \
-          --compile_no_warning_as_error \
-          --parallel \
-          --path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" \
-          --osx_arch x86_64 \
-          --use_vcpkg \
-          --cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
-          --cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-osx" \
-          --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
-          --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
-      env:
-        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
-      shell: bash
-
-    - name: "Run vcpkg(arm64-osx)"
-      uses: lukka/run-vcpkg@v11
-      with:
-        vcpkgDirectory: "${{ runner.temp }}/vcpkg"
-        doNotUpdateVcpkg: true
-        runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
-        vcpkgJsonGlob: "cmake/vcpkg.json"
-        vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
-      env:
-        VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
-        VCPKG_DEFAULT_TRIPLET: "arm64-osx"
-        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
-        # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
-
-    - name: "Run build.py(arm64-osx)"
-      run: |
-        python ./tools/ci_build/build.py \
-          --build_dir "build/arm64-osx" \
-          --skip_submodule_sync \
-          --skip_tests \
-          --compile_no_warning_as_error \
-          --parallel \
-          --path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" \
-          --osx_arch arm64 \
-          --use_vcpkg \
-          --cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
-          --cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-osx" \
-          --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
-          --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
-      env:
-        VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
-      shell: bash
-
   Objective-C-StaticAnalysis:
     runs-on: macos-14
 

.github/workflows/sca.yml

@@ -36,10 +36,6 @@ jobs:
         run: azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" cuda_sdk
 
 
-      - name: Install ONNX
-        run: |
-          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
-
       # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
       - name: Build code
         env:
@@ -117,7 +113,6 @@ jobs:
       - name: Delete build folder
         run: |
           if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
-          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
 
       # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
       - name: Build code
@@ -157,7 +152,6 @@ jobs:
       - name: Delete build folder
         run: |
           if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
-          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x86 -install_prefix D:\b\Debug\installed -build_config Debug
 
       # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
       - name: Build code

.github/workflows/windows.yml

@@ -37,102 +37,8 @@ jobs:
       - name: Delete build folder
         run: |
           if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
-          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
+
 
       # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
       - name: Build code
-        run: python tools\ci_build\build.py --windows_sdk_version 10.0.22621.0 --enable_training --build_java --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_pybind --use_cuda --cuda_home=${{ github.workspace }}\cuda_sdk\v12.2 --enable_cuda_profiling  --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75
-
-  Vcpkg:
-    runs-on: "windows-latest"
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.11.x'
-          architecture: 'x64'
-
-      - name: "Run vcpkg(x64-windows)"
-        uses: lukka/run-vcpkg@v11
-        with:
-          vcpkgDirectory: "C:/vcpkg" # use VCPKG_INSTALLATION_ROOT of the image
-          vcpkgGitCommitId: "b322364f06308bdd24823f9d8f03fe0cc86fd46f" # 2024.12.16
-          runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
-          vcpkgJsonGlob: "cmake/vcpkg.json"
-          vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
-        env:
-          VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
-          VCPKG_DEFAULT_TRIPLET: "x64-windows"
-          VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
-          # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
-
-      - name: "Run compile_schema.py"
-        run: |
-          # Runner's host triplet should be x64-windows or arm64-windows
-          $FLATC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-windows/tools/flatbuffers"
-          $env:PATH="$FLATC_DIR;$env:PATH"
-          flatc --version
-          $FLATC_PATH = Join-Path "$FLATC_DIR" "flatc.exe"
-          python onnxruntime/core/flatbuffers/schema/compile_schema.py --flatc "$FLATC_PATH"
-          python onnxruntime/lora/adapter_format/compile_schema.py --flatc "$FLATC_PATH"
-        shell: pwsh
-
-      - name: "Detect protoc"
-        id: protoc-detect
-        run: |
-          $PROTOC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-windows/tools/protobuf"
-          $env:PATH="$PROTOC_DIR;$env:PATH"
-          protoc --version
-          $PROTOC_PATH = Join-Path "$PROTOC_DIR" "protoc.exe"
-          "protoc_path=$PROTOC_PATH" >> $env:GITHUB_OUTPUT
-        shell: pwsh
-
-      - name: "Run build.py(x64-windows)"
-        run: |
-          python tools\ci_build\build.py `
-            --build_dir "cmake_build/x64-windows" `
-            --skip_submodule_sync `
-            --skip_tests `
-            --compile_no_warning_as_error `
-            --parallel `
-            --path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" `
-            --use_vcpkg `
-            --cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" `
-            --cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-windows" `
-            --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" `
-            --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
-        env:
-          VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
-        shell: pwsh
-
-      - name: "Run vcpkg(arm64-windows)"
-        uses: lukka/run-vcpkg@v11
-        with:
-          vcpkgDirectory: "C:/vcpkg" # use VCPKG_INSTALLATION_ROOT of the image
-          doNotUpdateVcpkg: true
-          runVcpkgInstall: true # vcpkg install --x-manifest-root cmake --x-install-root .build --overlay-triplets cmake/vcpkg-triplets/default
-          vcpkgJsonGlob: "cmake/vcpkg.json"
-          vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
-        env:
-          VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
-          VCPKG_DEFAULT_TRIPLET: "arm64-windows"
-          VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
-          # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
-
-      - name: "Run build.py(arm64-windows)"
-        run: |
-          python tools\ci_build\build.py `
-            --build_dir "cmake_build/arm64-windows" --arm64 `
-            --skip_submodule_sync `
-            --skip_tests `
-            --compile_no_warning_as_error `
-            --parallel `
-            --path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" `
-            --use_vcpkg `
-            --cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" `
-            --cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-windows" `
-            --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" `
-            --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
-        env:
-          VCPKG_OVERLAY_TRIPLETS: "${{ github.workspace }}/cmake/vcpkg-triplets/default"
-        shell: pwsh
+        run: python tools\ci_build\build.py --windows_sdk_version 10.0.22621.0 --enable_training --build_java --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_pybind --use_cuda --cuda_home=${{ github.workspace }}\cuda_sdk\v12.2 --enable_cuda_profiling  --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75

cmake/CMakeLists.txt

@@ -5,18 +5,9 @@
 # Minimum CMake required
 cmake_minimum_required(VERSION 3.26)
 
-cmake_policy(SET CMP0069 NEW)
-set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
-
-cmake_policy(SET CMP0092 NEW)
-cmake_policy(SET CMP0091 NEW)
-cmake_policy(SET CMP0117 NEW)
 # Don't let cmake set a default value for CMAKE_CUDA_ARCHITECTURES
 cmake_policy(SET CMP0104 OLD)
-
-# Enable Hot Reload for MSVC compilers if supported.
-cmake_policy(SET CMP0141 NEW)
-
+cmake_policy(SET CMP0148 NEW)
 # Project
 project(onnxruntime C CXX ASM)
 
@@ -59,8 +50,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 # Enable CTest
 enable_testing()
-include(Dart)
-
+include(CTest)
 if (NOT CMAKE_BUILD_TYPE)
   message(STATUS "Build type not set - using RelWithDebInfo")
   set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose build type: Debug Release RelWithDebInfo MinSizeRel." FORCE)
@@ -959,7 +949,7 @@ if (onnxruntime_USE_QNN)
 
     if (MSVC OR ${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
       file(GLOB QNN_LIB_FILES LIST_DIRECTORIES false "${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/libQnn*.so"
-	                                     "${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/Qnn*.dll"
+                                             "${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/Qnn*.dll"
                                              "${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/libHtpPrepare.so"
                                              "${onnxruntime_QNN_HOME}/lib/${QNN_ARCH_ABI}/HtpPrepare.dll")
       if (${QNN_ARCH_ABI} STREQUAL "aarch64-windows-msvc" OR ${QNN_ARCH_ABI} STREQUAL "arm64x-windows-msvc")
@@ -1123,7 +1113,10 @@ function(onnxruntime_set_compile_flags target_name)
       foreach(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
         target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:I${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY}>")
       endforeach()
-
+      if(VCPKG_INSTALLED_DIR)
+        cmake_path(NATIVE_PATH VCPKG_INSTALLED_DIR NORMALIZE VCPKG_INSTALLED_DIR_native)
+        target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:SHELL:/external:I ${VCPKG_INSTALLED_DIR_native}>")
+      endif()
       foreach(onnxruntime_external_lib IN LISTS onnxruntime_EXTERNAL_LIBRARIES)
         #TODO: the list contains cmake keywords like "debug". We should exclude them.
         if(TARGET ${onnxruntime_external_lib})
@@ -1153,7 +1146,9 @@ function(onnxruntime_set_compile_flags target_name)
       # We do not treat warnings from 3rd-party libraries as errors. In order to do that, we need to add their header files locations to /external:I.
       target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/experimental:external>" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /experimental:external>")
       target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:W0>" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:W0>")
-      target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:templates->" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:templates->")
+      #if(NOT onnxruntime_USE_VCPKG)
+      #  target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:templates->" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:templates->")
+      #endif()
       target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:I${CMAKE_CURRENT_SOURCE_DIR}>" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:I${CMAKE_CURRENT_SOURCE_DIR}>")
       target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:I${CMAKE_CURRENT_BINARY_DIR}>" "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /external:I${CMAKE_CURRENT_BINARY_DIR}>")
       if (onnxruntime_ENABLE_STATIC_ANALYSIS)
@@ -1714,6 +1709,9 @@ endif()
 
 
 if (WIN32)
+  if (onnxruntime_USE_MIMALLOC)
+   list(APPEND onnxruntime_EXTERNAL_LIBRARIES mimalloc-static)
+  endif()
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${SYS_PATH_LIB})
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES debug Dbghelp)
   # In a onecore build the umbrella libs already contains references to the APIs in advapi32, so in onecore build we do not need to link to advapi32

cmake/external/abseil-cpp.cmake

@@ -22,12 +22,13 @@ endif()
 # currently absl::lts_20240116 which affects abseil-cpp.natvis debugger
 # visualization file, that must be adjusted accordingly, unless we eliminate
 # that namespace at build time.
-FetchContent_Declare(
+onnxruntime_fetchcontent_declare(
     abseil_cpp
     URL ${DEP_URL_abseil_cpp}
     URL_HASH SHA1=${DEP_SHA1_abseil_cpp}
     PATCH_COMMAND ${ABSL_PATCH_COMMAND}
     FIND_PACKAGE_ARGS 20240722 NAMES absl
+    EXCLUDE_FROM_ALL
 )
 
 onnxruntime_fetchcontent_makeavailable(abseil_cpp)

cmake/external/composable_kernel.cmake

@@ -2,11 +2,12 @@ set(PATCH_CLANG ${PROJECT_SOURCE_DIR}/patches/composable_kernel/Fix_Clang_Build.
 set(PATCH_GFX12X ${PROJECT_SOURCE_DIR}/patches/composable_kernel/Add_gfx12x_support.patch)
 
 include(FetchContent)
-FetchContent_Declare(composable_kernel
+onnxruntime_fetchcontent_declare(composable_kernel
   URL ${DEP_URL_composable_kernel}
   URL_HASH SHA1=${DEP_SHA1_composable_kernel}
   PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PATCH_CLANG} &&
                 ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PATCH_GFX12X}
+  EXCLUDE_FROM_ALL
 )
 
 FetchContent_GetProperties(composable_kernel)

cmake/external/cudnn_frontend.cmake

@@ -1,12 +1,13 @@
-include(FetchContent)
-FetchContent_Declare(
+
+onnxruntime_fetchcontent_declare(
   cudnn_frontend
   URL ${DEP_URL_cudnn_frontend}
   URL_HASH SHA1=${DEP_SHA1_cudnn_frontend}
+  EXCLUDE_FROM_ALL
 )
 
-set(CUDNN_FRONTEND_BUILD_SAMPLES OFF)
-set(CUDNN_FRONTEND_BUILD_UNIT_TESTS OFF)
-set(CUDNN_FRONTEND_BUILD_PYTHON_BINDINGS OFF)
+set(CUDNN_FRONTEND_BUILD_SAMPLES OFF CACHE BOOL "" FORCE)
+set(CUDNN_FRONTEND_BUILD_UNIT_TESTS OFF CACHE BOOL "" FORCE)
+set(CUDNN_FRONTEND_BUILD_PYTHON_BINDINGS OFF CACHE BOOL "" FORCE)
 set(CUDNN_PATH ${onnxruntime_CUDNN_HOME})
-FetchContent_MakeAvailable(cudnn_frontend)
+onnxruntime_fetchcontent_makeavailable(cudnn_frontend)

cmake/external/cutlass.cmake

@@ -1,8 +1,9 @@
 include(FetchContent)
-FetchContent_Declare(
+onnxruntime_fetchcontent_declare(
   cutlass
   URL ${DEP_URL_cutlass}
   URL_HASH SHA1=${DEP_SHA1_cutlass}
+  EXCLUDE_FROM_ALL
 )
 
 FetchContent_GetProperties(cutlass)

cmake/external/dml.cmake

@@ -100,10 +100,11 @@ else()
   endif()
 endif()
 
-FetchContent_Declare(
+onnxruntime_fetchcontent_declare(
     directx_headers
     URL ${DEP_URL_directx_headers}
     URL_HASH SHA1=${DEP_SHA1_directx_headers}
+    EXCLUDE_FROM_ALL
 )
 
 FetchContent_Populate(directx_headers)