Merge branch 'main' into add-binarycache-command

# Conflicts:
#	include/vcpkg/archives.h
#	src/vcpkg/archives.cpp
#	src/vcpkg/binarycaching.cpp

Generate-SpdxLicenseList.ps1

@@ -37,16 +37,15 @@ function Transform-JsonFile {
 
     $fileContent = @(
         "// Data downloaded from $Uri",
-        "// Generated by scripts/Generate-SpdxLicenseList.ps1",
-        "{")
+        "// Generated by Generate-SpdxLicenseList.ps1")
     $json.$OuterName |
         Sort-Object -Property $Id -Culture '' |
         ForEach-Object {
-        $fileContent += "    `"$($_.$Id)`","
+        $fileContent += "`"$($_.$Id)`","
     }
-    $fileContent += "}"
 
-    $fileContent -join "`n" | Out-File -FilePath $OutFile -Encoding 'utf8'
+    ($fileContent -join "`n") + "`n" `
+        | Out-File -FilePath $OutFile -Encoding 'utf8' -NoNewline
 }
 
 $baseUrl = "https://raw.githubusercontent.com/$GithubRepository/$Commit/json"

LocProject.json

@@ -1 +1,15 @@
-{}
+{ 
+  "Projects": [
+    {
+      "LanguageSet": "VS_Main_Languages",
+      "LocItems": [
+        {
+          "SourceFile": "locales/messages.json",
+          "CopyOption": "LangIDOnName",
+          "LclFile": "src/localization-team/{Lang}/messages.json.lcl",
+          "OutputPath": "locales"
+        }
+      ]
+    }
+  ]
+}

azure-pipelines/Format-CxxCode.ps1

@@ -16,6 +16,10 @@ if ($null -ne $clangFormat)
 
 if ($IsWindows)
 {
+    if ([String]::IsNullOrEmpty($clangFormat) -or -not (Test-Path $clangFormat))
+    {
+        $clangFormat = 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm\x64\bin\clang-format.exe'
+    }
     if ([String]::IsNullOrEmpty($clangFormat) -or -not (Test-Path $clangFormat))
     {
         $clangFormat = 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\Llvm\x64\bin\clang-format.exe'

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg-configuration.json

@@ -0,0 +1,13 @@
+{
+  "default-registry": null,
+  "registries": [
+    {
+      "kind": "filesystem",
+      "path": "./vcpkg_registry",
+      "packages": [
+        "arrow",
+        "bloom-filter"
+      ]
+    }
+  ]
+}

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg.json

@@ -0,0 +1,15 @@
+{
+  "name": "cycle-detected",
+  "version": "0.1.0",
+  "dependencies": [
+    "arrow",
+    "bloom-filter"
+  ],
+  "overrides": [
+    {
+      "name": "arrow",
+      "version": "6.0.0.20210925",
+      "port-version": 4
+    }
+  ]
+}

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg_registry/ports/arrow/portfile.cmake

@@ -0,0 +1 @@
+// intentionally empty

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg_registry/ports/arrow/vcpkg.json

@@ -0,0 +1,5 @@
+{
+  "name": "arrow",
+  "version": "6.0.0.20210925",
+  "description": "Cross-language development platform for in-memory analytics"
+}

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg_registry/ports/bloom-filter/portfile.cmake

@@ -0,0 +1 @@
+// intentionally empty

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg_registry/ports/bloom-filter/vcpkg.json

@@ -0,0 +1,7 @@
+{
+  "name": "bloom-filter",
+  "version": "0.2.0",
+  "port-version": 1,
+  "description": "bloom filter",
+  "dependencies": ["arrow"]
+}

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg_registry/versions/a-/arrow.json

@@ -0,0 +1,9 @@
+{
+  "versions": [
+    {
+      "version": "6.0.0.20210925",
+      "port-version": 4,
+      "path": "$/ports/arrow"
+    }
+  ]
+}

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg_registry/versions/b-/bloom-filter.json

@@ -0,0 +1,9 @@
+{
+  "versions": [
+    {
+      "version": "0.2.0",
+      "port-version": 1,
+      "path": "$/ports/bloom-filter"
+    }
+  ]
+}

azure-pipelines/e2e_ports/mismatched-version-database/vcpkg_registry/versions/baseline.json

@@ -0,0 +1,13 @@
+{
+  "$doc": "https://github.com/microsoft/vcpkg/blob/master/docs/maintainers/registries.md#filesystem-registries",
+  "default": {
+    "arrow": {
+      "baseline": "6.0.0.20210925",
+      "port-version": 4
+    },
+    "bloom-filter": {
+      "baseline": "0.2.0",
+      "port-version": 1
+    }
+  }
+}

azure-pipelines/end-to-end-tests-dir/manifest-reserialize.ps1

@@ -0,0 +1,20 @@
+. "$PSScriptRoot/../end-to-end-tests-prelude.ps1"
+
+Write-Trace "test re-serializing every manifest"
+$manifestDir = "$TestingRoot/manifest-dir"
+New-Item -Path $manifestDir -ItemType Directory | Out-Null
+
+$ports = Get-ChildItem "$env:VCPKG_ROOT/ports"
+
+$ports | % {
+    if (Test-Path "$_/vcpkg.json") {
+        Copy-Item "$_/vcpkg.json" "$manifestDir" | Out-Null
+        $x = Get-Content "$manifestDir/vcpkg.json" -Raw
+        Run-Vcpkg -EndToEndTestSilent format-manifest "$manifestDir/vcpkg.json" | Out-Null
+        Throw-IfFailed "$_/vcpkg.json"
+        $y = Get-Content "$manifestDir/vcpkg.json" -Raw
+        if ($x -ne $y) {
+            throw "Expected formatting manifest $_/vcpkg.json to cause no modifications"
+        }
+    }
+}

azure-pipelines/end-to-end-tests-dir/versions.ps1

@@ -71,12 +71,23 @@ Throw-IfFailed
 $CurrentTest = "default baseline"
 $out = Run-Vcpkg @commonArgs "--feature-flags=versions" install --x-manifest-root=$versionFilesPath/default-baseline-1 2>&1 | Out-String
 Throw-IfNotFailed
-if ($out -notmatch ".*Error: while checking out baseline.*")
+if ($out -notmatch ".*Error: while checking out baseline\.*")
 {
     $out
     throw "Expected to fail due to missing baseline"
 }
 
+$CurrentTest = "mismatched version database"
+$out = Run-Vcpkg @commonArgs "--feature-flags=versions" install --x-manifest-root="$PSScriptRoot/../e2e_ports/mismatched-version-database" 2>&1 | Out-String
+Throw-IfNotFailed
+if (($out -notmatch ".*Error: Failed to load port because version specs did not match*") -or
+  ($out -notmatch ".*Expected: [email protected]#4.*") -or
+  ($out -notmatch ".*Actual: [email protected].*"))
+{
+    $out
+    throw "Expected to fail due to mismatched versions between portfile and the version database"
+}
+
 git -C "$env:VCPKG_ROOT" fetch https://github.com/vicroms/test-registries
 foreach ($opt_registries in @("",",registries"))
 {

azure-pipelines/end-to-end-tests-prelude.ps1

@@ -70,9 +70,13 @@ function Require-FileNotExists {
 }
 
 function Throw-IfFailed {
+    [CmdletBinding()]
+    Param(
+        [string]$Message = ""
+    )
     if ($LASTEXITCODE -ne 0) {
         Write-Stack
-        throw "'$Script:CurrentTest' had a step with a nonzero exit code"
+        throw "'$Script:CurrentTest' had a step with a nonzero exit code: $Message"
     }
 }
 
@@ -89,11 +93,14 @@ function Write-Trace ([string]$text) {
 
 function Run-Vcpkg {
     Param(
+        [Parameter(Mandatory = $false)]
+        [Switch]$EndToEndTestSilent,
+
         [Parameter(ValueFromRemainingArguments)]
         [string[]]$TestArgs
     )
     $Script:CurrentTest = "$VcpkgExe $($testArgs -join ' ')"
-    Write-Host $Script:CurrentTest
+    if (!$EndToEndTestSilent) { Write-Host $Script:CurrentTest }
     & $VcpkgExe @testArgs
 }
 

azure-pipelines/pipelines.yml

@@ -19,6 +19,8 @@ jobs:
       failOnStderr: true
     - bash: build.amd64.debug/vcpkg-test
       displayName: 'Run vcpkg tests'
+      env:
+        VCPKG_ROOT: UNIT_TESTS_SHOULD_NOT_USE_VCPKG_ROOT
     - task: PowerShell@2
       displayName: 'Run vcpkg end-to-end tests'
       inputs:
@@ -93,7 +95,7 @@ jobs:
           git clone https://github.com/microsoft/vcpkg $env:VCPKG_ROOT -n
           git -C "$env:VCPKG_ROOT" checkout $sha
     - task: CmdLine@2
-      displayName: "Build vcpkg with CMake, and Run Tests"
+      displayName: "Build vcpkg with CMake"
       inputs:
         script: |
           call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86
@@ -102,6 +104,11 @@ jobs:
           ninja.exe -C build.x86.debug
           build.x86.debug\vcpkg-test.exe
         failOnStderr: true
+    - script: build.x86.debug\vcpkg-test.exe
+      displayName: "Run vcpkg tests"
+      failOnStderr: true
+      env:
+        VCPKG_ROOT: UNIT_TESTS_SHOULD_NOT_USE_VCPKG_ROOT
     - task: PowerShell@2
       displayName: 'Run vcpkg end-to-end tests'
       inputs:
@@ -118,7 +125,7 @@ jobs:
       displayName: 'Generate the Messages File'
       inputs:
         script: |
-          build.x86.debug\vcpkg.exe x-generate-default-message-map >locales/en.json
+          build.x86.debug\vcpkg.exe x-generate-default-message-map locales/messages.json
     - task: Powershell@2
       displayName: 'Create Diff'
       inputs:

azure-pipelines/signing.yml

@@ -19,12 +19,6 @@ parameters:
   default:
 
 variables:
-  - ${{ if eq(parameters.VcpkgBaseVersionOverride, '') }}:
-    - name: VCPKG_BASE_VERSION
-      value: $[format('{0:yyyy}-{0:MM}-{0:dd}', pipeline.startTime)]
-  - ${{ if ne(parameters.VcpkgBaseVersionOverride, '') }}:
-    - name: VCPKG_BASE_VERSION
-      value: ${{parameters.VcpkgBaseVersionOverride}}
   - name: TeamName
     value: vcpkg
   - group: vcpkg-dependency-source-blobs
@@ -47,15 +41,34 @@ variables:
 jobs:
   - job: arch_independent
     displayName: 'Build and Sign Arch-Independent Scripts and vcpkg-ce'
+    # The first job records VCPKG_INITIAL_BASE_VERSION as VCPKG_BASE_VERSION so that all subsequent stages agree
+    # on the value; AzureDevOps appears to repeat evaluation of variables such that crossing UTC's day start
+    # would make subsequent pipeline stages use a different day producing a broken build.
+    # Note that pipeline.startTime seems to refer to the start of the *job*, not the overall pipeline run.
+    variables:
+    - ${{ if eq(parameters.VcpkgBaseVersionOverride, '') }}:
+      - name: VCPKG_INITIAL_BASE_VERSION
+        value: $[format('{0:yyyy}-{0:MM}-{0:dd}', pipeline.startTime)]
+    - ${{ if ne(parameters.VcpkgBaseVersionOverride, '') }}:
+      - name: VCPKG_INITIAL_BASE_VERSION
+        value: ${{parameters.VcpkgBaseVersionOverride}}
     pool:
       name: 'VSEngSS-MicroBuild2022-1ES'
     steps:
+    - task: Powershell@2
+      displayName: 'Lock VCPKG_BASE_VERSION'
+      name: versions
+      inputs:
+        pwsh: true
+        targetType: 'inline'
+        script: |
+          Write-Host "##vso[task.setvariable variable=VCPKG_BASE_VERSION;isOutput=true]$env:VCPKG_INITIAL_BASE_VERSION"
     - task: Powershell@2
       displayName: 'Lock Installer Scripts Versions'
       inputs:
         pwsh: true
         filePath: vcpkg-init/lock-versions.ps1
-        arguments: '-Destination "$(Build.BinariesDirectory)" -VcpkgBaseVersion $(VCPKG_BASE_VERSION)'
+        arguments: '-Destination "$(Build.BinariesDirectory)" -VcpkgBaseVersion $(VCPKG_INITIAL_BASE_VERSION)'
     # Build and test vcpkg-ce
     - task: PowerShell@2
       displayName: 'Download vcpkg-ce sources'
@@ -174,6 +187,7 @@ jobs:
     variables:
       VCPKG_STANDALONE_BUNDLE_SHA: $[ dependencies.arch_independent.outputs['shas.VCPKG_STANDALONE_BUNDLE_SHA'] ]
       VCPKG_CE_SHA: $[ dependencies.arch_independent.outputs['shas.VCPKG_CE_SHA'] ]
+      VCPKG_BASE_VERSION: $[ dependencies.arch_independent.outputs['versions.VCPKG_BASE_VERSION'] ]
     steps:
     - task: CmdLine@2
       displayName: "Download fmt library"
@@ -207,6 +221,7 @@ jobs:
     variables:
       VCPKG_STANDALONE_BUNDLE_SHA: $[ dependencies.arch_independent.outputs['shas.VCPKG_STANDALONE_BUNDLE_SHA'] ]
       VCPKG_CE_SHA: $[ dependencies.arch_independent.outputs['shas.VCPKG_CE_SHA'] ]
+      VCPKG_BASE_VERSION: $[ dependencies.arch_independent.outputs['versions.VCPKG_BASE_VERSION'] ]
     steps:
     - task: CmdLine@2
       displayName: "Download fmt library"
@@ -241,6 +256,7 @@ jobs:
     variables:
       VCPKG_STANDALONE_BUNDLE_SHA: $[ dependencies.arch_independent.outputs['shas.VCPKG_STANDALONE_BUNDLE_SHA'] ]
       VCPKG_CE_SHA: $[ dependencies.arch_independent.outputs['shas.VCPKG_CE_SHA'] ]
+      VCPKG_BASE_VERSION: $[ dependencies.arch_independent.outputs['versions.VCPKG_BASE_VERSION'] ]
     steps:
     - task: CmdLine@2
       displayName: "Download fmt library"
@@ -280,6 +296,7 @@ jobs:
     variables:
       VCPKG_STANDALONE_BUNDLE_SHA: $[ dependencies.arch_independent.outputs['shas.VCPKG_STANDALONE_BUNDLE_SHA'] ]
       VCPKG_CE_SHA: $[ dependencies.arch_independent.outputs['shas.VCPKG_CE_SHA'] ]
+      VCPKG_BASE_VERSION: $[ dependencies.arch_independent.outputs['versions.VCPKG_BASE_VERSION'] ]
     steps:
     - task: PowerShell@2
       displayName: "Download fmt library"

include/vcpkg/archives.h

@@ -1,9 +1,11 @@
 #pragma once
 
+#include <vcpkg/base/fwd/span.h>
+#include <vcpkg/base/fwd/system.process.h>
+
 #include <vcpkg/fwd/vcpkgpaths.h>
 
 #include <vcpkg/base/files.h>
-#include <vcpkg/base/system.process.h>
 
 namespace vcpkg
 {
@@ -17,6 +19,11 @@ namespace vcpkg
                                   View<StringView> files,
                                   const Path& destination_dir);
 
+    // Compress the source directory into the destination file.
+    int compress_directory_to_zip(const VcpkgPaths& paths, const Path& source, const Path& destination);
+
+    Command decompress_zip_archive_cmd(const VcpkgPaths& paths, const Path& dst, const Path& archive_path);
+
     std::vector<ExitCodeAndOutput> decompress_in_parallel(View<Command> jobs);
     void decompress_in_parallel(LineInfo, View<Command> jobs);
 }

include/vcpkg/base/basic_checks.h

@@ -25,6 +25,9 @@ namespace vcpkg::Checks
     // Display an error message to the user and exit the tool.
     [[noreturn]] void exit_with_message(const LineInfo& line_info, StringView error_message);
 
+    // Display an error message and the line to the user and exit the tool.
+    [[noreturn]] void exit_with_message_and_line(const LineInfo& line_info, StringView error_message);
+
     // If expression is false, call exit_fail.
     void check_exit(const LineInfo& line_info, bool expression);
 

include/vcpkg/base/cofffilereader.h

@@ -15,6 +15,7 @@ namespace vcpkg
         AMD64 = 0x8664,    // x64
         ARM = 0x1c0,       // ARM little endian
         ARM64 = 0xaa64,    // ARM64 little endian
+        ARM64EC = 0xa641,  // ARM64 "emulation compatible"
         ARMNT = 0x1c4,     // ARM Thumb-2 little endian
         EBC = 0xebc,       // EFI byte code
         I386 = 0x14c,      // Intel 386 or later processors and compatible processors