From c7fd737dceed981677303c307921fcdc1129ecd5 Mon Sep 17 00:00:00 2001 From: autoantwort <41973254+autoantwort@users.noreply.github.com> Date: Fri, 17 Jan 2025 21:16:08 +0100 Subject: [PATCH] Output log files as collapsed sections in CI (#1556) --- include/vcpkg/base/contractual-constants.h | 1 + include/vcpkg/commands.build.h | 11 +- include/vcpkg/fwd/vcpkgcmdarguments.h | 16 +++ include/vcpkg/vcpkgcmdarguments.h | 6 +- src/vcpkg/commands.build.cpp | 137 ++++++++++++++++++--- src/vcpkg/commands.install.cpp | 21 ++-- src/vcpkg/commands.z-print-config.cpp | 2 +- src/vcpkg/vcpkgcmdarguments.cpp | 46 ++++--- 8 files changed, 189 insertions(+), 51 deletions(-) diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h index 0be218d064..c135480272 100644 --- a/include/vcpkg/base/contractual-constants.h +++ b/include/vcpkg/base/contractual-constants.h @@ -348,6 +348,7 @@ namespace vcpkg inline constexpr StringLiteral FileInclude = "include"; inline constexpr StringLiteral FileIncomplete = "incomplete"; inline constexpr StringLiteral FileInfo = "info"; + inline constexpr StringLiteral FileIssueBodyMD = "issue_body.md"; inline constexpr StringLiteral FileLicense = "LICENSE"; inline constexpr StringLiteral FileLicenseDotTxt = "LICENSE.txt"; inline constexpr StringLiteral FilePortfileDotCMake = "portfile.cmake"; diff --git a/include/vcpkg/commands.build.h b/include/vcpkg/commands.build.h index 7dd6481a0c..d62c4f538a 100644 --- a/include/vcpkg/commands.build.h +++ b/include/vcpkg/commands.build.h @@ -94,13 +94,18 @@ namespace vcpkg StringLiteral to_string_locale_invariant(const BuildResult build_result); LocalizedString to_string(const BuildResult build_result); LocalizedString create_user_troubleshooting_message(const InstallPlanAction& action, + CIKind detected_ci, const VcpkgPaths& paths, - const Optional& issue_body); + const std::vector& error_logs, + const Optional& maybe_issue_body); inline void print_user_troubleshooting_message(const InstallPlanAction& action, + CIKind detected_ci, const VcpkgPaths& paths, - Optional&& issue_body) + const std::vector& error_logs, + Optional&& maybe_issue_body) { - msg::println(Color::error, create_user_troubleshooting_message(action, paths, issue_body)); + msg::println(Color::error, + create_user_troubleshooting_message(action, detected_ci, paths, error_logs, maybe_issue_body)); } /// diff --git a/include/vcpkg/fwd/vcpkgcmdarguments.h b/include/vcpkg/fwd/vcpkgcmdarguments.h index 1c827c6f74..bbf195ae56 100644 --- a/include/vcpkg/fwd/vcpkgcmdarguments.h +++ b/include/vcpkg/fwd/vcpkgcmdarguments.h @@ -20,4 +20,20 @@ namespace vcpkg struct VcpkgCmdArguments; struct FeatureFlagSettings; struct PortApplicableSetting; + + enum class CIKind + { + None, + GithubActions, + GitLabCI, + AzurePipelines, + AppVeyor, + AwsCodeBuild, + CircleCI, + HerokuCI, + JenkinsCI, + TeamCityCI, + TravisCI, + Generic + }; } diff --git a/include/vcpkg/vcpkgcmdarguments.h b/include/vcpkg/vcpkgcmdarguments.h index 12b2452dd1..cb505d234d 100644 --- a/include/vcpkg/vcpkgcmdarguments.h +++ b/include/vcpkg/vcpkgcmdarguments.h @@ -298,7 +298,8 @@ namespace vcpkg f.dependency_graph = dependency_graph_enabled(); return f; } - const Optional& detected_ci_environment() const { return m_detected_ci_environment; } + const Optional& detected_ci_environment_name() const { return m_detected_ci_environment_name; } + CIKind detected_ci() const { return m_detected_ci_environment_type; } const std::string& get_command() const noexcept { return command; } @@ -333,7 +334,8 @@ namespace vcpkg std::string command; - Optional m_detected_ci_environment; + Optional m_detected_ci_environment_name; + CIKind m_detected_ci_environment_type; friend LocalizedString usage_for_command(const CommandMetadata& command_metadata); CmdParser parser; diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 745c6d0e13..55f804282c 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -207,7 +207,7 @@ namespace vcpkg msg::print(Color::warning, warnings); } msg::println_error(create_error_message(result, spec)); - msg::print(create_user_troubleshooting_message(*action, paths, nullopt)); + msg::print(create_user_troubleshooting_message(*action, args.detected_ci(), paths, {}, nullopt)); return 1; } case BuildResult::Excluded: @@ -1694,43 +1694,144 @@ namespace vcpkg return "https://github.com/microsoft/vcpkg/issues?q=is%3Aissue+is%3Aopen+in%3Atitle+" + spec_name; } - static std::string make_gh_issue_open_url(StringView spec_name, StringView triplet, StringView path) + static std::string make_gh_issue_open_url(StringView spec_name, StringView triplet, StringView body) { return Strings::concat("https://github.com/microsoft/vcpkg/issues/new?title=[", spec_name, "]+Build+error+on+", triplet, - "&body=Copy+issue+body+from+", - Strings::percent_encode(path)); + "&body=", + Strings::percent_encode(body)); + } + + static bool is_collapsible_ci_kind(CIKind kind) + { + switch (kind) + { + case CIKind::GithubActions: + case CIKind::GitLabCI: + case CIKind::AzurePipelines: return true; + case CIKind::None: + case CIKind::AppVeyor: + case CIKind::AwsCodeBuild: + case CIKind::CircleCI: + case CIKind::HerokuCI: + case CIKind::JenkinsCI: + case CIKind::TeamCityCI: + case CIKind::TravisCI: + case CIKind::Generic: return false; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + static void append_file_collapsible(LocalizedString& output, + CIKind kind, + const ReadOnlyFilesystem& fs, + const Path& file) + { + auto title = file.filename(); + auto contents = fs.read_contents(file, VCPKG_LINE_INFO); + switch (kind) + { + case CIKind::GithubActions: + // https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#grouping-log-lines + output.append_raw("::group::") + .append_raw(title) + .append_raw('\n') + .append_raw(contents) + .append_raw("::endgroup::\n"); + break; + case CIKind::GitLabCI: + { + // https://docs.gitlab.com/ee/ci/jobs/job_logs.html#custom-collapsible-sections + using namespace std::chrono; + std::string section_name; + std::copy_if(title.begin(), title.end(), std::back_inserter(section_name), [](char c) { + return c == '.' || ParserBase::is_alphanum(c); + }); + const auto timestamp = duration_cast(system_clock::now().time_since_epoch()).count(); + output + .append_raw( + fmt::format("\\e[0Ksection_start:{}:{}[collapsed=true]\r\\e[0K", timestamp, section_name)) + .append_raw(title) + .append_raw('\n') + .append_raw(contents) + .append_raw(fmt::format("\\e[0Ksection_end:{}:{}\r\\e[0K\n", timestamp, section_name)); + } + break; + case CIKind::AzurePipelines: + // https://learn.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#formatting-commands + output.append_raw("##vso[task.uploadfile]") + .append_raw(file) + .append_raw('\n') + .append_raw("##[group]") + .append_raw(title) + .append_raw('\n') + .append_raw(contents) + .append_raw("##[endgroup]\n"); + break; + case CIKind::None: + case CIKind::AppVeyor: + case CIKind::AwsCodeBuild: + case CIKind::CircleCI: + case CIKind::HerokuCI: + case CIKind::JenkinsCI: + case CIKind::TeamCityCI: + case CIKind::TravisCI: + case CIKind::Generic: Checks::unreachable(VCPKG_LINE_INFO, "CIKind not collapsible"); + default: Checks::unreachable(VCPKG_LINE_INFO); + } } LocalizedString create_user_troubleshooting_message(const InstallPlanAction& action, + CIKind detected_ci, const VcpkgPaths& paths, - const Optional& issue_body) + const std::vector& error_logs, + const Optional& maybe_issue_body) { const auto& spec_name = action.spec.name(); const auto& triplet_name = action.spec.triplet().to_string(); LocalizedString result = msg::format(msgBuildTroubleshootingMessage1).append_raw('\n'); result.append_indent().append_raw(make_gh_issue_search_url(spec_name)).append_raw('\n'); - result.append(msgBuildTroubleshootingMessage2).append_raw('\n'); - if (issue_body.has_value()) + result.append(msgBuildTroubleshootingMessage2).append_raw('\n').append_indent(); + + if (auto issue_body = maybe_issue_body.get()) { - const auto path = issue_body.get()->generic_u8string(); - result.append_indent().append_raw(make_gh_issue_open_url(spec_name, triplet_name, path)).append_raw('\n'); - if (!paths.get_filesystem().find_from_PATH("gh").empty()) + auto& fs = paths.get_filesystem(); + // The 'body' content is not localized because it becomes part of the posted GitHub issue + // rather than instructions for the current user of vcpkg. + if (is_collapsible_ci_kind(detected_ci)) { - Command gh("gh"); - gh.string_arg("issue").string_arg("create").string_arg("-R").string_arg("microsoft/vcpkg"); - gh.string_arg("--title").string_arg(fmt::format("[{}] Build failure on {}", spec_name, triplet_name)); - gh.string_arg("--body-file").string_arg(path); - - result.append(msgBuildTroubleshootingMessageGH).append_raw('\n'); - result.append_indent().append_raw(gh.command_line()); + auto body = fmt::format("Copy issue body from collapsed section \"{}\" in the ci log output", + issue_body->filename()); + result.append_raw(make_gh_issue_open_url(spec_name, triplet_name, body)).append_raw('\n'); + append_file_collapsible(result, detected_ci, fs, *issue_body); + for (Path error_log_path : error_logs) + { + append_file_collapsible(result, detected_ci, fs, error_log_path); + } + } + else + { + const auto path = issue_body->generic_u8string(); + auto body = fmt::format("Copy issue body from {}", path); + result.append_raw(make_gh_issue_open_url(spec_name, triplet_name, body)).append_raw('\n'); + auto gh_path = fs.find_from_PATH("gh"); + if (!gh_path.empty()) + { + Command gh(gh_path[0]); + gh.string_arg("issue").string_arg("create").string_arg("-R").string_arg("microsoft/vcpkg"); + gh.string_arg("--title").string_arg( + fmt::format("[{}] Build failure on {}", spec_name, triplet_name)); + gh.string_arg("--body-file").string_arg(path); + result.append(msgBuildTroubleshootingMessageGH).append_raw('\n'); + result.append_indent().append_raw(gh.command_line()); + } } } else { - result.append_indent() + result .append_raw("https://github.com/microsoft/vcpkg/issues/" "new?template=report-package-build-failure.md&title=[") .append_raw(spec_name) diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 1c8f7f9849..b7eb7ec8e0 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -603,14 +603,19 @@ namespace vcpkg if (result.code != BuildResult::Succeeded && build_options.keep_going == KeepGoing::No) { this_install.print_elapsed_time(); - print_user_troubleshooting_message(action, paths, result.stdoutlog.then([&](auto&) -> Optional { - auto issue_body_path = paths.installed().root() / "vcpkg" / "issue_body.md"; - paths.get_filesystem().write_contents( - issue_body_path, - create_github_issue(args, result, paths, action, include_manifest_in_github_issue), - VCPKG_LINE_INFO); - return issue_body_path; - })); + print_user_troubleshooting_message( + action, + args.detected_ci(), + paths, + result.error_logs, + result.stdoutlog.then([&](auto&) -> Optional { + auto issue_body_path = paths.installed().root() / FileVcpkg / FileIssueBodyMD; + paths.get_filesystem().write_contents( + issue_body_path, + create_github_issue(args, result, paths, action, include_manifest_in_github_issue), + VCPKG_LINE_INFO); + return issue_body_path; + })); Checks::exit_fail(VCPKG_LINE_INFO); } diff --git a/src/vcpkg/commands.z-print-config.cpp b/src/vcpkg/commands.z-print-config.cpp index d5ad761209..9cbb7b9db7 100644 --- a/src/vcpkg/commands.z-print-config.cpp +++ b/src/vcpkg/commands.z-print-config.cpp @@ -48,7 +48,7 @@ namespace vcpkg obj.insert(JsonIdHostTriplet, host_triplet.canonical_name()); obj.insert(JsonIdVcpkgRoot, paths.root.native()); obj.insert(JsonIdTools, paths.tools.native()); - if (auto ci_env = args.detected_ci_environment().get()) + if (auto ci_env = args.detected_ci_environment_name().get()) { obj.insert(JsonIdDetectedCIEnvironment, *ci_env); } diff --git a/src/vcpkg/vcpkgcmdarguments.cpp b/src/vcpkg/vcpkgcmdarguments.cpp index 3f5edcdb9a..ffd8667bee 100644 --- a/src/vcpkg/vcpkgcmdarguments.cpp +++ b/src/vcpkg/vcpkgcmdarguments.cpp @@ -15,55 +15,62 @@ namespace { using namespace vcpkg; - constexpr std::pair KNOWN_CI_VARIABLES[]{ + struct CIRecord + { + StringLiteral env_var; + StringLiteral name; + CIKind type; + }; + + constexpr CIRecord KNOWN_CI_VARIABLES[]{ // Opt-out from CI detection - {EnvironmentVariableVcpkgNoCi, "VCPKG_NO_CI"}, + {EnvironmentVariableVcpkgNoCi, "VCPKG_NO_CI", CIKind::None}, // Azure Pipelines // https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables#system-variables - {EnvironmentVariableTfBuild, "Azure_Pipelines"}, + {EnvironmentVariableTfBuild, "Azure_Pipelines", CIKind::AzurePipelines}, // AppVeyor // https://www.appveyor.com/docs/environment-variables/ - {EnvironmentVariableAppveyor, "AppVeyor"}, + {EnvironmentVariableAppveyor, "AppVeyor", CIKind::AppVeyor}, // AWS Code Build // https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html - {EnvironmentVariableCodebuildBuildId, "AWS_CodeBuild"}, + {EnvironmentVariableCodebuildBuildId, "AWS_CodeBuild", CIKind::AwsCodeBuild}, // CircleCI // https://circleci.com/docs/env-vars#built-in-environment-variables - {EnvironmentVariableCircleCI, "Circle_CI"}, + {EnvironmentVariableCircleCI, "Circle_CI", CIKind::CircleCI}, // GitHub Actions // https://docs.github.com/en/actions/learn-github-actions/ - {EnvironmentVariableGitHubActions, "GitHub_Actions"}, + {EnvironmentVariableGitHubActions, "GitHub_Actions", CIKind::GithubActions}, // GitLab // https://docs.gitlab.com/ee/ci/variables/predefined_variables.html - {EnvironmentVariableGitLabCI, "GitLab_CI"}, + {EnvironmentVariableGitLabCI, "GitLab_CI", CIKind::GitLabCI}, // Heroku // https://devcenter.heroku.com/articles/heroku-ci#immutable-environment-variables - {EnvironmentVariableHerokuTestRunId, "Heroku_CI"}, + {EnvironmentVariableHerokuTestRunId, "Heroku_CI", CIKind::HerokuCI}, // Jenkins // https://wiki.jenkins.io/display/JENKINS/Building+a+software+project#Buildingasoftwareproject-belowJenkinsSetEnvironmentVariables - {EnvironmentVariableJenkinsHome, "Jenkins_CI"}, - {EnvironmentVariableJenkinsUrl, "Jenkins_CI"}, + {EnvironmentVariableJenkinsHome, "Jenkins_CI", CIKind::JenkinsCI}, + {EnvironmentVariableJenkinsUrl, "Jenkins_CI", CIKind::JenkinsCI}, // TeamCity // https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#Predefined+Server+Build+Parameters - {EnvironmentVariableTeamcityVersion, "TeamCity_CI"}, + {EnvironmentVariableTeamcityVersion, "TeamCity_CI", CIKind::TeamCityCI}, // Travis CI // https://docs.travis-ci.com/user/environment-variables/#default-environment-variables - {EnvironmentVariableTravis, "Travis_CI"}, + {EnvironmentVariableTravis, "Travis_CI", CIKind::TravisCI}, // Generic CI environment variables - {EnvironmentVariableCI, "Generic"}, - {EnvironmentVariableBuildId, "Generic"}, - {EnvironmentVariableBuildNumber, "Generic"}, + {EnvironmentVariableCI, "Generic", CIKind::Generic}, + {EnvironmentVariableBuildId, "Generic", CIKind::Generic}, + {EnvironmentVariableBuildNumber, "Generic", CIKind::Generic}, }; constexpr StringLiteral KNOWN_CI_REPOSITORY_IDENTIFIERS[] = { @@ -581,9 +588,10 @@ namespace vcpkg // detect whether we are running in a CI environment for (auto&& ci_env_var : KNOWN_CI_VARIABLES) { - if (get_env(ci_env_var.first).has_value()) + if (get_env(ci_env_var.env_var).has_value()) { - m_detected_ci_environment = ci_env_var.second; + m_detected_ci_environment_name = ci_env_var.name; + m_detected_ci_environment_type = ci_env_var.type; break; } } @@ -790,7 +798,7 @@ namespace vcpkg void VcpkgCmdArguments::track_environment_metrics() const { MetricsSubmission submission; - if (auto ci_env = m_detected_ci_environment.get()) + if (auto ci_env = m_detected_ci_environment_name.get()) { Debug::println("Detected CI environment: ", *ci_env); submission.track_string(StringMetric::DetectedCiEnvironment, *ci_env);