From a7a8d24de9b4b8c48ffec7d743bfda86b73fa4f6 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 23 Jan 2025 10:16:05 +0100 Subject: [PATCH 1/3] chore(workflows/pr-test): run rari alongside legacy Use variables to control which tests to run. --- .github/workflows/pr-test-new-ci.yml | 141 --------------------------- .github/workflows/pr-test.yml | 132 ++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 143 deletions(-) delete mode 100644 .github/workflows/pr-test-new-ci.yml diff --git a/.github/workflows/pr-test-new-ci.yml b/.github/workflows/pr-test-new-ci.yml deleted file mode 100644 index f1c564315d7f381..000000000000000 --- a/.github/workflows/pr-test-new-ci.yml +++ /dev/null @@ -1,141 +0,0 @@ -# This file tests more or less everything related to a pull request. All -# in one big job. At the end, if all the testing passes, it proceeds -# to upload all the files that were built to our Dev environment. -# This way, if the tests passed, you'll be able to review the built -# pages on a public URL. - -name: PR Test - new CI - -on: - pull_request: - branches: - - main - -jobs: - tests: - if: github.repository == 'mdn/content' && startsWith(github.event.pull_request.title, '[new-ci]') - runs-on: ubuntu-latest - # Set the permissions to `read-all`, preventing the workflow from - # any accidental write access to the repository. - permissions: read-all - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - # This is the directory where the built files will be placed. - # It's also hardcoded in the `yarn build` command in package.json. - # If you change it here, you must also make the same change in - # package.json. - BUILD_OUT_ROOT: build - - steps: - - uses: actions/checkout@v4 - - - name: Get changed files - run: | - # Use the GitHub API to get the list of changed files - # documentation: https://docs.github.com/rest/commits/commits#compare-two-commits - DIFF_DOCUMENTS=$(gh api repos/{owner}/{repo}/compare/${BASE_SHA}...${HEAD_SHA} \ - --jq '.files | .[] | select(.status|IN("added", "modified", "renamed", "copied", "changed")) | .filename') - - # filter out files that are not markdown files - GIT_DIFF_CONTENT=$(echo "${DIFF_DOCUMENTS}" | egrep -i "^files/.*\.(md)$" | xargs) - echo "GIT_DIFF_CONTENT=${GIT_DIFF_CONTENT}" >> $GITHUB_ENV - - # filter out files that are not attachments - GIT_DIFF_FILES=$(echo "${DIFF_DOCUMENTS}" | egrep -i "^files/.*\.(png|jpeg|jpg|gif|svg|webp)$" | xargs) - echo "GIT_DIFF_FILES=${GIT_DIFF_FILES}" >> $GITHUB_ENV - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Node.js environment - if: ${{ env.GIT_DIFF_CONTENT }} || ${{ env.GIT_DIFF_FILES }} - uses: actions/setup-node@v4 - with: - node-version-file: ".nvmrc" - cache: yarn - - - name: Install all yarn packages - if: ${{ env.GIT_DIFF_CONTENT }} || ${{ env.GIT_DIFF_FILES }} - run: yarn --frozen-lockfile - env: - # https://github.com/microsoft/vscode-ripgrep#github-api-limit-note - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Build changed content - id: build-content - if: env.GIT_DIFF_CONTENT - env: - CONTENT_ROOT: ${{ github.workspace }}/files - - # This is so that if there's a single 'unsafe_html' flaw, it - # completely fails the build. - # But all other flaws should be 'warn', so that we can include - # information about the flaws when we analyze the built PR. - BUILD_FLAW_LEVELS: "unsafe_html: error, *:warn" - - # Because we build these pages in a way that you get a toolbar, - # so the flaws can be displayed, but we don't want any of the - # other toolbar features like "Fix fixable flaws" or "Quick-edit" - # we set this to disable that stuff. - REACT_APP_CRUD_MODE_READONLY: true - - BUILD_LIVE_SAMPLES_BASE_URL: https://live.mdnyalp.dev - BUILD_LEGACY_LIVE_SAMPLES_BASE_URL: https://live-samples.mdn.allizom.net - - # In these builds, we never care for or need the ability to sign in. - # This environment variable will disable that functionality entirely. - REACT_APP_DISABLE_AUTH: true - - # TODO: This should be implicit when `CI=true` - BUILD_NO_PROGRESSBAR: true - - # Playground - REACT_APP_PLAYGROUND_BASE_HOST: mdnyalp.dev - - # rari - LIVE_SAMPLES_BASE_URL: https://live.mdnyalp.dev - INTERACTIVE_EXAMPLES_BASE_URL: https://interactive-examples.mdn.allizom.net - - run: | - # The reason this script isn't in `package.json` is because - # you don't need that script as a writer. It's only used in CI - # and it can't use the default CONTENT_ROOT that gets set in - # package.json. - echo Y|yarn rari update - ARGS=$(echo $GIT_DIFF_CONTENT | sed -E -e "s#(^| )files#\1-f $PWD/files#g") - yarn rari build --no-basic --json-issues --data-issues $ARGS - yarn yari-render-html - - echo "Disk usage size of the build" - du -sh $BUILD_OUT_ROOT - - # Save the PR number into the build - echo ${{ github.event.number }} > ${BUILD_OUT_ROOT}/NR - - # Download the raw diff blob and store that inside the build - # directory. - # The purpose of this is for the PR Review Companion to later - # be able to use this raw diff file for the benefit of analyzing. - wget https://github.com/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}.diff -O ${BUILD_OUT_ROOT}/DIFF - - - name: Merge static assets with built documents - if: env.GIT_DIFF_CONTENT - run: | - # Exclude the .map files, as they're used for debugging JS and CSS. - rsync -a --exclude "*.map" node_modules/@mdn/yari/client/build/ $BUILD_OUT_ROOT - # Show the final disk usage size of the build. - du -sh $BUILD_OUT_ROOT - - - uses: actions/upload-artifact@v4 - if: env.GIT_DIFF_CONTENT - with: - name: build - path: ${{ env.BUILD_OUT_ROOT }} - - - name: Check changed files - if: env.GIT_DIFF_FILES - run: | - echo $GIT_DIFF_FILES - - export CONTENT_ROOT=$(pwd)/files - yarn filecheck $GIT_DIFF_FILES diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 283fd35de64c847..875780295bfb419 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -12,8 +12,8 @@ on: - main jobs: - tests: - if: github.repository == 'mdn/content' && !startsWith(github.event.pull_request.title, '[new-ci]') + legacy: + if: github.repository == 'mdn/content' && vars.TEST_LEGACY == 'true' runs-on: ubuntu-latest # Set the permissions to `read-all`, preventing the workflow from # any accidental write access to the repository. @@ -132,3 +132,131 @@ jobs: export CONTENT_ROOT=$(pwd)/files yarn filecheck $GIT_DIFF_FILES + + tests: + if: github.repository == 'mdn/content' && vars.TEST_SKIP_RARI != 'true' + runs-on: ubuntu-latest + # Set the permissions to `read-all`, preventing the workflow from + # any accidental write access to the repository. + permissions: read-all + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + # This is the directory where the built files will be placed. + # It's also hardcoded in the `yarn build` command in package.json. + # If you change it here, you must also make the same change in + # package.json. + BUILD_OUT_ROOT: build + + steps: + - uses: actions/checkout@v4 + + - name: Get changed files + run: | + # Use the GitHub API to get the list of changed files + # documentation: https://docs.github.com/rest/commits/commits#compare-two-commits + DIFF_DOCUMENTS=$(gh api repos/{owner}/{repo}/compare/${BASE_SHA}...${HEAD_SHA} \ + --jq '.files | .[] | select(.status|IN("added", "modified", "renamed", "copied", "changed")) | .filename') + + # filter out files that are not markdown files + GIT_DIFF_CONTENT=$(echo "${DIFF_DOCUMENTS}" | egrep -i "^files/.*\.(md)$" | xargs) + echo "GIT_DIFF_CONTENT=${GIT_DIFF_CONTENT}" >> $GITHUB_ENV + + # filter out files that are not attachments + GIT_DIFF_FILES=$(echo "${DIFF_DOCUMENTS}" | egrep -i "^files/.*\.(png|jpeg|jpg|gif|svg|webp)$" | xargs) + echo "GIT_DIFF_FILES=${GIT_DIFF_FILES}" >> $GITHUB_ENV + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js environment + if: env.GIT_DIFF_CONTENT || env.GIT_DIFF_FILES + uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + cache: yarn + + - name: Install all yarn packages + if: env.GIT_DIFF_CONTENT || env.GIT_DIFF_FILES + run: yarn --frozen-lockfile + env: + # https://github.com/microsoft/vscode-ripgrep#github-api-limit-note + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Build changed content + id: build-content + if: env.GIT_DIFF_CONTENT + env: + CONTENT_ROOT: ${{ github.workspace }}/files + + # This is so that if there's a single 'unsafe_html' flaw, it + # completely fails the build. + # But all other flaws should be 'warn', so that we can include + # information about the flaws when we analyze the built PR. + BUILD_FLAW_LEVELS: "unsafe_html: error, *:warn" + + # Because we build these pages in a way that you get a toolbar, + # so the flaws can be displayed, but we don't want any of the + # other toolbar features like "Fix fixable flaws" or "Quick-edit" + # we set this to disable that stuff. + REACT_APP_CRUD_MODE_READONLY: true + + BUILD_LIVE_SAMPLES_BASE_URL: https://live.mdnyalp.dev + BUILD_LEGACY_LIVE_SAMPLES_BASE_URL: https://live-samples.mdn.allizom.net + + # In these builds, we never care for or need the ability to sign in. + # This environment variable will disable that functionality entirely. + REACT_APP_DISABLE_AUTH: true + + # TODO: This should be implicit when `CI=true` + BUILD_NO_PROGRESSBAR: true + + # Playground + REACT_APP_PLAYGROUND_BASE_HOST: mdnyalp.dev + + # rari + LIVE_SAMPLES_BASE_URL: https://live.mdnyalp.dev + INTERACTIVE_EXAMPLES_BASE_URL: https://interactive-examples.mdn.allizom.net + + run: | + # The reason this script isn't in `package.json` is because + # you don't need that script as a writer. It's only used in CI + # and it can't use the default CONTENT_ROOT that gets set in + # package.json. + echo Y|yarn rari update + ARGS=$(echo $GIT_DIFF_CONTENT | sed -E -e "s#(^| )files#\1-f $PWD/files#g") + yarn rari build --no-basic --json-issues --data-issues $ARGS + yarn yari-render-html + + echo "Disk usage size of the build" + du -sh $BUILD_OUT_ROOT + + # Save the PR number into the build + echo ${{ github.event.number }} > ${BUILD_OUT_ROOT}/NR + + # Download the raw diff blob and store that inside the build + # directory. + # The purpose of this is for the PR Review Companion to later + # be able to use this raw diff file for the benefit of analyzing. + wget https://github.com/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}.diff -O ${BUILD_OUT_ROOT}/DIFF + + - name: Merge static assets with built documents + if: env.GIT_DIFF_CONTENT + run: | + # Exclude the .map files, as they're used for debugging JS and CSS. + rsync -a --exclude "*.map" node_modules/@mdn/yari/client/build/ $BUILD_OUT_ROOT + # Show the final disk usage size of the build. + du -sh $BUILD_OUT_ROOT + + - uses: actions/upload-artifact@v4 + if: env.GIT_DIFF_CONTENT + with: + name: build + path: ${{ env.BUILD_OUT_ROOT }} + + - name: Check changed files + if: env.GIT_DIFF_FILES + run: | + echo $GIT_DIFF_FILES + + export CONTENT_ROOT=$(pwd)/files + yarn filecheck $GIT_DIFF_FILES From d7be5498c4a87cec943cf4000123e084069a1e74 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 23 Jan 2025 11:08:24 +0100 Subject: [PATCH 2/3] fix(workflows/pr-review-companion): remove outdated dependency --- .github/workflows/pr-review-companion.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-review-companion.yml b/.github/workflows/pr-review-companion.yml index 573a3e65e193908..b57546120437759 100644 --- a/.github/workflows/pr-review-companion.yml +++ b/.github/workflows/pr-review-companion.yml @@ -7,7 +7,7 @@ name: PR review companion on: workflow_run: - workflows: ["PR Test", "PR Test - new CI"] + workflows: ["PR Test"] types: - completed From 8c91c3b8675d8456ba5e6b4e8079d841a837beb4 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 23 Jan 2025 11:13:27 +0100 Subject: [PATCH 3/3] fix(workflows/pr-test): upload artifact only once Introduces TEST_UPLOAD_LEGACY variable, currently set to true. --- .github/workflows/pr-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 875780295bfb419..443728b38aed970 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -120,7 +120,7 @@ jobs: du -sh $BUILD_OUT_ROOT - uses: actions/upload-artifact@v4 - if: env.GIT_DIFF_CONTENT + if: env.GIT_DIFF_CONTENT && vars.TEST_UPLOAD_LEGACY == 'true' with: name: build path: ${{ env.BUILD_OUT_ROOT }} @@ -248,7 +248,7 @@ jobs: du -sh $BUILD_OUT_ROOT - uses: actions/upload-artifact@v4 - if: env.GIT_DIFF_CONTENT + if: env.GIT_DIFF_CONTENT && (vars.TEST_LEGACY != 'true' || vars.TEST_UPLOAD_LEGACY != 'true') with: name: build path: ${{ env.BUILD_OUT_ROOT }}