Merge pull request #2321 from hzeller/feature-20250106-compdb-prefetch

make-compilation-db: trigger relevant dependency fetches in script.

.github/bin/make-compilation-db.sh

@@ -16,19 +16,38 @@
 set -u
 set -e
 
-BANT=${BANT:-bant}
+# Which bazel and bant to use can be chosen by environment variables
+BAZEL=${BAZEL:-bazel}
+BANT=${BANT:-needs-to-be-compiled-locally}
 
-if [ ! -e bazel-bin ]; then
-  echo "Before creating compilation DB, run bazel build first to fetch deps."
-  exit 1
+if [ "${BANT}" = "needs-to-be-compiled-locally" ]; then
+  # Bant not given, compile from bzlmod dep. We need to do that before
+  # we run other bazel rules below as we change the cxxopt flags. Remember
+  # the full realpath of the resulting binary to be immune to symbolic-link
+  # switcharoo by bazel.
+  ${BAZEL} build -c opt --cxxopt=-std=c++20 @bant//bant:bant >/dev/null 2>&1
+  BANT=$(realpath bazel-bin/external/bant*/bant/bant)
 fi
 
-bazel run --cxxopt=-std=c++20 @bant//bant:bant -- \
-      -C $(pwd) compile-flags 2>/dev/null > compile_flags.txt
+BAZEL_OPTS="-c opt --noshow_progress"
+# Bazel-build all targets that generate files, so that they can be
+# seen in dependency analysis.
+${BAZEL} build ${BAZEL_OPTS} $(${BANT} list-targets | \
+  egrep "genrule|cc_proto_library|genlex|genyacc" | awk '{print $3}')
 
-# Bant does not see yet the flex dependency inside the toolchain
+# Some selected targets to trigger all dependency fetches from MODULE.bazel
+# verilog-y-final to create a header, kzip creator to trigger build of any.pb.h
+# and some test that triggers fetching nlohmann_json and gtest
+${BAZEL} build ${BAZEL_OPTS} //verible/verilog/parser:verilog-y-final \
+  //verible/verilog/tools/kythe:verible-verilog-kythe-kzip-writer \
+  //verible/common/lsp:json-rpc-dispatcher_test
+
+# bant does not distinguish the includes per file yet, so instead of
+# a compile_commands.json, we can just as well create a simpler
+# compile_flags.txt which is easier to digest for all kinds of tools anyway.
+${BANT} compile-flags 2>/dev/null > compile_flags.txt
+
+# Bant does not see the flex dependency inside the toolchain yet.
 for d in bazel-out/../../../external/*flex*/src/FlexLexer.h ; do
   echo "-I$(dirname $d)" >> compile_flags.txt
 done
-
-echo "Now, re-run original build to make all artifacts visible to clang-tidy"

.github/bin/run-build-cleaner.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Verible Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -u
+set -e
+
+# Which bazel and bant to use can be chosen by environment variables
+BAZEL=${BAZEL:-bazel}
+BANT=${BANT:-needs-to-be-compiled-locally}
+
+if [ "${BANT}" = "needs-to-be-compiled-locally" ]; then
+  # Bant not given, compile from bzlmod dep.
+  ${BAZEL} build -c opt --cxxopt=-std=c++20 @bant//bant:bant >/dev/null 2>&1
+  BANT=$(realpath bazel-bin/external/bant*/bant/bant)
+fi
+
+DWYU_OUT="${TMPDIR:-/tmp}/dwyu.out"
+
+if "${BANT}" -q dwyu ... ; then
+  echo "Dependencies ok"
+else
+  echo
+  echo "^ Please run buildozer commands to fix the dependencies and amend PR"
+  exit 1
+fi

.github/bin/run-clang-format.sh → .github/bin/run-format.sh

@@ -29,7 +29,7 @@ find . -name "*.h" -o -name "*.cc" \
   | egrep -v 'third_party/|external_libs/|.github/' \
   | xargs -P2 ${CLANG_FORMAT} -i
 
-# If we have buildifier installed, use that on BUILD files
+# If we have buildifier installed, use that to format BUILD files
 if command -v ${BUILDIFIER} >/dev/null; then
   echo "Run $(buildifier --version)"
   ${BUILDIFIER} -lint=fix MODULE.bazel $(find . -name BUILD -o -name "*.bzl")

.github/bin/smoke-test.sh

@@ -138,11 +138,11 @@ ExpectedFailCount[lint:ibex]=14
 ExpectedFailCount[project:ibex]=211
 ExpectedFailCount[preprocessor:ibex]=385
 
-ExpectedFailCount[syntax:opentitan]=61
-ExpectedFailCount[lint:opentitan]=61
-ExpectedFailCount[project:opentitan]=947
+ExpectedFailCount[syntax:opentitan]=65
+ExpectedFailCount[lint:opentitan]=65
+ExpectedFailCount[project:opentitan]=951
 ExpectedFailCount[formatter:opentitan]=1
-ExpectedFailCount[preprocessor:opentitan]=2561
+ExpectedFailCount[preprocessor:opentitan]=2560
 
 ExpectedFailCount[syntax:sv-tests]=77
 ExpectedFailCount[lint:sv-tests]=76
@@ -211,11 +211,11 @@ ExpectedFailCount[preprocessor:scr1]=46
 ExpectedFailCount[project:serv]=1
 ExpectedFailCount[preprocessor:serv]=1
 
-ExpectedFailCount[syntax:basejump_stl]=480
-ExpectedFailCount[lint:basejump_stl]=480
-ExpectedFailCount[project:basejump_stl]=595
+ExpectedFailCount[syntax:basejump_stl]=481
+ExpectedFailCount[lint:basejump_stl]=481
+ExpectedFailCount[project:basejump_stl]=596
 ExpectedFailCount[formatter:basejump_stl]=1
-ExpectedFailCount[preprocessor:basejump_stl]=631
+ExpectedFailCount[preprocessor:basejump_stl]=632
 
 ExpectedFailCount[syntax:opl3_fpga]=3
 ExpectedFailCount[lint:opl3_fpga]=3

.github/workflows/verible-ci.yml

@@ -25,7 +25,7 @@ env:
 jobs:
 
 
-  VerifyFormatting:
+  CheckFormatAndBuildClean:
     runs-on: ubuntu-24.04
     steps:
 
@@ -41,7 +41,11 @@ jobs:
 
     - name: Run formatting style check
       run: |
-        CLANG_FORMAT=clang-format-17 ./.github/bin/run-clang-format.sh
+        echo "--- check formatting ---"
+        CLANG_FORMAT=clang-format-17 ./.github/bin/run-format.sh
+        echo "--- check build dependencies ---"
+        ./.github/bin/run-build-cleaner.sh
+        echo "--- check potential problems ---"
         ./.github/bin/check-potential-problems.sh
 
     - name: 📤 Upload performance graphs
@@ -72,11 +76,11 @@ jobs:
         apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 15CF4D18AF4F7421
         wget https://apt.llvm.org/llvm.sh
         chmod +x ./llvm.sh
-        ./llvm.sh 17
+        ./llvm.sh 18
 
     - name: Install other Dependencies
       run: |
-        apt -qq -y install clang-tidy-11 clang-tidy-17 build-essential git wget
+        apt -qq -y install clang-tidy-11 clang-tidy-18 build-essential git wget
         source ./.github/settings.sh
         ./.github/bin/install-bazel.sh
 
@@ -97,24 +101,17 @@ jobs:
 
     - name: Run clang tidy
       run: |
-        echo "::group::Trigger fetch and preparation"
-        bazel build -c opt :install-binaries
-        echo "::endgroup::"
-
         echo "::group::Make Compilation DB"
         .github/bin/make-compilation-db.sh
         cat compile_flags.txt
         echo "::endgroup::"
 
-        echo "::group::Re-etablish links"
-        bazel build -c opt :install-binaries
-        echo "::endgroup::"
-
-        # For runtime references, use clang-tidy 11 that still has it, everything else: latest.
+        # For runtime references, use clang-tidy 11 that still has it,
+        # everything else: latest.
         CLANG_TIDY=clang-tidy-11 ./.github/bin/run-clang-tidy-cached.cc --checks="-*,google-runtime-references" \
            || ( cat verible_clang-tidy.out ; exit 1)
 
-        CLANG_TIDY=clang-tidy-17 ./.github/bin/run-clang-tidy-cached.cc \
+        CLANG_TIDY=clang-tidy-18 ./.github/bin/run-clang-tidy-cached.cc \
             || ( cat verible_clang-tidy.out ; exit 1)
 
     - name: 📤 Upload performance graphs
@@ -124,40 +121,6 @@ jobs:
         name: "diag"
         path: "**/plot_*.svg"
 
-  RunBantBuildCleaner:
-    # Running http://bant.build/ to check all dependencies in BUILD files.
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Get Bant
-        run: |
-          # TODO: provide this as action where we simply say with version=...
-          VERSION="v0.1.9"
-          STATIC_VERSION="bant-${VERSION}-linux-static-x86_64"
-          wget "https://github.com/hzeller/bant/releases/download/${VERSION}/${STATIC_VERSION}.tar.gz"
-          tar xvzf "${STATIC_VERSION}.tar.gz"
-          mkdir -p bin
-          ln -sf ../"${STATIC_VERSION}/bin/bant" bin/
-          bin/bant -V
-
-      - name: Build Project genrules
-        run: |
-          # Fetch all dependencies and run genrules for bant to see every file
-          # that makes it into the compile. Use bant itself to find genrules.
-          bazel build $(bin/bant -q genrule-outputs | awk '{print $2}') \
-              //verible/common/analysis:command-file-lexer \
-              //verible/verilog/parser:verilog-lex \
-              //verible/verilog/parser:verilog-y \
-              //verible/verilog/parser:verilog-y-final
-
-      - name: Run bant build-cleaner
-        run: |
-          bin/bant dwyu ...
-
   Check:
     container: ubuntu:jammy
     runs-on: [self-hosted, Linux, X64, gcp-custom-runners]