checks: use linguist to detect file types

This commit also allows to skip the generated files checks by adding the
file to the .gitattributes file

Signed-off-by: Alessandro Comodi <[email protected]>

.gitattributes

@@ -0,0 +1,7 @@
+# Linguist - vendored override
+
+third_party/* linguist-vendored
+
+# Linguist - generated override
+
+checks/tests/license/test-missing-spdx-generated.v linguist-generated

checks/Dockerfile

@@ -5,6 +5,16 @@ RUN apt-get update -qq \
  && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \
     ca-certificates \
     git \
+    make \
+    cmake \
+    pkg-config \
+    libicu-dev \
+    zlib1g-dev \
+    libcurl4-openssl-dev \
+    ruby-dev \
+    libssl-dev \
+    build-essential \
+ && gem install github-linguist \
  && apt-get autoclean && apt-get clean && apt-get -y autoremove \
  && update-ca-certificates \
  && rm -rf /var/lib/apt/lists/*

checks/checks.py

@@ -17,6 +17,8 @@
 import re
 import sys
 import tempfile
+import subprocess
+import json
 
 
 __path__ = pathlib.Path(__file__).resolve().parent
@@ -417,6 +419,15 @@ def main(args):
     logging.debug('Starting search in: %s', root_dir)
 
     errors = {}
+
+    json_data = subprocess.check_output("github-linguist --json", shell=True).decode('utf8')
+    files_data = json.loads(json_data)
+
+    ftypes = {}
+    for ftype, fpaths in files_data.items():
+        for fpath in fpaths:
+            ftypes[fpath] = ftype
+
     for root, dirs, files in os.walk(root_dir):
         rpath = pathlib.Path(root).resolve()
         fdebug(rpath, 'Searching')
@@ -463,9 +474,9 @@ def main(args):
                 fwarn(fpath, 'Skipping nonfile')
                 continue
 
-            ftype = detect_file_type(fpath)
+            ftype = ftypes.get(os.path.relpath(fpath, root_dir), None)
             if ftype is None:
-                finfo(fpath, 'Skipping unknown file type')
+                finfo(fpath, 'Skipping generated or unknown file')
                 continue
 
             ferrors = []