shipit_uplift, shipit_frontend: Load one directory at a time instead …

…of generating a huge report (#511)

src/shipit_frontend/src/App/CodeCoverage.elm

@@ -53,7 +53,7 @@ init backend_uplift_url =
             }
     in
         -- Load code coverage data
-        ( model, loadArtifact model )
+        ( model, Cmd.none )
 
 
 
@@ -72,24 +72,22 @@ update msg model user =
 
 setDirectory : Model -> Maybe String -> ( Model, Cmd Msg )
 setDirectory model path =
-    ( { model | path = path }, Cmd.none )
+    ( { model | path = path, directories = NotAsked }, loadArtifact { model | path = path } )
 
 
 loadArtifact : Model -> Cmd Msg
 loadArtifact model =
     let
-        -- TODO: use environment variables here
-        hookGroup =
-            "project-releng"
-
-        hookId =
-            "services-staging-shipit-code-coverage"
-
-        artifact =
-            "coverage_by_dir.json"
-
         url =
-            (model.backend_uplift_url ++ "/coverage_by_dir")
+            (model.backend_uplift_url
+                ++ "/coverage_by_dir"
+                ++ case model.path of
+                    Just path ->
+                        "?path=" ++ path
+
+                    Nothing ->
+                        ""
+            )
 
         request =
             Http.request

src/shipit_uplift/shipit_uplift/api.py

@@ -6,9 +6,6 @@
 from __future__ import absolute_import
 
 import pickle
-import json
-import os
-import time
 from flask import abort, request
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.exc import IntegrityError
@@ -371,16 +368,5 @@ def create_patch_status(bugzilla_id):
     return serialize_patch_status(ps)
 
 
-def coverage_by_dir():
-    fname = 'coverage_by_dir.json'
-
-    if os.path.exists(fname) and os.path.getctime(fname) > (time() - 86400):
-        with open(fname, 'r') as f:
-            return json.load(f)
-
-    data = coverage_by_dir_impl.generate()
-
-    with open(fname, 'w') as f:
-        json.dump(data, f)
-
-    return data
+def coverage_by_dir(path=''):
+    return coverage_by_dir_impl.generate(path)

src/shipit_uplift/shipit_uplift/api.yml

@@ -144,6 +144,12 @@ paths:
   /coverage_by_dir:
     get:
       operationId: "shipit_uplift.api.coverage_by_dir"
+      parameters:
+      - name: path
+        in: query
+        description: Path
+        required: false
+        type: string
       responses:
         200:
           description: Code coverage information by directory

src/shipit_uplift/shipit_uplift/coverage_by_dir_impl.py

@@ -12,7 +12,6 @@ class CoverageService(Enum):
 
 
 COVERAGE_SERVICE = CoverageService.CODECOV
-MAX_LEVEL = 3
 
 
 def get_mercurial_commit(github_commit):
@@ -105,29 +104,20 @@ def get_coverage(commit_sha, prev_commit_sha, directory):
     }
 
 
-def get_directories(directory='', curLevel=0):
-    if curLevel == MAX_LEVEL or '.hg' in directory:
-        return []
-
+def get_directories(directory=''):
     r = requests.get('https://hg.mozilla.org/mozilla-central/json-file/tip/' + directory)
-    dirs = [d['abspath'].lstrip('/') for d in r.json()['directories']]
-
-    subdirs = []
-    for d in dirs:
-        subdirs += get_directories(d, curLevel + 1)
-
-    return dirs + subdirs
+    return [d['abspath'].lstrip('/') for d in r.json()['directories']]
 
 
-def generate():
+def generate(path):
     latest_commit, previous_commit = get_coverage_builds()
 
     latest_mercurial_commit = get_mercurial_commit(latest_commit)
     previous_mercurial_commit = get_mercurial_commit(previous_commit)
 
     changesets = load_changesets(previous_mercurial_commit, latest_mercurial_commit)
 
-    directories = get_directories()
+    directories = get_directories(path)
 
     data = dict()
     all_bugs = set()