Merge pull request #374 from openlawlibrary/renatav/bf-update

Updater pipeline and initial work on the breadth-first update

CHANGELOG.md

@@ -7,19 +7,20 @@ and this project adheres to [Semantic Versioning][semver].
 
 ## [Unreleased]
 
-
 ### Added
 
-
 ### Changed
 
+- Implement updater pipeline ([374])
+- Improve error messages and error logging ([374])
+- Update target repositories in a breadth-first way ([374])
 
 ### Fixed
 
+[374]: https://github.com/openlawlibrary/taf/pull/374
 
 ## [0.28.0] - 11/10/2023
 
-
 ### Added
 
 - Implement tests for the functions which are directly called by the cli (API package) ([362])
@@ -83,9 +84,7 @@ and this project adheres to [Semantic Versioning][semver].
 [357]: https://github.com/openlawlibrary/taf/pull/357
 [355]: https://github.com/openlawlibrary/taf/pull/355
 [354]: https://github.com/openlawlibrary/taf/pull/354
-[353]: https://github.com/openlawlibrary/taf/pull/353
 [352]: https://github.com/openlawlibrary/taf/pull/352
-[351]: https://github.com/openlawlibrary/taf/pull/351
 [349]: https://github.com/openlawlibrary/taf/pull/349
 [346]: https://github.com/openlawlibrary/taf/pull/346
 [343]: https://github.com/openlawlibrary/taf/pull/343
@@ -107,7 +106,6 @@ and this project adheres to [Semantic Versioning][semver].
 
 ### Fixed
 
-[349]: https://github.com/openlawlibrary/taf/pull/349
 
 ## [0.26.0] - 07/12/2023
 
@@ -126,7 +124,6 @@ and this project adheres to [Semantic Versioning][semver].
 
 - Fix create repository ([325])
 
-
 [325]: https://github.com/openlawlibrary/taf/pull/325
 [321]: https://github.com/openlawlibrary/taf/pull/321
 [320]: https://github.com/openlawlibrary/taf/pull/320
@@ -144,11 +141,9 @@ and this project adheres to [Semantic Versioning][semver].
 
 - Fix execution of scripts ([311])
 
-
 [313]: https://github.com/openlawlibrary/taf/pull/313
 [311]: https://github.com/openlawlibrary/taf/pull/311
 
-
 ## [0.24.0] - 02/21/2023
 
 ### Added
@@ -162,7 +157,6 @@ and this project adheres to [Semantic Versioning][semver].
 
 - Use `generate_and_write_unencrypted_rsa_keypair` for no provided password ([305])
 
-
 [309]: https://github.com/openlawlibrary/taf/pull/309
 [308]: https://github.com/openlawlibrary/taf/pull/308
 [305]: https://github.com/openlawlibrary/taf/pull/305
@@ -174,8 +168,8 @@ and this project adheres to [Semantic Versioning][semver].
 ### Changed
 
 ### Fixed
-- Fix `clone_or_pull` method ([303])
 
+- Fix `clone_or_pull` method ([303])
 
 [303]: https://github.com/openlawlibrary/taf/pull/303
 
@@ -201,6 +195,7 @@ and this project adheres to [Semantic Versioning][semver].
 ### Changed
 
 ### Fixed
+
 - Pin `pyOpenSSL` to newer version ([299])
 
 [299]: https://github.com/openlawlibrary/taf/pull/299
@@ -212,7 +207,8 @@ and this project adheres to [Semantic Versioning][semver].
 ### Changed
 
 ### Fixed
- - Add missing tuf import in `log.py` ([298])
+
+- Add missing tuf import in `log.py` ([298])
 
 [298]: https://github.com/openlawlibrary/taf/pull/298
 
@@ -226,17 +222,14 @@ and this project adheres to [Semantic Versioning][semver].
 
 - Remove _tuf_patches in `__init__.py` ([297])
 
-
 [297]: https://github.com/openlawlibrary/taf/pull/297
 
 ### Added
 
 ### Changed
 
-
 ### Fixed
 
-
 ## [0.22.1] - 12/14/2022
 
 ### Added
@@ -247,10 +240,8 @@ and this project adheres to [Semantic Versioning][semver].
 
 - Move _tuf_patches to repository lib ([296])
 
-
 [296]: https://github.com/openlawlibrary/taf/pull/296
 
-
 ## [0.22.0] - 12/09/2022
 
 ### Added
@@ -276,7 +267,6 @@ and this project adheres to [Semantic Versioning][semver].
 - Fix `all_commits_since_commit` to validate provided commit ([278])
 - Remove pin for `PyOpenSSL` ([273])
 
-
 [294]: https://github.com/openlawlibrary/taf/pull/294
 [293]: https://github.com/openlawlibrary/taf/pull/293
 [292]: https://github.com/openlawlibrary/taf/pull/292
@@ -1043,8 +1033,7 @@ and this project adheres to [Semantic Versioning][semver].
 [0.13.2]: https://github.com/openlawlibrary/taf/compare/v0.13.1...v0.13.2
 [0.13.1]: https://github.com/openlawlibrary/taf/compare/v0.13.0...v0.13.1
 [0.13.0]: https://github.com/openlawlibrary/taf/compare/v0.12.0...v0.13.0
-[0.12.0]: https://github.com/openlawlibrary/taf/compare/v0.11.2...v0.12.0
-[0.11.1]: https://github.com/openlawlibrary/taf/compare/v0.11.1...v0.11.2
+[0.12.0]: https://github.com/openlawlibrary/taf/compare/v0.11.1...v0.12.0
 [0.11.1]: https://github.com/openlawlibrary/taf/compare/v0.11.0...v0.11.1
 [0.11.0]: https://github.com/openlawlibrary/taf/compare/v0.10.1...v0.11.0
 [0.10.1]: https://github.com/openlawlibrary/taf/compare/v0.10.0...v0.10.1

taf/auth_repo.py

@@ -3,7 +3,7 @@
 import tempfile
 import fnmatch
 
-from typing import Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional, Union
 from collections import defaultdict
 from contextlib import contextmanager
 from pathlib import Path
@@ -36,6 +36,7 @@ def __init__(
         conf_directory_root: Optional[str] = None,
         out_of_band_authentication: Optional[str] = None,
         path: Optional[Union[Path, str]] = None,
+        alias: Optional[str] = None,
         *args,
         **kwargs,
     ):
@@ -51,6 +52,7 @@ def __init__(
           custom (dict): a dictionary containing other data
           default_branch (str): repository's default branch, automatically determined if not specified
           out_of_band_authentication (str): manually specified initial commit
+          alias: Repository's alias, which will be used in logging statements to reference it
         """
         super().__init__(
             library_dir,
@@ -60,6 +62,7 @@ def __init__(
             default_branch,
             allow_unsafe,
             path,
+            alias,
             *args,
             **kwargs,
         )
@@ -142,6 +145,8 @@ def last_validated_commit(self) -> Optional[str]:
 
     @property
     def log_prefix(self) -> str:
+        if self.alias:
+            return f"{self.alias}: "
         return f"Auth repo {self.name}: "
 
     def get_target(self, target_name, commit=None, safely=True) -> Optional[Dict]:
@@ -213,14 +218,69 @@ def set_last_validated_commit(self, commit: str):
         self._log_debug(f"setting last validated commit to: {commit}")
         Path(self.conf_dir, self.LAST_VALIDATED_FILENAME).write_text(commit)
 
+    def targets_data_by_auth_commits(
+        self,
+        commits: List[str],
+        target_repos: Optional[List[str]] = None,
+        custom_fns: Optional[Dict[str, Callable]] = None,
+        default_branch: Optional[str] = None,
+        excluded_target_globs: Optional[List[str]] = None,
+    ) -> Dict[str, Dict[str, Dict[str, Any]]]:
+        """
+        Return a dictionary where each target repository has associated authentication commits,
+        and for each authentication commit, there's a dictionary of the branch, commit and custom data.
+
+        {
+            'target_repo1': {
+                'auth_commit1': {'branch': 'branch1', 'commit': 'commit1', 'custom': {}},
+                'auth_commit2': {'branch': 'branch1', 'commit': 'commit2', 'custom': {}},
+                ...
+            },
+            'target_repo2': {
+                ...
+            },
+            ...
+        }
+
+        """
+        repositories_commits: Dict[str, Dict[str, Dict[str, Any]]] = {}
+        targets = self.targets_at_revisions(
+            *commits, target_repos=target_repos, default_branch=default_branch
+        )
+        excluded_target_globs = excluded_target_globs or []
+        for commit in commits:
+            for target_path, target_data in targets[commit].items():
+                if any(
+                    fnmatch.fnmatch(target_path, excluded_target_glob)
+                    for excluded_target_glob in excluded_target_globs
+                ):
+                    continue
+
+                target_branch = target_data.get("branch")
+                target_commit = target_data.get("commit")
+                target_data.setdefault("custom", {})
+                if custom_fns is not None and target_path in custom_fns:
+                    target_data["custom"].update(custom_fns[target_path](target_commit))
+
+                repositories_commits.setdefault(target_path, {})[commit] = {
+                    "branch": target_branch,
+                    "commit": target_commit,
+                    "custom": target_data.get("custom"),
+                }
+
+        self._log_debug(
+            f"new commits per repositories according to target files: {repositories_commits}"
+        )
+        return repositories_commits
+
     def sorted_commits_and_branches_per_repositories(
         self,
         commits: List[str],
         target_repos: Optional[List[str]] = None,
         custom_fns: Optional[Dict[str, Callable]] = None,
         default_branch: Optional[str] = None,
         excluded_target_globs: Optional[List[str]] = None,
-    ) -> Dict[str, Dict[str, List[Dict]]]:
+    ) -> Dict[str, Dict[str, List[Dict[str, Any]]]]:
         """Return a dictionary consisting of branches and commits belonging
         to it for every target repository:
         {

taf/git.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+import datetime
 import json
 import itertools
 import os
@@ -36,6 +37,7 @@ def __init__(
         default_branch: Optional[str] = None,
         allow_unsafe: Optional[bool] = False,
         path: Optional[Union[Path, str]] = None,
+        alias: Optional[str] = None,
         *args,
         **kwargs,
     ):
@@ -52,6 +54,7 @@ def __init__(
           default_branch (str): repository's default branch, automatically determined if not specified
           allow_unsafe: allow a git's security mechanism which prevents execution of git commands if
           the containing directory is owned by a different user to be ignored
+          alias: Repository's alias, which will be used in logging statements to reference it
         """
         if isinstance(library_dir, str):
             library_dir = Path(library_dir)
@@ -62,7 +65,6 @@ def __init__(
             raise InvalidRepositoryError(
                 "Both library_dir and name need to be specified"
             )
-
         if name is not None and library_dir is not None:
             self.name = self._validate_repo_name(name)
             self.path = self._validate_repo_path(library_dir, name, path)
@@ -88,6 +90,7 @@ def __init__(
             self.library_dir = self.library_dir.resolve()
             self.path = self._validate_repo_path(path)
 
+        self.alias = alias
         self.urls = self._validate_urls(urls)
         self.allow_unsafe = allow_unsafe
         self.custom = custom or {}
@@ -167,6 +170,8 @@ def initial_commit(self) -> str:
 
     @property
     def log_prefix(self) -> str:
+        if self.alias:
+            return f"{self.alias}: "
         return f"Repo {self.name}: "
 
     @property
@@ -241,7 +246,12 @@ def _get_default_branch_from_local(self) -> str:
         branch = self._git("symbolic-ref HEAD --short", reraise_error=True)
         return branch
 
-    def _get_default_branch_from_remote(self, url: str) -> str:
+    def _get_default_branch_from_remote(self, url: str) -> Optional[str]:
+        if not self.is_git_repository:
+            self._log_debug(
+                "Repository does not exist. Could not determined default branch from remote"
+            )
+            return None
         branch = self._git(
             f"ls-remote --symref {url} HEAD",
             log_error=True,
@@ -283,7 +293,7 @@ def all_commits_on_branch(
             )
         if branch:
             branch_obj = repo.branches.get(branch)
-            if branch is None:
+            if branch_obj is None:
                 raise GitError(
                     self,
                     message=f"Error occurred while getting commits of branch {branch}. Branch does not exist",
@@ -307,8 +317,9 @@ def all_commits_since_commit(
         specified or currently checked out branch
 
         Raises:
-            exceptions.GitError: An error occured with provided commit SHA
+            exceptions.GitError: An error occurred with provided commit SHA
         """
+
         if since_commit is None:
             return self.all_commits_on_branch(branch=branch, reverse=reverse)
 
@@ -798,6 +809,20 @@ def delete_remote_branch(
             remote = self.remotes[0]
         self._git(f"push {remote} --delete {branch_name}", log_error=True)
 
+    def get_commit_date(self, commit_sha: str) -> str:
+        """Returns commit date of the given commit"""
+        repo = self.pygit_repo
+        if repo is None:
+            raise GitError(
+                "Could not get commit message. pygit repository could not be instantiated."
+            )
+        commit = repo.get(commit_sha)
+        date = datetime.datetime.utcfromtimestamp(
+            commit.commit_time + commit.commit_time_offset
+        )
+        formatted_date = date.strftime("%Y-%m-%d")
+        return formatted_date
+
     def get_commit_message(self, commit_sha: str) -> str:
         """Returns commit message of the given commit"""
         repo = self.pygit_repo
@@ -812,7 +837,7 @@ def get_commit_sha(self, behind_head: str) -> str:
         """Get commit sha of HEAD~{behind_head}"""
         return self._git("rev-parse HEAD~{}", behind_head)
 
-    def get_default_branch(self, url: Optional[str] = None) -> str:
+    def get_default_branch(self, url: Optional[str] = None) -> Optional[str]:
         """Get the default branch of the repository. If url is provided, return the
         default branch from the remote. Otherwise, return the default
         branch from the local repository."""
@@ -1290,7 +1315,9 @@ def _determine_default_branch(self) -> Optional[str]:
         # try to get the default branch from the local repository
         errors = []
         try:
-            return self.get_default_branch()
+            branch = self.get_default_branch()
+            if branch is not None:
+                return branch
         except GitError as e:
             errors.append(e)
             pass

taf/tests/test_api/test_create_repository.py

@@ -120,4 +120,5 @@ def test_create_repository_when_add_repositories_json(
     for role in ("targets", "delegated_role", "inner_role"):
         assert role in targets_roles
     check_if_targets_signed(auth_repo, "targets", "repositories.json", "mirrors.json")
-    validate_repository(repo_path)
+    # we are not validating target repositories, just the authentication repository
+    validate_repository(repo_path, excluded_target_globs="*")