diff --git a/runbot_merge/__manifest__.py b/runbot_merge/__manifest__.py index 3688a1e78..2a807c481 100644 --- a/runbot_merge/__manifest__.py +++ b/runbot_merge/__manifest__.py @@ -10,6 +10,7 @@ 'models/crons/git_maintenance.xml', 'models/crons/cleanup_scratch_branches.xml', 'models/crons/issues_closer.xml', + 'models/crons/staging_reifier.xml', 'data/runbot_merge.pull_requests.feedback.template.csv', 'views/res_partner.xml', 'views/runbot_merge_project.xml', diff --git a/runbot_merge/models/crons/__init__.py b/runbot_merge/models/crons/__init__.py index 0d8255a02..7d3506def 100644 --- a/runbot_merge/models/crons/__init__.py +++ b/runbot_merge/models/crons/__init__.py @@ -1,3 +1,4 @@ from . import git_maintenance from . import cleanup_scratch_branches from . import issues_closer +from . import staging_reifier diff --git a/runbot_merge/models/crons/staging_reifier.py b/runbot_merge/models/crons/staging_reifier.py new file mode 100644 index 000000000..a018aeec4 --- /dev/null +++ b/runbot_merge/models/crons/staging_reifier.py @@ -0,0 +1,350 @@ +""" Implements the reification of stagings into cross-repo git commits +""" +from __future__ import annotations + +import abc +import datetime +import itertools +import json +from dataclasses import dataclass +from subprocess import PIPE +from types import SimpleNamespace +from typing import TypedDict + +from odoo import api, fields, models +from ....runbot_merge import git +from ..batch import Batch +from ..pull_requests import Branch, StagingCommits + + +class Project(models.Model): + _inherit = 'runbot_merge.project' + + repo_flat_layout = fields.Char() + repo_pythonpath_layout = fields.Char() + + +class Stagings(models.Model): + _inherit = 'runbot_merge.stagings' + + # can not make these required because we need to have created the staging + # in order to know what its id is (also makes handling of batches easier) + # and NOT NULL can't be deferred + hash_flat_layout = fields.Char() + hash_pythonpath_layout = fields.Char() + + id: int + target: Branch + batch_ids: Batch + commits: StagingCommits + staged_at: datetime.datetime + + +class Repository(models.Model): + _inherit = 'runbot_merge.repository' + + name: str + pythonpath_location = fields.Char( + help="Where the submodule should be located in the pythonpath layout, " + "if empty defaults to the root, unless `pythonpath_link_path` is " + "set, then defaults to `.repos`." + "" + "%(fullname)s, %(owner)s, and %(name)s are available as" + "placeholders in the path", + ) + pythonpath_link_path = fields.Char( + help="Where the repository should be symlinked in the pythonpath layout," + " leave empty to not symlink." + "" + "%(fullname)s, %(owner)s, and %(name)s are available as" + "placeholders in the path", + ) + pythonpath_link_target = fields.Char( + help="Where the symlink should point to inside the repository, " + "leave empty for the root", + ) + +class Reifier(models.Model): + _name = 'runbot_merge.staging.reifier' + _description = "transcriptor of staging into cross-repo git commits" + _order = 'create_date asc, id asc' + + staging_id: Stagings = fields.Many2one('runbot_merge.stagings', required=True) + previous_staging_id: Stagings = fields.Many2one('runbot_merge.stagings') + + @api.model_create_multi + def create(self, vals_list): + self.env.ref('runbot_merge.staging_reifier')._trigger() + return super().create(vals_list) + + def _run(self): + projects = self.env['runbot_merge.project'] + branches = set() + while r := self.search([], limit=1): + reify(r.staging_id, r.previous_staging_id) + projects |= r.staging_id.target.project_id + branches.add(r.staging_id.target) + r.unlink() + self.env.cr.commit() + + for project in projects: + if name_flat := project.repo_flat_layout: + git.get_local(SimpleNamespace( + name=name_flat, + project_id=project, + )).push('origin', 'refs/heads/*:refs/heads/*') + if name_pythonpath := project.repo_pythonpath_layout: + git.get_local(SimpleNamespace( + name=name_pythonpath, + project_id=project, + )).push('origin', 'refs/heads/*:refs/heads/*') +def reify(staging: Stagings, prev: Stagings) -> None: + project = staging.target.project_id + repos = project.repo_ids.having_branch(staging.target) + + repo_flat = None + commit_flat = prev.hash_flat_layout + if name_flat := project.repo_flat_layout: + repo_flat = git.get_local(SimpleNamespace( + name=name_flat, + project_id=project, + )).with_config(check=True, encoding="utf-8", stdout=PIPE) + + repo_pythonpath = None + commit_pythonpath = prev.hash_pythonpath_layout + if name_pythonpath := project.repo_pythonpath_layout: + repo_pythonpath = git.get_local(SimpleNamespace( + name=name_pythonpath, + project_id=project, + )).with_config(check=True, encoding="utf-8", stdout=PIPE) + + if prev: + commits = {c.repository_id.name: c.commit_id.sha for c in prev.commits} + else: + # if there is no `prev` then we need to run the staging in reverse in + # order to get all the "root" commits for the staging, then we need to + # get their parents in order to know what state the repos were in + # before the staging, and *that* lets us figure out the per-batch state + commits = {c.repository_id.name: c.commit_id.sha for c in staging.commits} + + # need to work off of the PR commits, because if no PR touches a + # repository then `commits` could hold *the first commit in the + # repository*, in which case we don't want to remove it + pr_commits = {} + for batch in reversed(staging.batch_ids): + # FIXME: broken if the PR has more than one commit and was rebased... + pr_commits.update( + (pr.repository.name, json.loads(pr.commits_map)['']) + for pr in batch.prs + ) + + for r, c in pr_commits.items(): + repo = git.get_local(SimpleNamespace( + name=r, + project_id=project, + )).with_config(encoding="utf-8", stdout=PIPE) + if parent := repo.rev_parse('--revs-only', c + '~').stdout.strip(): + commits[r] = parent + else: + # if a PR's commit has no parent then the PR itself introduced + # the repo (what?) in which case we don't have the repo at the + # staging and should ignore it + del commits[r] + + for batch in staging.batch_ids: + commits.update( + (pr.repository.name, json.loads(pr.commits_map)['']) + for pr in batch.prs + ) + prs = batch.prs.sorted('repository').mapped('display_name') + meta = json.dumps({ + 'staging': staging.id, + 'batch': batch.id, + 'label': batch.name, + 'pull_requests': prs, + 'commits': commits, + }) + + message = f"{staging.id}.{batch.id}: {batch.name}\n\n- " + "\n- ".join(prs) + if repo_flat: + tree_flat = layout_flat( + repo_flat, {'meta.json': GitBlob(meta)}, repos, commits) + commit_flat = repo_flat.commit_tree( + tree=tree_flat, + message=message, + parents=[commit_flat] if commit_flat else [], + author=(project.github_name, project.github_email, staging.staged_at.isoformat(timespec='seconds')), + committer=(project.github_name, project.github_email, staging.staged_at.isoformat(timespec='seconds')), + ).stdout.strip() + if repo_pythonpath: + tree_pythonpath = layout_pythonpath( + repo_pythonpath, + {'meta.json': GitBlob(meta)}, + repos, + commits, + ) + commit_pythonpath = repo_pythonpath.commit_tree( + tree=tree_pythonpath, + message=message, + parents=[commit_pythonpath] if commit_pythonpath else [], + author=('robodoo', 'robodoo@odoo.com', staging.staged_at.isoformat(timespec='seconds')), + committer=('robodoo', 'robodoo@odoo.com', staging.staged_at.isoformat(timespec='seconds')), + ).stdout.strip() + + if repo_flat: + repo_flat.update_ref( + f'refs/heads/{staging.target.name}', + commit_flat, + prev.hash_flat_layout or '', + ) + if repo_pythonpath: + repo_pythonpath.update_ref( + f'refs/heads/{staging.target.name}', + commit_pythonpath, + prev.hash_pythonpath_layout or '', + ) + staging.write({ + 'hash_flat_layout': commit_flat, + 'hash_pythonpath_layout': commit_pythonpath, + }) + + +def make_submodule(repo: str, path: str) -> str: + _, n = repo.split('/') + return f"""[submodule.{n}] + path = {path} + url = https://github.com/{repo} +""" + +def layout_flat( + repo: git.Repo, + tree_init: dict[str, GitObject], + repos: Repository, + commits: dict[str, str], +) -> str: + gitmodules = "\n".join( + make_submodule(repo.name, repo.name.split('/')[-1]) + for repo in repos + if repo.name in commits + ) + + return GitTree({ + **tree_init, + ".gitmodules": GitBlob(gitmodules), + **{ + repo.name.split('/')[-1]: GitCommit(cc) + for repo in repos + if (cc := commits.get(repo.name)) + } + }).write(repo, 0) + +def layout_pythonpath( + repo: git.Repo, + tree_init: dict[str, GitObject], + repos: Repository, + commits: dict[str, str], +) -> str: + gitmodules = [] + tree = GitTree(tree_init) + for repo_id in repos: + commit = commits.get(repo_id.name) + if commit is None: + continue + + owner, name = repo_id.name.split('/') + tmpl = {'fullname': repo_id.name, 'owner': owner, 'name': name} + if p := repo_id.pythonpath_location: + path = p % tmpl + elif repo_id.pythonpath_link_path: + path = ".repos/%(name)s" % tmpl + else: + path = '%(name)s' % tmpl + link = (repo_id.pythonpath_link_path or '') % tmpl + target = repo_id.pythonpath_link_target or '' + + tree.set(path, GitCommit(commit)) + gitmodules.append(make_submodule(repo_id.name, path)) + + if link: + if target: + target = f"{path}/{target}" + else: + target = path + tree.set(link, GitLink(target)) + + tree.set(".gitmodules", GitBlob("\n".join(gitmodules))) + return tree.write(repo, 0) + +class Commit(TypedDict): + staging_id: int + commit_id: int + repository_id: int + +class GitObject(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def mode(self) -> str: + ... + + @property + @abc.abstractmethod + def type(self) -> str: + ... + + @abc.abstractmethod + def write(self, repo: git.Repo, depth: int) -> str: + ... + + def assert_tree(self) -> GitTree: + assert isinstance(self, GitTree) + return self + +@dataclass +class GitCommit(GitObject): + sha: str + mode = "160000" + type = "commit" + + def write(self, repo: git.Repo, _: int) -> str: + return self.sha + +@dataclass +class GitTree(GitObject): + members: dict[str, GitObject] + mode = "40000" + type = "tree" + + def set(self, path: str, obj: GitObject) -> None: + p, _, target = path.rpartition('/') + assert target, f"{path!r} can't be empty" + if p: + for part in p.split('/'): + self = self.members.setdefault(part, GitTree({})).assert_tree() + self.members[target] = obj + + def write(self, repo: git.Repo, depth: int) -> str: + assert all(self.members), \ + f"One of the tree entries has an empty filename {self.members}" + return repo.with_config(input="".join( + f"{obj.mode} {obj.type} {obj.write(repo, depth+1)}\t{name}\n" + for name, obj in self.members.items() + )).mktree().stdout.strip() + +@dataclass +class GitLink(GitObject): + reference: str + mode = "120000" + type = "blob" + + def write(self, repo: git.Repo, depth: int) -> str: + target = "/".join(itertools.repeat("..", depth)) + "/" + self.reference + return repo.with_config(input=target).hash_object("--stdin", "-w").stdout.strip() + +@dataclass +class GitBlob(GitObject): + content: str + mode = "100644" + type = "blob" + + def write(self, repo: git.Repo, _: int) -> str: + return repo.with_config(input=self.content).hash_object('--stdin', '-w').stdout.strip() diff --git a/runbot_merge/models/crons/staging_reifier.xml b/runbot_merge/models/crons/staging_reifier.xml new file mode 100644 index 000000000..3cc0ac9f3 --- /dev/null +++ b/runbot_merge/models/crons/staging_reifier.xml @@ -0,0 +1,23 @@ + + + Access to staging reifier is useless + + 0 + 0 + 0 + 0 + + + + Reifiy stagings to cross-repo commits + + code + model._run() + + -1 + + + diff --git a/runbot_merge/models/pull_requests.py b/runbot_merge/models/pull_requests.py index 53f7e6b9b..be5b1832d 100644 --- a/runbot_merge/models/pull_requests.py +++ b/runbot_merge/models/pull_requests.py @@ -2435,6 +2435,37 @@ def check_status(self): }) if self.issues_to_close: self.env['runbot_merge.issues_closer'].create(self.issues_to_close) + + # FIXME: error prone, should probably store the previous + # staging on creation instead? + last2 = self.with_context(active_search=False).search([ + ('state', '=', 'success'), + ('target', '=', self.target.id), + ], order='id desc', limit=2) + if len(last2) == 2: + _self, previous = last2 + elif self.target != project.branch_ids[:1]: + _self = last2 + previous = self.search([ + ('state', '=', 'success'), + ('target', '=', project.branch_ids[:1].id), + ('active', '=', False), + ], order='id desc', limit=1) + else: + # no previous branch + _self = last2 + previous = self.browse() + if self == _self: + self.env['runbot_merge.staging.reifier'].create({ + 'previous_staging_id': previous.id, + 'staging_id': self.id, + }) + else: + _logger.warning( + "Got different self (%s) and last success (%s)", + self, + _self, + ) finally: self.write({'active': False}) elif self.state == 'failure' or self.is_timed_out(): diff --git a/runbot_merge/tests/test_staging_reifier.py b/runbot_merge/tests/test_staging_reifier.py new file mode 100644 index 000000000..c09a89dda --- /dev/null +++ b/runbot_merge/tests/test_staging_reifier.py @@ -0,0 +1,95 @@ +import copy +import pprint + +import pytest + +from utils import Commit, to_pr + + +def test_basic(make_repo, project, env, setreviewers, config, users, partners, pytestconfig): + repos = {} + #region project setup + project.repo_pythonpath_layout = make_repo('pythonpath', hooks=False).name + for name, conf in zip('abcd', [ + # flat-style repo (odoo/documentation) + {}, + # flat repo with symlink into (odoo/odoo) + { + 'pythonpath_location': '%(name)s', + 'pythonpath_link_path': 'community/odoo/addons', + 'pythonpath_link_target': 'b', + }, + # modules directory style + {'pythonpath_location': '%(name)s/odoo/addons'}, + # upgrade style + { + 'pythonpath_link_path': '%(name)s/odoo/upgrade', + 'pythonpath_link_target': 'd', + }, + ]): + r = repos[name] = make_repo(name) + env['runbot_merge.repository'].create({ + 'project_id': project.id, + 'name': r.name, + 'required_statuses': 'default', + 'group_id': False, + **conf, + }) + setreviewers(*project.repo_ids) + env['runbot_merge.events_sources'].create([{'repository': r.name} for r in project.repo_ids]) + #endregion + #region repos setup + for repo_name, r in repos.items(): + with r: + r.make_commits( + None, + Commit('initial', tree={ + 'x': '1', + f'{repo_name}/b': '2', + f'{repo_name}/c': '3', + }), + ref='heads/master', + ) + #endregion + #region setup PR + with (r := repos['b']): + [c] = r.make_commits('master', Commit('second', tree={'b/c': '42'}), ref='heads/other') + pr = r.make_pr(target='master', title='title', head='other') + env.run_crons() + with (r := repos['b']): + pr.post_comment('hansen r+', config['role_reviewer']['token']) + r.post_status(c, 'success') + env.run_crons() + # endregion + + pr_id = to_pr(env, pr) + assert not pr_id.blocked + staging = env['runbot_merge.stagings'].search([]) + assert staging + for r in repos.values(): + with r: + r.post_status('staging.master', 'success') + env.run_crons() + assert staging.state == 'success' + assert not staging.active + assert staging.hash_pythonpath_layout + + repo_pythonpath = copy.copy(repos['a']) + repo_pythonpath.name = project.repo_pythonpath_layout + c = repo_pythonpath.commit('master') + t = repo_pythonpath.read_tree(c, recursive=True) + del t['.gitmodules'] + del t['meta.json'] + + def name(repo_name): + return repos[repo_name].name.split('/')[1] + def ref(repo_name): + return '@' + repos[repo_name].commit('master').id + assert t == { + name('a'): ref('a'), + name('b'): ref('b'), + 'community/odoo/addons': f'../../../{name("b")}/b', + f'{name("c")}/odoo/addons': ref('c'), + f'.repos/{name("d")}': ref('d'), + f'{name("d")}/odoo/upgrade': f'../../../.repos/{name("d")}/d', + }