From 2d1f6f38571c6fcc698935a7c249fe7f002eb29e Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Thu, 12 Mar 2020 14:00:19 +0000 Subject: [PATCH 1/2] Tasks and migrations env utilities The rendering context now includes `_copier_conf`, which is a copy of the raw `ConfigData` object, but removing anything not JSON-serializable (a.k.a. `data.now` and `data.make_secret`). This is useful for a number of purposes: 1. Imagine you have migration scripts but you want to store them on your template and NOT in your copies. Then you can use `_copier_conf.src_path` to find the local clone and call anything from there. This fixes #95 and fixes #157. 2. In the execution environment, it includes `$STAGE` (can be `task`, `after` or `before`), and in the case of migrations also `$VERSION_{FROM,TO,CURRENT}` to let migration scripts know where they are in the moment they're called, so we can reuse migration scripts and unleash their full power. Now you can write your scripts in any language you want, and use any task execution system you want. Tests include a bash script and a python script, and docs include how to integrate with Invoke. @Tecnativa TT20357 --- README.md | 25 +++++++- copier/config/objects.py | 8 +-- copier/main.py | 28 ++++++--- copier/tools.py | 24 ++++++-- pyproject.toml | 2 +- .../demo_migrations/.copier-answers.yml.tmpl | 2 + tests/demo_migrations/copier.yaml | 24 ++++++++ .../delete-in-migration-v2.txt | 1 + tests/demo_migrations/delete-in-tasks.txt | 1 + tests/demo_migrations/migrations.py | 9 +++ tests/demo_migrations/tasks.sh | 4 ++ tests/test_migrations.py | 60 +++++++++++++++++++ 12 files changed, 167 insertions(+), 21 deletions(-) create mode 100644 tests/demo_migrations/.copier-answers.yml.tmpl create mode 100644 tests/demo_migrations/copier.yaml create mode 100644 tests/demo_migrations/delete-in-migration-v2.txt create mode 100644 tests/demo_migrations/delete-in-tasks.txt create mode 100755 tests/demo_migrations/migrations.py create mode 100755 tests/demo_migrations/tasks.sh create mode 100644 tests/test_migrations.py diff --git a/README.md b/README.md index 937b4637c..0d0545fa2 100644 --- a/README.md +++ b/README.md @@ -195,21 +195,34 @@ _exclude: _skip_if_exists: # Commands to be executed after the copy +# They have the $STAGE=task environment variable _tasks: + # Strings get executed under system's default shell - "git init" - - "rm [[ name_of_the_project ]]/README.md" + - "rm [[ name_of_the_project / 'README.md' ]]" + # Arrays are executed without shell, saving you the work of escaping arguments + - [invoke, "--search-root=[[ _copier_conf.src_path ]]", after-copy] + # You are able to output the full conf to JSON, to be parsed by your script, + # but you cannot use the normal `|tojson` filter; instead, use `.json()` + - [invoke, end-process, "--full-conf=[[ _copier_conf.json() ]]"] # Migrations are like tasks, but they are executed: # - Evaluated using PEP 440 # - In the same order as declared here # - Only when new version >= declared version > old version # - Only when updating +# - After being rendered with the same renderer as the rest of the template +# - With the same supported syntaxes as `_tasks`, above +# - With $VERSION_FROM, $VERSION_TO and $VERSION_CURRENT, $STAGE (before/after) +# environment variables _migrations: - version: v1.0.0 before: - rm ./old-folder after: - - pre-commit install + # [[ _copier_conf.src_path ]] points to the path where the template was + # cloned, so it can be helpful to run migration scripts stored there. + - invoke -r [[ _copier_conf.src_path ]] -c migrations migrate $VERSION_CURRENT # Additional paths, from where to search for templates _extra_paths: @@ -303,6 +316,14 @@ Copier includes: - `now()` to get current UTC time. - `make_secret()` to get a random string. +- `_copier_answers` includes the current answers dict, but slightly modified to make it suitable to [autoupdate your project safely](#the-copier-answers-yml-file): + - It doesn't contain secret answers. + - It doesn't contain any data that is not easy to render to JSON or YAML. +- `_copier_conf` includes the current copier `ConfigData` object, also slightly modified: + - It only contains JSON-serializable data. + - But you have to serialize it with `[[ _copier_conf.json() ]]` instead of `[[ _copier_conf|tojson ]]`. + - ⚠️ It contains secret answers inside its `.data` key. + - Modifying it doesn't alter the current rendering configuration. ### Builtin filters diff --git a/copier/config/objects.py b/copier/config/objects.py index 6695b7744..0561bd5bb 100644 --- a/copier/config/objects.py +++ b/copier/config/objects.py @@ -2,7 +2,7 @@ from hashlib import sha512 from os import urandom from pathlib import Path -from typing import Any, Sequence, Tuple +from typing import Any, Sequence, Tuple, Union from pydantic import BaseModel, Extra, StrictBool, validator @@ -55,8 +55,8 @@ class Config: class Migrations(BaseModel): version: str - before: StrSeq = () - after: StrSeq = () + before: Sequence[Union[str, StrSeq]] = () + after: Sequence[Union[str, StrSeq]] = () class ConfigData(BaseModel): @@ -66,7 +66,7 @@ class ConfigData(BaseModel): extra_paths: PathSeq = () exclude: StrOrPathSeq = DEFAULT_EXCLUDE skip_if_exists: StrOrPathSeq = () - tasks: StrSeq = () + tasks: Sequence[Union[str, StrSeq]] = () envops: EnvOps = EnvOps() templates_suffix: str = DEFAULT_TEMPLATES_SUFFIX original_src_path: OptStr diff --git a/copier/main.py b/copier/main.py index b2b874820..4646b73b8 100644 --- a/copier/main.py +++ b/copier/main.py @@ -2,11 +2,12 @@ import os import shutil import subprocess +import sys import tempfile from pathlib import Path -from typing import Callable, List, Optional, Tuple +from typing import Callable, Dict, List, Optional, Sequence, Tuple -from plumbum import local +from plumbum import colors, local from plumbum.cli.terminal import ask from plumbum.cmd import git @@ -176,7 +177,9 @@ def copy_local(conf: ConfigData) -> None: if not conf.quiet: print("") # padding space - run_tasks(conf, render, conf.tasks) + run_tasks( + conf, render, [{"task": t, "extra_env": {"STAGE": "task"}} for t in conf.tasks] + ) if not conf.quiet: print("") # padding space @@ -319,9 +322,18 @@ def overwrite_file(conf: ConfigData, dst_path: Path, rel_path: Path) -> bool: return bool(ask(f" Overwrite {dst_path}?", default=True)) -def run_tasks(conf: ConfigData, render: Renderer, tasks: StrSeq) -> None: +def run_tasks(conf: ConfigData, render: Renderer, tasks: Sequence[Dict]) -> None: for i, task in enumerate(tasks): - task = render.string(task) - # TODO: should we respect the `quiet` flag here as well? - printf(f" > Running task {i + 1} of {len(tasks)}", task, style=Style.OK) - subprocess.run(task, shell=True, check=True, cwd=conf.dst_path) + task_cmd = task["task"] + use_shell = isinstance(task_cmd, str) + if use_shell: + task_cmd = render.string(task_cmd) + else: + task_cmd = [render.string(part) for part in task_cmd] + if not conf.quiet: + print( + colors.info | f" > Running task {i + 1} of {len(tasks)}: {task_cmd}", + file=sys.stderr, + ) + with local.cwd(conf.dst_path), local.env(**task.get("extra_env", {})): + subprocess.run(task_cmd, shell=use_shell, check=True, env=local.env) diff --git a/copier/tools.py b/copier/tools.py index 45b4b5fcb..ecebebc47 100644 --- a/copier/tools.py +++ b/copier/tools.py @@ -3,7 +3,7 @@ import shutil import unicodedata from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, Dict, List, Optional, Union import colorama import pathspec @@ -21,7 +21,6 @@ JSONSerializable, StrOrPath, StrOrPathSeq, - StrSeq, T, ) @@ -129,7 +128,11 @@ def __init__(self, conf: ConfigData) -> None: and isinstance(k, JSONSerializable) and isinstance(v, JSONSerializable) ) - self.data = dict(conf.data, _copier_answers=answers) + self.data = dict( + conf.data, + _copier_answers=answers, + _copier_conf=conf.copy(deep=True, exclude={"data": {"now", "make_secret"}}), + ) self.env.filters["to_nice_yaml"] = to_nice_yaml def __call__(self, fullpath: StrOrPath) -> str: @@ -160,17 +163,26 @@ def match(path: StrOrPath) -> bool: return match -def get_migration_tasks(conf: ConfigData, stage: str) -> StrSeq: +def get_migration_tasks(conf: ConfigData, stage: str) -> List[Dict]: """Get migration objects that match current version spec. Versions are compared using PEP 440. """ - result: StrSeq = [] + result: List[Dict] = [] if not conf.old_commit or not conf.commit: return result vfrom = version.parse(conf.old_commit) vto = version.parse(conf.commit) + extra_env = { + "STAGE": stage, + "VERSION_FROM": conf.old_commit, + "VERSION_TO": conf.commit, + } for migration in conf.migrations: if vto >= version.parse(migration.version) > vfrom: - result += migration.dict().get(stage, []) + extra_env = dict(extra_env, VERSION_CURRENT=str(migration.version)) + result += [ + {"task": task, "extra_env": extra_env} + for task in migration.dict().get(stage, []) + ] return result diff --git a/pyproject.toml b/pyproject.toml index 29399a15f..8dc081018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ target-version = ['py36'] combine_as_imports = true force_grid_wrap = 0 include_trailing_comma = true -known_third_party = ["colorama", "jinja2", "packaging", "pathspec", "pkg_resources", "plumbum", "pydantic", "pytest", "six", "yaml", "yamlinclude"] +known_third_party = ["colorama", "jinja2", "packaging", "pathspec", "pkg_resources", "plumbum", "py", "pydantic", "pytest", "six", "yaml", "yamlinclude"] line_length = 88 multi_line_output = 3 # black interop use_parentheses = true diff --git a/tests/demo_migrations/.copier-answers.yml.tmpl b/tests/demo_migrations/.copier-answers.yml.tmpl new file mode 100644 index 000000000..791dccef2 --- /dev/null +++ b/tests/demo_migrations/.copier-answers.yml.tmpl @@ -0,0 +1,2 @@ +# Changes here will be overwritten by Copier +[[ _copier_answers|to_nice_yaml ]] diff --git a/tests/demo_migrations/copier.yaml b/tests/demo_migrations/copier.yaml new file mode 100644 index 000000000..0e9211804 --- /dev/null +++ b/tests/demo_migrations/copier.yaml @@ -0,0 +1,24 @@ +_exclude: + - tasks.sh + - migrations.py + - .git + +_tasks: + - "[[ _copier_conf.src_path / 'tasks.sh' ]] 1" + - ["[[ _copier_conf.src_path / 'tasks.sh' ]]", 2] + +_migrations: + # This migration is never executed because it's the 1st version copied, and + # migrations are only executed when updating + - version: v1.0.0 + before: + - &mig + - "[[ _copier_conf.src_path / 'migrations.py' ]]" + - "[[ _copier_conf.json() ]]" + after: + - *mig + - version: v2 + before: [*mig] + after: + - *mig + - "rm delete-in-migration-$VERSION_CURRENT.txt" diff --git a/tests/demo_migrations/delete-in-migration-v2.txt b/tests/demo_migrations/delete-in-migration-v2.txt new file mode 100644 index 000000000..5aaa317ae --- /dev/null +++ b/tests/demo_migrations/delete-in-migration-v2.txt @@ -0,0 +1 @@ +This file will be deleted after migrating to v2. diff --git a/tests/demo_migrations/delete-in-tasks.txt b/tests/demo_migrations/delete-in-tasks.txt new file mode 100644 index 000000000..87c8f9d00 --- /dev/null +++ b/tests/demo_migrations/delete-in-tasks.txt @@ -0,0 +1 @@ +This file will be deleted in tasks. diff --git a/tests/demo_migrations/migrations.py b/tests/demo_migrations/migrations.py new file mode 100755 index 000000000..6c498e0a0 --- /dev/null +++ b/tests/demo_migrations/migrations.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +import json +import os +import sys + +NAME = "{VERSION_FROM}-{VERSION_CURRENT}-{VERSION_TO}-{STAGE}.json" + +with open(NAME.format(**os.environ), "w") as fd: + json.dump(sys.argv, fd) diff --git a/tests/demo_migrations/tasks.sh b/tests/demo_migrations/tasks.sh new file mode 100755 index 000000000..50b26976b --- /dev/null +++ b/tests/demo_migrations/tasks.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +echo $STAGE "$@" >> created-with-tasks.txt +git init +rm -f delete-in-tasks.txt diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 000000000..243be6c70 --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,60 @@ +from glob import glob +from pathlib import Path +from shutil import copytree + +import py +import yaml +from plumbum import local +from plumbum.cmd import git + +from copier import copy + +from .helpers import PROJECT_TEMPLATE + +SRC = Path(f"{PROJECT_TEMPLATE}_migrations").absolute() + + +def test_migrations_and_tasks(tmpdir: py.path.local): + """Check migrations and tasks are run properly.""" + # Convert demo_migrations in a git repository with 2 versions + git_src, dst = tmpdir / "src", tmpdir / "dst" + copytree(SRC, git_src) + with local.cwd(git_src): + git("init") + git("config", "user.name", "Copier Test") + git("config", "user.email", "test@copier") + git("add", ".") + git("commit", "-m1") + git("tag", "v1.0.0") + git("commit", "--allow-empty", "-m2") + git("tag", "v2.0") + # Copy it in v1 + copy(src_path=str(git_src), dst_path=str(dst), vcs_ref="v1.0.0") + # Check copy was OK + assert (dst / "created-with-tasks.txt").read() == "task 1\ntask 2\n" + assert not (dst / "delete-in-tasks.txt").exists() + assert (dst / "delete-in-migration-v2.txt").isfile() + assert not (dst / "migrations.py").exists() + assert not (dst / "tasks.sh").exists() + assert not glob(str(dst / "*-before.txt")) + assert not glob(str(dst / "*-after.txt")) + answers = yaml.safe_load((dst / ".copier-answers.yml").read()) + assert answers == {"_commit": "v1.0.0", "_src_path": str(git_src)} + # Save changes in downstream repo + with local.cwd(dst): + git("add", ".") + git("config", "user.name", "Copier Test") + git("config", "user.email", "test@copier") + git("commit", "-m1") + # Update it to v2 + copy(dst_path=str(dst), force=True) + # Check update was OK + assert (dst / "created-with-tasks.txt").read() == "task 1\ntask 2\n" * 2 + assert not (dst / "delete-in-tasks.txt").exists() + assert not (dst / "delete-in-migration-v2.txt").exists() + assert not (dst / "migrations.py").exists() + assert not (dst / "tasks.sh").exists() + assert (dst / "v1.0.0-v2-v2.0-before.json").isfile() + assert (dst / "v1.0.0-v2-v2.0-after.json").isfile() + answers = yaml.safe_load((dst / ".copier-answers.yml").read()) + assert answers == {"_commit": "v2.0", "_src_path": str(git_src)} From 4ef0a8bc76d48d4eac0ce7e2a1691fbf2b58855e Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Fri, 13 Mar 2020 13:36:14 +0000 Subject: [PATCH 2/2] Cache ~/.local in CI builds This will make dependencies installation much faster. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bda665d0..48290ad1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,11 @@ jobs: - uses: actions/cache@v1 with: path: ~/.cache - key: ${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }} + key: cache-${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }} + - uses: actions/cache@v1 + with: + path: ~/.local + key: local-${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }} - run: python -m pip install poetry - run: poetry install - name: Run pre-commit