diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bda665d0..48290ad1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,11 @@ jobs: - uses: actions/cache@v1 with: path: ~/.cache - key: ${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }} + key: cache-${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }} + - uses: actions/cache@v1 + with: + path: ~/.local + key: local-${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }} - run: python -m pip install poetry - run: poetry install - name: Run pre-commit diff --git a/README.md b/README.md index 937b4637c..0d0545fa2 100644 --- a/README.md +++ b/README.md @@ -195,21 +195,34 @@ _exclude: _skip_if_exists: # Commands to be executed after the copy +# They have the $STAGE=task environment variable _tasks: + # Strings get executed under system's default shell - "git init" - - "rm [[ name_of_the_project ]]/README.md" + - "rm [[ name_of_the_project / 'README.md' ]]" + # Arrays are executed without shell, saving you the work of escaping arguments + - [invoke, "--search-root=[[ _copier_conf.src_path ]]", after-copy] + # You are able to output the full conf to JSON, to be parsed by your script, + # but you cannot use the normal `|tojson` filter; instead, use `.json()` + - [invoke, end-process, "--full-conf=[[ _copier_conf.json() ]]"] # Migrations are like tasks, but they are executed: # - Evaluated using PEP 440 # - In the same order as declared here # - Only when new version >= declared version > old version # - Only when updating +# - After being rendered with the same renderer as the rest of the template +# - With the same supported syntaxes as `_tasks`, above +# - With $VERSION_FROM, $VERSION_TO and $VERSION_CURRENT, $STAGE (before/after) +# environment variables _migrations: - version: v1.0.0 before: - rm ./old-folder after: - - pre-commit install + # [[ _copier_conf.src_path ]] points to the path where the template was + # cloned, so it can be helpful to run migration scripts stored there. + - invoke -r [[ _copier_conf.src_path ]] -c migrations migrate $VERSION_CURRENT # Additional paths, from where to search for templates _extra_paths: @@ -303,6 +316,14 @@ Copier includes: - `now()` to get current UTC time. - `make_secret()` to get a random string. +- `_copier_answers` includes the current answers dict, but slightly modified to make it suitable to [autoupdate your project safely](#the-copier-answers-yml-file): + - It doesn't contain secret answers. + - It doesn't contain any data that is not easy to render to JSON or YAML. +- `_copier_conf` includes the current copier `ConfigData` object, also slightly modified: + - It only contains JSON-serializable data. + - But you have to serialize it with `[[ _copier_conf.json() ]]` instead of `[[ _copier_conf|tojson ]]`. + - ⚠️ It contains secret answers inside its `.data` key. + - Modifying it doesn't alter the current rendering configuration. ### Builtin filters diff --git a/copier/config/objects.py b/copier/config/objects.py index 6695b7744..0561bd5bb 100644 --- a/copier/config/objects.py +++ b/copier/config/objects.py @@ -2,7 +2,7 @@ from hashlib import sha512 from os import urandom from pathlib import Path -from typing import Any, Sequence, Tuple +from typing import Any, Sequence, Tuple, Union from pydantic import BaseModel, Extra, StrictBool, validator @@ -55,8 +55,8 @@ class Config: class Migrations(BaseModel): version: str - before: StrSeq = () - after: StrSeq = () + before: Sequence[Union[str, StrSeq]] = () + after: Sequence[Union[str, StrSeq]] = () class ConfigData(BaseModel): @@ -66,7 +66,7 @@ class ConfigData(BaseModel): extra_paths: PathSeq = () exclude: StrOrPathSeq = DEFAULT_EXCLUDE skip_if_exists: StrOrPathSeq = () - tasks: StrSeq = () + tasks: Sequence[Union[str, StrSeq]] = () envops: EnvOps = EnvOps() templates_suffix: str = DEFAULT_TEMPLATES_SUFFIX original_src_path: OptStr diff --git a/copier/main.py b/copier/main.py index b2b874820..4646b73b8 100644 --- a/copier/main.py +++ b/copier/main.py @@ -2,11 +2,12 @@ import os import shutil import subprocess +import sys import tempfile from pathlib import Path -from typing import Callable, List, Optional, Tuple +from typing import Callable, Dict, List, Optional, Sequence, Tuple -from plumbum import local +from plumbum import colors, local from plumbum.cli.terminal import ask from plumbum.cmd import git @@ -176,7 +177,9 @@ def copy_local(conf: ConfigData) -> None: if not conf.quiet: print("") # padding space - run_tasks(conf, render, conf.tasks) + run_tasks( + conf, render, [{"task": t, "extra_env": {"STAGE": "task"}} for t in conf.tasks] + ) if not conf.quiet: print("") # padding space @@ -319,9 +322,18 @@ def overwrite_file(conf: ConfigData, dst_path: Path, rel_path: Path) -> bool: return bool(ask(f" Overwrite {dst_path}?", default=True)) -def run_tasks(conf: ConfigData, render: Renderer, tasks: StrSeq) -> None: +def run_tasks(conf: ConfigData, render: Renderer, tasks: Sequence[Dict]) -> None: for i, task in enumerate(tasks): - task = render.string(task) - # TODO: should we respect the `quiet` flag here as well? - printf(f" > Running task {i + 1} of {len(tasks)}", task, style=Style.OK) - subprocess.run(task, shell=True, check=True, cwd=conf.dst_path) + task_cmd = task["task"] + use_shell = isinstance(task_cmd, str) + if use_shell: + task_cmd = render.string(task_cmd) + else: + task_cmd = [render.string(part) for part in task_cmd] + if not conf.quiet: + print( + colors.info | f" > Running task {i + 1} of {len(tasks)}: {task_cmd}", + file=sys.stderr, + ) + with local.cwd(conf.dst_path), local.env(**task.get("extra_env", {})): + subprocess.run(task_cmd, shell=use_shell, check=True, env=local.env) diff --git a/copier/tools.py b/copier/tools.py index 45b4b5fcb..ecebebc47 100644 --- a/copier/tools.py +++ b/copier/tools.py @@ -3,7 +3,7 @@ import shutil import unicodedata from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, Dict, List, Optional, Union import colorama import pathspec @@ -21,7 +21,6 @@ JSONSerializable, StrOrPath, StrOrPathSeq, - StrSeq, T, ) @@ -129,7 +128,11 @@ def __init__(self, conf: ConfigData) -> None: and isinstance(k, JSONSerializable) and isinstance(v, JSONSerializable) ) - self.data = dict(conf.data, _copier_answers=answers) + self.data = dict( + conf.data, + _copier_answers=answers, + _copier_conf=conf.copy(deep=True, exclude={"data": {"now", "make_secret"}}), + ) self.env.filters["to_nice_yaml"] = to_nice_yaml def __call__(self, fullpath: StrOrPath) -> str: @@ -160,17 +163,26 @@ def match(path: StrOrPath) -> bool: return match -def get_migration_tasks(conf: ConfigData, stage: str) -> StrSeq: +def get_migration_tasks(conf: ConfigData, stage: str) -> List[Dict]: """Get migration objects that match current version spec. Versions are compared using PEP 440. """ - result: StrSeq = [] + result: List[Dict] = [] if not conf.old_commit or not conf.commit: return result vfrom = version.parse(conf.old_commit) vto = version.parse(conf.commit) + extra_env = { + "STAGE": stage, + "VERSION_FROM": conf.old_commit, + "VERSION_TO": conf.commit, + } for migration in conf.migrations: if vto >= version.parse(migration.version) > vfrom: - result += migration.dict().get(stage, []) + extra_env = dict(extra_env, VERSION_CURRENT=str(migration.version)) + result += [ + {"task": task, "extra_env": extra_env} + for task in migration.dict().get(stage, []) + ] return result diff --git a/pyproject.toml b/pyproject.toml index 29399a15f..8dc081018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ target-version = ['py36'] combine_as_imports = true force_grid_wrap = 0 include_trailing_comma = true -known_third_party = ["colorama", "jinja2", "packaging", "pathspec", "pkg_resources", "plumbum", "pydantic", "pytest", "six", "yaml", "yamlinclude"] +known_third_party = ["colorama", "jinja2", "packaging", "pathspec", "pkg_resources", "plumbum", "py", "pydantic", "pytest", "six", "yaml", "yamlinclude"] line_length = 88 multi_line_output = 3 # black interop use_parentheses = true diff --git a/tests/demo_migrations/.copier-answers.yml.tmpl b/tests/demo_migrations/.copier-answers.yml.tmpl new file mode 100644 index 000000000..791dccef2 --- /dev/null +++ b/tests/demo_migrations/.copier-answers.yml.tmpl @@ -0,0 +1,2 @@ +# Changes here will be overwritten by Copier +[[ _copier_answers|to_nice_yaml ]] diff --git a/tests/demo_migrations/copier.yaml b/tests/demo_migrations/copier.yaml new file mode 100644 index 000000000..0e9211804 --- /dev/null +++ b/tests/demo_migrations/copier.yaml @@ -0,0 +1,24 @@ +_exclude: + - tasks.sh + - migrations.py + - .git + +_tasks: + - "[[ _copier_conf.src_path / 'tasks.sh' ]] 1" + - ["[[ _copier_conf.src_path / 'tasks.sh' ]]", 2] + +_migrations: + # This migration is never executed because it's the 1st version copied, and + # migrations are only executed when updating + - version: v1.0.0 + before: + - &mig + - "[[ _copier_conf.src_path / 'migrations.py' ]]" + - "[[ _copier_conf.json() ]]" + after: + - *mig + - version: v2 + before: [*mig] + after: + - *mig + - "rm delete-in-migration-$VERSION_CURRENT.txt" diff --git a/tests/demo_migrations/delete-in-migration-v2.txt b/tests/demo_migrations/delete-in-migration-v2.txt new file mode 100644 index 000000000..5aaa317ae --- /dev/null +++ b/tests/demo_migrations/delete-in-migration-v2.txt @@ -0,0 +1 @@ +This file will be deleted after migrating to v2. diff --git a/tests/demo_migrations/delete-in-tasks.txt b/tests/demo_migrations/delete-in-tasks.txt new file mode 100644 index 000000000..87c8f9d00 --- /dev/null +++ b/tests/demo_migrations/delete-in-tasks.txt @@ -0,0 +1 @@ +This file will be deleted in tasks. diff --git a/tests/demo_migrations/migrations.py b/tests/demo_migrations/migrations.py new file mode 100755 index 000000000..6c498e0a0 --- /dev/null +++ b/tests/demo_migrations/migrations.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +import json +import os +import sys + +NAME = "{VERSION_FROM}-{VERSION_CURRENT}-{VERSION_TO}-{STAGE}.json" + +with open(NAME.format(**os.environ), "w") as fd: + json.dump(sys.argv, fd) diff --git a/tests/demo_migrations/tasks.sh b/tests/demo_migrations/tasks.sh new file mode 100755 index 000000000..50b26976b --- /dev/null +++ b/tests/demo_migrations/tasks.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +echo $STAGE "$@" >> created-with-tasks.txt +git init +rm -f delete-in-tasks.txt diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 000000000..243be6c70 --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,60 @@ +from glob import glob +from pathlib import Path +from shutil import copytree + +import py +import yaml +from plumbum import local +from plumbum.cmd import git + +from copier import copy + +from .helpers import PROJECT_TEMPLATE + +SRC = Path(f"{PROJECT_TEMPLATE}_migrations").absolute() + + +def test_migrations_and_tasks(tmpdir: py.path.local): + """Check migrations and tasks are run properly.""" + # Convert demo_migrations in a git repository with 2 versions + git_src, dst = tmpdir / "src", tmpdir / "dst" + copytree(SRC, git_src) + with local.cwd(git_src): + git("init") + git("config", "user.name", "Copier Test") + git("config", "user.email", "test@copier") + git("add", ".") + git("commit", "-m1") + git("tag", "v1.0.0") + git("commit", "--allow-empty", "-m2") + git("tag", "v2.0") + # Copy it in v1 + copy(src_path=str(git_src), dst_path=str(dst), vcs_ref="v1.0.0") + # Check copy was OK + assert (dst / "created-with-tasks.txt").read() == "task 1\ntask 2\n" + assert not (dst / "delete-in-tasks.txt").exists() + assert (dst / "delete-in-migration-v2.txt").isfile() + assert not (dst / "migrations.py").exists() + assert not (dst / "tasks.sh").exists() + assert not glob(str(dst / "*-before.txt")) + assert not glob(str(dst / "*-after.txt")) + answers = yaml.safe_load((dst / ".copier-answers.yml").read()) + assert answers == {"_commit": "v1.0.0", "_src_path": str(git_src)} + # Save changes in downstream repo + with local.cwd(dst): + git("add", ".") + git("config", "user.name", "Copier Test") + git("config", "user.email", "test@copier") + git("commit", "-m1") + # Update it to v2 + copy(dst_path=str(dst), force=True) + # Check update was OK + assert (dst / "created-with-tasks.txt").read() == "task 1\ntask 2\n" * 2 + assert not (dst / "delete-in-tasks.txt").exists() + assert not (dst / "delete-in-migration-v2.txt").exists() + assert not (dst / "migrations.py").exists() + assert not (dst / "tasks.sh").exists() + assert (dst / "v1.0.0-v2-v2.0-before.json").isfile() + assert (dst / "v1.0.0-v2-v2.0-after.json").isfile() + answers = yaml.safe_load((dst / ".copier-answers.yml").read()) + assert answers == {"_commit": "v2.0", "_src_path": str(git_src)}