Merge pull request #160 from Tecnativa/conf-available-to-template

Tasks and migrations env utilities

.github/workflows/ci.yml

@@ -26,7 +26,11 @@ jobs:
       - uses: actions/cache@v1
         with:
           path: ~/.cache
-          key: ${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }}
+          key: cache-${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }}
+      - uses: actions/cache@v1
+        with:
+          path: ~/.local
+          key: local-${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }}
       - run: python -m pip install poetry
       - run: poetry install
       - name: Run pre-commit

README.md

@@ -195,21 +195,34 @@ _exclude:
 _skip_if_exists:
 
 # Commands to be executed after the copy
+# They have the $STAGE=task environment variable
 _tasks:
+  # Strings get executed under system's default shell
   - "git init"
-  - "rm [[ name_of_the_project ]]/README.md"
+  - "rm [[ name_of_the_project / 'README.md' ]]"
+  # Arrays are executed without shell, saving you the work of escaping arguments
+  - [invoke, "--search-root=[[ _copier_conf.src_path ]]", after-copy]
+  # You are able to output the full conf to JSON, to be parsed by your script,
+  # but you cannot use the normal `|tojson` filter; instead, use `.json()`
+  - [invoke, end-process, "--full-conf=[[ _copier_conf.json() ]]"]
 
 # Migrations are like tasks, but they are executed:
 # - Evaluated using PEP 440
 # - In the same order as declared here
 # - Only when new version >= declared version > old version
 # - Only when updating
+# - After being rendered with the same renderer as the rest of the template
+# - With the same supported syntaxes as `_tasks`, above
+# - With $VERSION_FROM, $VERSION_TO and $VERSION_CURRENT, $STAGE (before/after)
+#   environment variables
 _migrations:
   - version: v1.0.0
     before:
       - rm ./old-folder
     after:
-      - pre-commit install
+      # [[ _copier_conf.src_path ]] points to the path where the template was
+      # cloned, so it can be helpful to run migration scripts stored there.
+      - invoke -r [[ _copier_conf.src_path ]] -c migrations migrate $VERSION_CURRENT
 
 # Additional paths, from where to search for templates
 _extra_paths:
@@ -303,6 +316,14 @@ Copier includes:
 
 - `now()` to get current UTC time.
 - `make_secret()` to get a random string.
+- `_copier_answers` includes the current answers dict, but slightly modified to make it suitable to [autoupdate your project safely](#the-copier-answers-yml-file):
+  - It doesn't contain secret answers.
+  - It doesn't contain any data that is not easy to render to JSON or YAML.
+- `_copier_conf` includes the current copier `ConfigData` object, also slightly modified:
+  - It only contains JSON-serializable data.
+  - But you have to serialize it with `[[ _copier_conf.json() ]]` instead of `[[ _copier_conf|tojson ]]`.
+  - ⚠️ It contains secret answers inside its `.data` key.
+  - Modifying it doesn't alter the current rendering configuration.
 
 ### Builtin filters
 

copier/config/objects.py

@@ -2,7 +2,7 @@
 from hashlib import sha512
 from os import urandom
 from pathlib import Path
-from typing import Any, Sequence, Tuple
+from typing import Any, Sequence, Tuple, Union
 
 from pydantic import BaseModel, Extra, StrictBool, validator
 
@@ -55,8 +55,8 @@ class Config:
 
 class Migrations(BaseModel):
     version: str
-    before: StrSeq = ()
-    after: StrSeq = ()
+    before: Sequence[Union[str, StrSeq]] = ()
+    after: Sequence[Union[str, StrSeq]] = ()
 
 
 class ConfigData(BaseModel):
@@ -66,7 +66,7 @@ class ConfigData(BaseModel):
     extra_paths: PathSeq = ()
     exclude: StrOrPathSeq = DEFAULT_EXCLUDE
     skip_if_exists: StrOrPathSeq = ()
-    tasks: StrSeq = ()
+    tasks: Sequence[Union[str, StrSeq]] = ()
     envops: EnvOps = EnvOps()
     templates_suffix: str = DEFAULT_TEMPLATES_SUFFIX
     original_src_path: OptStr

copier/main.py

@@ -2,11 +2,12 @@
 import os
 import shutil
 import subprocess
+import sys
 import tempfile
 from pathlib import Path
-from typing import Callable, List, Optional, Tuple
+from typing import Callable, Dict, List, Optional, Sequence, Tuple
 
-from plumbum import local
+from plumbum import colors, local
 from plumbum.cli.terminal import ask
 from plumbum.cmd import git
 
@@ -176,7 +177,9 @@ def copy_local(conf: ConfigData) -> None:
     if not conf.quiet:
         print("")  # padding space
 
-    run_tasks(conf, render, conf.tasks)
+    run_tasks(
+        conf, render, [{"task": t, "extra_env": {"STAGE": "task"}} for t in conf.tasks]
+    )
     if not conf.quiet:
         print("")  # padding space
 
@@ -319,9 +322,18 @@ def overwrite_file(conf: ConfigData, dst_path: Path, rel_path: Path) -> bool:
     return bool(ask(f" Overwrite {dst_path}?", default=True))
 
 
-def run_tasks(conf: ConfigData, render: Renderer, tasks: StrSeq) -> None:
+def run_tasks(conf: ConfigData, render: Renderer, tasks: Sequence[Dict]) -> None:
     for i, task in enumerate(tasks):
-        task = render.string(task)
-        # TODO: should we respect the `quiet` flag here as well?
-        printf(f" > Running task {i + 1} of {len(tasks)}", task, style=Style.OK)
-        subprocess.run(task, shell=True, check=True, cwd=conf.dst_path)
+        task_cmd = task["task"]
+        use_shell = isinstance(task_cmd, str)
+        if use_shell:
+            task_cmd = render.string(task_cmd)
+        else:
+            task_cmd = [render.string(part) for part in task_cmd]
+        if not conf.quiet:
+            print(
+                colors.info | f" > Running task {i + 1} of {len(tasks)}: {task_cmd}",
+                file=sys.stderr,
+            )
+        with local.cwd(conf.dst_path), local.env(**task.get("extra_env", {})):
+            subprocess.run(task_cmd, shell=use_shell, check=True, env=local.env)

copier/tools.py

@@ -3,7 +3,7 @@
 import shutil
 import unicodedata
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 import colorama
 import pathspec
@@ -21,7 +21,6 @@
     JSONSerializable,
     StrOrPath,
     StrOrPathSeq,
-    StrSeq,
     T,
 )
 
@@ -129,7 +128,11 @@ def __init__(self, conf: ConfigData) -> None:
             and isinstance(k, JSONSerializable)
             and isinstance(v, JSONSerializable)
         )
-        self.data = dict(conf.data, _copier_answers=answers)
+        self.data = dict(
+            conf.data,
+            _copier_answers=answers,
+            _copier_conf=conf.copy(deep=True, exclude={"data": {"now", "make_secret"}}),
+        )
         self.env.filters["to_nice_yaml"] = to_nice_yaml
 
     def __call__(self, fullpath: StrOrPath) -> str:
@@ -160,17 +163,26 @@ def match(path: StrOrPath) -> bool:
     return match
 
 
-def get_migration_tasks(conf: ConfigData, stage: str) -> StrSeq:
+def get_migration_tasks(conf: ConfigData, stage: str) -> List[Dict]:
     """Get migration objects that match current version spec.
 
     Versions are compared using PEP 440.
     """
-    result: StrSeq = []
+    result: List[Dict] = []
     if not conf.old_commit or not conf.commit:
         return result
     vfrom = version.parse(conf.old_commit)
     vto = version.parse(conf.commit)
+    extra_env = {
+        "STAGE": stage,
+        "VERSION_FROM": conf.old_commit,
+        "VERSION_TO": conf.commit,
+    }
     for migration in conf.migrations:
         if vto >= version.parse(migration.version) > vfrom:
-            result += migration.dict().get(stage, [])
+            extra_env = dict(extra_env, VERSION_CURRENT=str(migration.version))
+            result += [
+                {"task": task, "extra_env": extra_env}
+                for task in migration.dict().get(stage, [])
+            ]
     return result

pyproject.toml

@@ -60,7 +60,7 @@ target-version = ['py36']
 combine_as_imports = true
 force_grid_wrap = 0
 include_trailing_comma = true
-known_third_party = ["colorama", "jinja2", "packaging", "pathspec", "pkg_resources", "plumbum", "pydantic", "pytest", "six", "yaml", "yamlinclude"]
+known_third_party = ["colorama", "jinja2", "packaging", "pathspec", "pkg_resources", "plumbum", "py", "pydantic", "pytest", "six", "yaml", "yamlinclude"]
 line_length = 88
 multi_line_output = 3  # black interop
 use_parentheses = true

tests/demo_migrations/.copier-answers.yml.tmpl

@@ -0,0 +1,2 @@
+# Changes here will be overwritten by Copier
+[[ _copier_answers|to_nice_yaml ]]

tests/demo_migrations/copier.yaml

@@ -0,0 +1,24 @@
+_exclude:
+  - tasks.sh
+  - migrations.py
+  - .git
+
+_tasks:
+  - "[[ _copier_conf.src_path / 'tasks.sh' ]] 1"
+  - ["[[ _copier_conf.src_path / 'tasks.sh' ]]", 2]
+
+_migrations:
+  # This migration is never executed because it's the 1st version copied, and
+  # migrations are only executed when updating
+  - version: v1.0.0
+    before:
+      - &mig
+        - "[[ _copier_conf.src_path / 'migrations.py' ]]"
+        - "[[ _copier_conf.json() ]]"
+    after:
+      - *mig
+  - version: v2
+    before: [*mig]
+    after:
+      - *mig
+      - "rm delete-in-migration-$VERSION_CURRENT.txt"

tests/demo_migrations/delete-in-migration-v2.txt

@@ -0,0 +1 @@
+This file will be deleted after migrating to v2.

tests/demo_migrations/delete-in-tasks.txt

@@ -0,0 +1 @@
+This file will be deleted in tasks.

tests/demo_migrations/migrations.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+
+NAME = "{VERSION_FROM}-{VERSION_CURRENT}-{VERSION_TO}-{STAGE}.json"
+
+with open(NAME.format(**os.environ), "w") as fd:
+    json.dump(sys.argv, fd)

tests/demo_migrations/tasks.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+echo $STAGE "$@" >> created-with-tasks.txt
+git init
+rm -f delete-in-tasks.txt

tests/test_migrations.py

@@ -0,0 +1,60 @@
+from glob import glob
+from pathlib import Path
+from shutil import copytree
+
+import py
+import yaml
+from plumbum import local
+from plumbum.cmd import git
+
+from copier import copy
+
+from .helpers import PROJECT_TEMPLATE
+
+SRC = Path(f"{PROJECT_TEMPLATE}_migrations").absolute()
+
+
+def test_migrations_and_tasks(tmpdir: py.path.local):
+    """Check migrations and tasks are run properly."""
+    # Convert demo_migrations in a git repository with 2 versions
+    git_src, dst = tmpdir / "src", tmpdir / "dst"
+    copytree(SRC, git_src)
+    with local.cwd(git_src):
+        git("init")
+        git("config", "user.name", "Copier Test")
+        git("config", "user.email", "test@copier")
+        git("add", ".")
+        git("commit", "-m1")
+        git("tag", "v1.0.0")
+        git("commit", "--allow-empty", "-m2")
+        git("tag", "v2.0")
+    # Copy it in v1
+    copy(src_path=str(git_src), dst_path=str(dst), vcs_ref="v1.0.0")
+    # Check copy was OK
+    assert (dst / "created-with-tasks.txt").read() == "task 1\ntask 2\n"
+    assert not (dst / "delete-in-tasks.txt").exists()
+    assert (dst / "delete-in-migration-v2.txt").isfile()
+    assert not (dst / "migrations.py").exists()
+    assert not (dst / "tasks.sh").exists()
+    assert not glob(str(dst / "*-before.txt"))
+    assert not glob(str(dst / "*-after.txt"))
+    answers = yaml.safe_load((dst / ".copier-answers.yml").read())
+    assert answers == {"_commit": "v1.0.0", "_src_path": str(git_src)}
+    # Save changes in downstream repo
+    with local.cwd(dst):
+        git("add", ".")
+        git("config", "user.name", "Copier Test")
+        git("config", "user.email", "test@copier")
+        git("commit", "-m1")
+    # Update it to v2
+    copy(dst_path=str(dst), force=True)
+    # Check update was OK
+    assert (dst / "created-with-tasks.txt").read() == "task 1\ntask 2\n" * 2
+    assert not (dst / "delete-in-tasks.txt").exists()
+    assert not (dst / "delete-in-migration-v2.txt").exists()
+    assert not (dst / "migrations.py").exists()
+    assert not (dst / "tasks.sh").exists()
+    assert (dst / "v1.0.0-v2-v2.0-before.json").isfile()
+    assert (dst / "v1.0.0-v2-v2.0-after.json").isfile()
+    answers = yaml.safe_load((dst / ".copier-answers.yml").read())
+    assert answers == {"_commit": "v2.0", "_src_path": str(git_src)}