diff --git a/.gitignore b/.gitignore index 5069aef93..7addf6024 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ cabal.project.local* # generated files /openapi.json /axios-bindings/ + +# Database dumps +*.sql diff --git a/README.md b/README.md index 67536cc0c..3107ac2d6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,123 @@ repository contains Haskell implementations of: * a database adapter for storing Primer programs; and * a web service. +# Running Primer + +## Local development + +To run Primer for local development, you'll need a PostgreSQL instance +to develop against. The most straightforward way to do this is via +some scripts that are included in this repo. Each script can be run +via the `nix run .#script-name` command, where `script-name` should be +replaced by one of the scripts described below. + +The first time you want to do local Primer development on a particular +system, you'll need to run the following commands from the top-level +directory in this repo: + +```sh +nix run .#deploy-postgresql-container +nix run .#create-local-db +``` + +In general, you should only need to run that sequence the first time +you do any Primer development on a new development machine, or if you +want to start over with a completely new PostgreSQL container for some +reason. The container & database those commands create will persist +across reboots, and will remain on your system until you delete them. + +Your usual Primer development workflow will look something like this: + +```sh +nix run .#run-primer +``` + +Note that you'll also need to run the `start-postgresql-container` +command if the `primer-postgres` container is not already running. +Typically, this will only happen after a reboot, or if you've manually +stopped the container. To determine whether the container is running, +use this command from the project's Nix shell: + +```sh +docker --context colima-primer ps +``` + +If it's running, you should see something like this: + +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +a818e6d5f3ef postgres:13.4-alpine3.14 "docker-entrypoint.s…" 29 hours ago Up 2 minutes 0.0.0.0:5432->5432/tcp, :::5432->5432/tcp postgres-primer +``` + +The details of each script in this repo follow: + +### `deploy-postgresql-container` + +This script does the following: + +* Uses [colima](https://github.com/abiosoft/colima) to configure a + Docker-compatible Linux container runtime on your system. +* Downloads the official PostgreSQL Docker image for the version of + PostgreSQL that we support. +* Creates a persistent Docker volume named `postgres-primer` to ensure + the database is preserved across container restarts and upgrades. +* Creates a Docker container that runs PostgreSQL and listens on + `localhost:5432`. + +Note that you do *not* need to install or run Docker in order to use +this or any other script in this repo, as Colima provides the required +container functionality. The scripts do use the `docker` command-line +utility, but only to manage the container, images, and persistent +volumes. + +If you're already running Docker, Colima works alongside it without +conflict. The scripts in this repo will run all containers in a +separate `colima-primer` Docker context, in order to keep the Primer +development environment from affecting any other Docker contexts you +may be using. + +### `start-postgresql-container` + +This script starts the `primer-postgres` container, assuming that it's +previously been deployed by the `deploy-postgresql-container` command. +The container will keep running until you reboot your host machine, or +you stop the container yourself. + +### `stop-postgresql-container` + +This script stops the `primer-postgres` container. + +### `create-local-db` + +This script creates the `primer` database in the local PostgreSQL +instance. This database must have been created before Primer can +connect to it. However, Primer is responsible for creating the schema +in the database, and it will do this automatically if the database is +initially empty. + +### `run-primer` + +This script runs the Primer service and connects to the local +PostgreSQL database. This is the script you'll run most often while +hacking on Primer. + +### Helper scripts + +* `delete-local-db` drops the Primer database from the local + PostgreSQL instance. **Warning**: this script will delete all of the + Primer programs in your local database. + +* `dump-local-db` dumps the Primer database from the local PostgreSQL + instance. It's mainly useful in combination with the + `restore-local-db` script. + +* `restore-local-db` restores a Primer database dump by dropping the + existing Primer database (**warning**: the existing database + contents will not be saved!), creating a fresh, empty Primer + database, and then loading the dump into the new database. This + script is most useful in combination with the `dump-local-db` + script. + # Generating Axios bindings We can automatically generate TypeScript Axios bindings for diff --git a/flake.lock b/flake.lock index ffbb7f153..c0ad138a4 100644 --- a/flake.lock +++ b/flake.lock @@ -85,11 +85,11 @@ }, "emacs-overlay": { "locked": { - "lastModified": 1642012880, - "narHash": "sha256-TOjm/NVua9SC7t+qi5AWBMwH2J3Sz5jrQBEqw8K+krk=", + "lastModified": 1642790458, + "narHash": "sha256-I3vriJApvy0LhwOpnTTQVvEpve1L7ple4vw/8vvYMOM=", "owner": "nix-community", "repo": "emacs-overlay", - "rev": "5db3fa544f264e5b5a11162475228446498827b2", + "rev": "fe974d5484e81c93c8b574f7b3f208334a322eb1", "type": "github" }, "original": { @@ -132,11 +132,11 @@ }, "flake-utils": { "locked": { - "lastModified": 1638122382, - "narHash": "sha256-sQzZzAbvKEqN9s0bzWuYmRaA03v40gaJ4+iL1LXjaeI=", + "lastModified": 1642700792, + "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=", "owner": "numtide", "repo": "flake-utils", - "rev": "74f7e4319258e287b0f9cb95426c9853b282730b", + "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba", "type": "github" }, "original": { @@ -147,11 +147,11 @@ }, "flake-utils_2": { "locked": { - "lastModified": 1638122382, - "narHash": "sha256-sQzZzAbvKEqN9s0bzWuYmRaA03v40gaJ4+iL1LXjaeI=", + "lastModified": 1642700792, + "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=", "owner": "numtide", "repo": "flake-utils", - "rev": "74f7e4319258e287b0f9cb95426c9853b282730b", + "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba", "type": "github" }, "original": { @@ -211,11 +211,11 @@ "hackage": { "flake": false, "locked": { - "lastModified": 1642036369, - "narHash": "sha256-kN36pFv7hJwsW6H2WdZp5iTEbTmsWVhKe75mjYXkjS4=", + "lastModified": 1643073363, + "narHash": "sha256-66oSXQKEDIOSQ2uKAS9facCX/Zuh/jFgyFDtxEqN9sk=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "5eb80fb402a7101029a11c682f825b02495bab7c", + "rev": "4ef9bd3a32316ce236164c7ebff00ebeb33236e2", "type": "github" }, "original": { @@ -235,11 +235,11 @@ "sops-nix": "sops-nix" }, "locked": { - "lastModified": 1642032635, - "narHash": "sha256-C5Eo1xJ7GqcWYlWwN91iIYuATM1a7E/M1oSeZuNdEeU=", + "lastModified": 1643123659, + "narHash": "sha256-Nt3g0c5cnfD6AUYd85KIfpH4A5E0D1N4Kojx9Hnbxyo=", "owner": "hackworthltd", "repo": "hacknix", - "rev": "f2b4a77c80d76f6fe376edc2ec4dc783ad62199d", + "rev": "88671ce85b2d0edd8e427fa59725b7e19d478101", "type": "github" }, "original": { @@ -272,11 +272,11 @@ "stackage": "stackage" }, "locked": { - "lastModified": 1642036517, - "narHash": "sha256-UkJDLRyuaVmDgqbX/VlKafN2b2N7LhS8e38m8QDQAWA=", + "lastModified": 1643073543, + "narHash": "sha256-g2l/KDWzMRTFRugNVcx3CPZeyA5BNcH9/zDiqFpprB4=", "owner": "input-output-hk", "repo": "haskell.nix", - "rev": "3186c23b460df062b8b20781c0f690c0fc084b2c", + "rev": "14f740c7c8f535581c30b1697018e389680e24cb", "type": "github" }, "original": { @@ -416,11 +416,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1641939011, - "narHash": "sha256-a0z5+KgAFkqrDSFI9dl80GpnvtCgWnEHMewASkGrSE0=", + "lastModified": 1642492175, + "narHash": "sha256-lo5hxiY8U4eREChAR7JD94zns92hQqmp3wg3qTFolPA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1b29f6fdb8ca9389f5000f479cdec42ceb67e161", + "rev": "9789ed45b759c8f770bcba4b06c1a1c5ec386664", "type": "github" }, "original": { @@ -432,11 +432,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1641939011, - "narHash": "sha256-a0z5+KgAFkqrDSFI9dl80GpnvtCgWnEHMewASkGrSE0=", + "lastModified": 1643087856, + "narHash": "sha256-EreC7gP/T566PDRZjqNoTvByTyvABEpJpWFtyNUDS0Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1b29f6fdb8ca9389f5000f479cdec42ceb67e161", + "rev": "8bd55a6a5ab05942af769c2aa2494044bff7f625", "type": "github" }, "original": { @@ -520,11 +520,11 @@ "stackage": { "flake": false, "locked": { - "lastModified": 1641864028, - "narHash": "sha256-8qrm8lZPqxAJ9YM3aYkip8f/ar10IsOCpj4TT8u33Dw=", + "lastModified": 1643073493, + "narHash": "sha256-5cPd1+i/skvJY9vJO1BhVRPcJObqkxDSywBEppDmb1U=", "owner": "input-output-hk", "repo": "stackage.nix", - "rev": "308844000fafade0754e8c6641d0277768050413", + "rev": "48e1188855ca38f3b7e2a8dba5352767a2f0a8f7", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index ad3c35ab4..d7c39e2c1 100644 --- a/flake.nix +++ b/flake.nix @@ -136,25 +136,115 @@ crossPlatforms = p: [ p.ghcjs ]; }; - # Ensure these scripts (and datafile) get built for all supported - # platforms by overriding their `meta.platforms`. - # Otherwise, they'll only be built for Linux. - - run-primer = (final.writeShellScriptBin "run-primer" - "${final.primer-service}/bin/primer-service serve . ${version} $@").overrideAttrs (drv: { - meta.platforms = final.lib.platforms.all; - }); - - run-primer-local-pgsql = (final.writeShellScriptBin "run-primer-local-pgsql" - "${final.primer-service}/bin/primer-service serve . ${version} --pgsql-url postgres://postgres:foobar@localhost:5432/primer $@").overrideAttrs (drv: { - meta.platforms = final.lib.platforms.all; - }); - - create-local-pgsql-db = (final.writeShellScriptBin "create-local-pgsql-db" - "${final.postgresql}/bin/createdb -h localhost -U postgres primer $@").overrideAttrs - (drv: { - meta.platforms = final.lib.platforms.all; - }); + + # Scripts for running a local PostgreSQL container in + # Docker, and a local primer-service instance. + + dockerContext = "colima-primer"; + postgresImageTag = "postgres:13.4-alpine3.14"; + postgresVolume = "postgres-primer"; + postgresContainer = "postgres-primer"; + postgresPassword = "primer-dev"; + postgresBaseUrl = "postgres://postgres:${postgresPassword}@localhost:5432"; + postgresPrimerUrl = "${postgresBaseUrl}/primer"; + + deploy-postgresql-container = final.writeShellApplication { + name = "deploy-postgresql-container"; + runtimeInputs = with final; [ + colima + docker + ]; + text = '' + colima start --runtime docker --profile primer + docker --context ${dockerContext} pull ${postgresImageTag} + docker volume create postgres-primer + docker --context ${dockerContext} run --detach --name=${postgresContainer} --publish 5432:5432 --volume ${postgresVolume}:/var/lib/postgresql/data -e POSTGRES_PASSWORD="${postgresPassword}" ${postgresImageTag} + ''; + }; + + start-postgresql-container = final.writeShellApplication { + name = "start-postgresql-container"; + runtimeInputs = with final; [ + docker + ]; + text = '' + docker --context ${dockerContext} start ${postgresContainer} + ''; + }; + + stop-postgresql-container = final.writeShellApplication { + name = "stop-postgresql-container"; + runtimeInputs = with final; [ + docker + ]; + text = '' + docker --context ${dockerContext} stop ${postgresContainer} + ''; + }; + + create-local-db = final.writeShellApplication { + name = "create-local-db"; + runtimeInputs = with final; [ + postgresql + ]; + text = '' + psql ${postgresBaseUrl} --command="CREATE DATABASE primer;" + ''; + }; + + delete-local-db = final.writeShellApplication { + name = "delete-local-db"; + runtimeInputs = with final; [ + postgresql + ]; + text = '' + psql ${postgresBaseUrl} --command="DROP DATABASE primer;" + ''; + }; + + dump-local-db = final.writeShellApplication { + name = "dump-local-db"; + runtimeInputs = with final; [ + coreutils + postgresql + ]; + text = '' + timestamp=$(date --utc --iso-8601=seconds) + dumpfile="primer-$timestamp.sql" + pg_dump ${postgresPrimerUrl} > "$dumpfile" + echo "Dumped local Primer database to $dumpfile" + ''; + }; + + restore-local-db = final.writeShellApplication { + name = "restore-local-db"; + runtimeInputs = with final; [ + postgresql + ]; + text = '' + if [[ $# -ne 1 ]]; then + echo "usage: restore-local-db db.sql" >&2 + exit 2 + fi + psql ${postgresBaseUrl} --command="DROP DATABASE primer;" || true + psql ${postgresBaseUrl} --command="CREATE DATABASE primer;" + psql ${postgresPrimerUrl} < "$1" + ''; + }; + + run-primer = final.writeShellApplication { + name = "run-primer"; + runtimeInputs = with final; [ + primer-service + ]; + text = '' + DATABASE_URL="${postgresPrimerUrl}" + export DATABASE_URL + primer-service serve . ${version} "$@" + ''; + }; + + # Generate the Primer service OpenAPI 3 spec file. primer-openapi-spec = (final.runCommand "primer-openapi" { } "${final.primer-openapi}/bin/primer-openapi > $out").overrideAttrs @@ -169,7 +259,8 @@ primer-service = primerFlake.packages."primer-service:exe:primer-service"; primer-openapi = primerFlake.packages."primer-service:exe:primer-openapi"; - inherit run-primer run-primer-local-pgsql create-local-pgsql-db primer-openapi-spec; + inherit deploy-postgresql-container start-postgresql-container stop-postgresql-container; + inherit run-primer create-local-db delete-local-db dump-local-db restore-local-db primer-openapi-spec; } ) ]; @@ -289,12 +380,11 @@ in { packages = - (hacknix.lib.flakes.filterPackagesByPlatform system - ({ - inherit (pkgs) primer-service; - inherit (pkgs) run-primer run-primer-local-pgsql create-local-pgsql-db primer-openapi-spec; - }) - ) + { + inherit (pkgs) primer-service; + inherit (pkgs) run-primer create-local-db delete-local-db dump-local-db restore-local-db primer-openapi-spec; + inherit (pkgs) deploy-postgresql-container start-postgresql-container stop-postgresql-container; + } // primerFlake.packages; # Notes: @@ -309,7 +399,8 @@ // primerFlake.checks; apps = { - inherit (pkgs) run-primer run-primer-local-pgsql create-local-pgsql-db; + inherit (pkgs) run-primer create-local-db delete-local-db dump-local-db restore-local-db primer-openapi-spec; + inherit (pkgs) deploy-postgresql-container start-postgresql-container stop-postgresql-container; } // primerFlake.apps; @@ -345,6 +436,11 @@ postgresql openapi-generator-cli + # For Docker support. + docker + lima + colima + # For Language Server support. nodejs-16_x ]);