build: add build step for buck2 (#3312)

* chore(flake): bump deps

* chore: add buck2 prelude

* chore: add custom-third-party/node/turbo

* refactor: shims -> shim

* chore: add npm_bin to simple toolchain

* refactor: some defaults for simple macros

* chore: use turbo_bin as turbo build name

* chore: starting Tiltfile

* chore: begin tsc_build

* fix: node_modules_path arg

* fix: put node modules in right dir

* chore: bump pnpm-lock

* build: add compile_typescript to Workspace provider

* build: use workspace aware node_modules (wip)

* fix: put resolutions in root package.json

* fix: add dummy workspace to turbo

* fix: add missing @types deps

* build: more deps to build_node_modules

* build: add more identifiers

* build: add runnable_tsc_build macro

* build: add runnable_tsc_build_bin macro

* fix: correct run path for 'runnable_tsc_build_bin'

* build: additional_dist_files to add protos

* chore: bump flake.lock

* chore: bump pnpm-lock.yaml

* build: wip getting nix to drive buck

* fix: add '__impure = true' to allow node_modules dep fetch

* fix: correct buck2 result paths in buildPhase

* build: get path patching to work

* build: fix path substitution

* chore: flake works

* chore: wip build nix in docker

* chore: add trigger and ws dockerfiles

* fix: yarn.lock

* chore: bump pnpm-lock.yaml

* chore: remove comments

* chore: use buck2 docker files in quickstart

* fix(quickstart): relative path in dockerfile

---------

Co-authored-by: vindard <[email protected]>

.buckconfig

@@ -0,0 +1,21 @@
+[repositories]
+root = .
+prelude = prelude
+toolchains = toolchains
+none = none
+
+[repository_aliases]
+config = prelude
+fbcode = none
+fbsource = none
+buck = none
+
+[parser]
+target_platform_detector_spec = target:root//...->prelude//platforms:default
+
+[project]
+ignore = \
+  .git, \
+  core/**/node_modules, \
+  shim/custom-third-party/node/**/node_modules
+

.editorconfig

@@ -0,0 +1,63 @@
+root = true
+
+[*]
+charset = utf-8
+tab_width = 4
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.nix]
+indent_style = space
+indent_size = 2
+tab_width = 2
+max_line_length = 80
+
+[*.rs]
+indent_style = space
+tab_width = 4
+insert_final_newline = true
+end_of_line = lf
+
+[*.sh]
+indent_style = space
+indent_size = 2
+max_line_length = 80
+binary_next_line = true
+switch_case_indent = true
+
+[*.py]
+indent_style = space
+indent_size = 4
+tab_width = 4
+max_line_length = 80
+
+[BUCK]
+indent_style = space
+indent_size = 4
+tab_width = 4
+max_line_length = 100
+
+[*.bzl]
+indent_style = space
+indent_size = 4
+tab_width = 4
+max_line_length = 100
+
+[*.bxl]
+indent_style = space
+indent_size = 4
+tab_width = 4
+max_line_length = 100
+
+[Tiltfile]
+indent_style = space
+indent_size = 4
+tab_width = 4
+max_line_length = 100
+
+[Dockerfile]
+indent_style = space
+indent_size = 4
+max_line_length = 80
+

.github/workflows/buck2-build.yaml

@@ -0,0 +1,17 @@
+name: "Buck2 build"
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  buck2-build:
+    name: Buck2 Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v22
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Buck2 build
+        run: nix develop -c buck2 build //core/api:api

.gitignore

@@ -46,3 +46,5 @@ junit.xml
 
 .direnv
 .env.local
+
+buck-out

BUCK

@@ -0,0 +1,9 @@
+load("@toolchains//workspace-pnpm:macros.bzl", "pnpm_workspace")
+
+pnpm_workspace(
+  name = "workspace",
+  child_packages = [
+    "//core/api:package.json"
+  ],
+  visibility = ["PUBLIC"],
+)

core/api-trigger/BUCK

@@ -0,0 +1,15 @@
+load(
+  "@toolchains//workspace-pnpm:macros.bzl",
+  "runnable_tsc_build_bin",
+)
+
+alias(
+    name = "runnable_build",
+    actual = "//core/api:runnable_build",
+)
+
+runnable_tsc_build_bin(
+  name = "api-trigger",
+  preload_file = "services/tracing.js",
+  run_file = "servers/trigger.js",
+)

core/api-trigger/Dockerfile

@@ -0,0 +1,32 @@
+FROM nixos/nix:latest AS builder
+ARG APP=api-trigger
+
+COPY . /workdir
+WORKDIR /workdir
+
+RUN set -eux; \
+    nix \
+    --extra-experimental-features "nix-command flakes impure-derivations ca-derivations" \
+    --option filter-syscalls false \
+    build \
+    ".#$APP";
+
+RUN mkdir -p /tmp/nix-store-closure /tmp/local-bin
+RUN cp -R $(nix-store --query --requisites result/) /tmp/nix-store-closure
+RUN ln -snf $(nix-store --query result/)/bin/* /tmp/local-bin/
+
+FROM gcr.io/distroless/static-debian11 AS final
+ARG APP=api-trigger
+
+WORKDIR /app/$APP
+COPY --from=builder /tmp/nix-store-closure /nix/store
+COPY --from=builder /tmp/local-bin/* /usr/local/bin/
+
+USER 1000
+
+ARG COMMITHASH
+ENV COMMITHASH ${COMMITHASH}
+
+CMD [ \
+    "/usr/local/bin/run" \
+    ]

core/api-ws-server/BUCK

@@ -0,0 +1,15 @@
+load(
+  "@toolchains//workspace-pnpm:macros.bzl",
+  "runnable_tsc_build_bin",
+)
+
+alias(
+    name = "runnable_build",
+    actual = "//core/api:runnable_build",
+)
+
+runnable_tsc_build_bin(
+  name = "api-ws-server",
+  preload_file = "services/tracing.js",
+  run_file = "servers/ws-server.js",
+)

core/api-ws-server/Dockerfile

@@ -0,0 +1,32 @@
+FROM nixos/nix:latest AS builder
+ARG APP=api-ws-server
+
+COPY . /workdir
+WORKDIR /workdir
+
+RUN set -eux; \
+    nix \
+    --extra-experimental-features "nix-command flakes impure-derivations ca-derivations" \
+    --option filter-syscalls false \
+    build \
+    ".#$APP";
+
+RUN mkdir -p /tmp/nix-store-closure /tmp/local-bin
+RUN cp -R $(nix-store --query --requisites result/) /tmp/nix-store-closure
+RUN ln -snf $(nix-store --query result/)/bin/* /tmp/local-bin/
+
+FROM gcr.io/distroless/static-debian11 AS final
+ARG APP=api-ws-server
+
+WORKDIR /app/$APP
+COPY --from=builder /tmp/nix-store-closure /nix/store
+COPY --from=builder /tmp/local-bin/* /usr/local/bin/
+
+USER 1000
+
+ARG COMMITHASH
+ENV COMMITHASH ${COMMITHASH}
+
+CMD [ \
+    "/usr/local/bin/run" \
+    ]

core/api/BUCK

@@ -0,0 +1,56 @@
+load(
+  "@toolchains//workspace-pnpm:macros.bzl",
+  "build_node_modules",
+  "tsc_build",
+  "runnable_tsc_build",
+  "runnable_tsc_build_bin",
+)
+
+export_file(
+  name = "package.json",
+  visibility = ["PUBLIC"],
+)
+
+build_node_modules(
+  name = "node_modules",
+)
+
+build_node_modules(
+  name = "node_modules_prod",
+  prod_only = True,
+)
+
+filegroup(
+    name = "src",
+    srcs = glob([
+        "src/**",
+        "tsconfig.json",
+        "tsconfig-build.json",
+    ]),
+)
+
+filegroup(
+  name = "protos",
+  srcs = glob([
+    "src/services/**/protos/**",
+    "src/services/**/proto/**"
+  ])
+)
+
+tsc_build(
+  name = "build",
+  tsconfig = "tsconfig-build.json",
+  srcs = [":src"],
+  additional_dist_files = [":protos"]
+)
+
+runnable_tsc_build(
+  name = "runnable_build",
+  visibility = ["PUBLIC"],
+)
+
+runnable_tsc_build_bin(
+  name = "api",
+  preload_file = "services/tracing.js",
+  run_file = "servers/graphql-main-server.js",
+)

core/api/Dockerfile

@@ -1,33 +1,32 @@
-FROM node:20-alpine AS BUILD_IMAGE
+FROM nixos/nix:latest AS builder
+ARG APP=api
 
-WORKDIR /app
+COPY . /workdir
+WORKDIR /workdir
 
-RUN apk update && apk add git
+RUN set -eux; \
+    nix \
+        --extra-experimental-features "nix-command flakes impure-derivations ca-derivations" \
+        --option filter-syscalls false \
+        build \
+        ".#$APP";
 
-COPY ./*.json ./yarn.lock ./
+RUN mkdir -p /tmp/nix-store-closure /tmp/local-bin
+RUN cp -R $(nix-store --query --requisites result/) /tmp/nix-store-closure
+RUN ln -snf $(nix-store --query result/)/bin/* /tmp/local-bin/
 
-RUN yarn install --frozen-lockfile
+FROM gcr.io/distroless/static-debian11 AS final
+ARG APP=api
 
-COPY ./src ./src
-
-RUN yarn build
-
-RUN yarn install --frozen-lockfile --production
-
-RUN touch .env
-
-FROM gcr.io/distroless/nodejs20-debian11
-COPY --from=BUILD_IMAGE /app/.env /app/.env
-COPY --from=BUILD_IMAGE /app/lib /app/lib
-COPY --from=BUILD_IMAGE /app/src/config/locales /app/lib/config/locales
-COPY --from=BUILD_IMAGE /app/node_modules /app/node_modules
-
-WORKDIR /app
-COPY ./*.js ./package.json ./tsconfig.json ./yarn.lock ./
+WORKDIR /app/$APP
+COPY --from=builder /tmp/nix-store-closure /nix/store
+COPY --from=builder /tmp/local-bin/* /usr/local/bin/
 
 USER 1000
 
 ARG COMMITHASH
 ENV COMMITHASH ${COMMITHASH}
 
-CMD ["lib/servers/graphql-main-server.js"]
+CMD [ \
+  "/usr/local/bin/run" \
+]

core/api/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "galoy",
+  "name": "api",
   "scripts": {
     "tsc-check-noimplicitany": "tsc --noEmit -p tsconfig.no-implicit-any.json",
     "tsc-check": "tsc --noEmit -p tsconfig.d.json && tsc --noEmit",
@@ -127,6 +127,10 @@
     "@types/babel__core": "^7.20.2",
     "@types/babel__preset-env": "^7.9.3",
     "@types/basic-auth": "^1.1.4",
+    "@types/body-parser": "^1.19.3",
+    "@types/cookie": "^0.5.2",
+    "@types/cookie-parser": "^1.4.4",
+    "@types/cors": "^2.8.14",
     "@types/dedent": "^0.7.0",
     "@types/eslint": "^8.44.2",
     "@types/express": "^4.17.15",
@@ -149,6 +153,8 @@
     "@types/node-jose": "^1.1.10",
     "@types/nodemon": "^1.19.2",
     "@types/react": "^18.2.21",
+    "@types/set-cookie-parser": "^2.4.2",
+    "@types/ws": "^8.5.6",
     "@typescript-eslint/eslint-plugin": "^6.7.0",
     "@typescript-eslint/parser": "^6.7.2",
     "@xascode/tscpaths": "0.1.4",