feat(cu): allow passing GRAPHQL_URL and ARWEAVE_URL separately. Refac…

…tor usages #551
permaweb · Mar 26, 2024 · 53a1d5d · 53a1d5d
1 parent a24fef4
commit 53a1d5d
Show file tree

Hide file tree

Showing 13 changed files with 154 additions and 108 deletions.
diff --git a/servers/cu/README.md b/servers/cu/README.md
@@ -40,8 +40,13 @@ Either command will start a server listening on `PORT` (`6363` by default).
 There are a few environment variables that you can set. Besides `WALLET`/`WALLET_FILE`, they
 each have a default:
 
-- `GATEWAY_URL`: The Arweave gateway for the CU to use fetch block metadata,
-  data on arweave, and Scheduler-Location data (defaults to `arweave.net`)
+- `GATEWAY_URL`: The url of the Arweave gateway to use. (Defaults to `https://arweave.net`)
+
+> `GATEWAY_URL` is solely used as a fallback for both `ARWEAVE_URL` and `GRAPHQL_URL`, if not provided (see below).
+
+- `ARWEAVE_URL`: The url for the Arweave http API server, to be used by the CU to fetch
+transaction data from Arweave, specifically ao `Modules`, and `Message` `Assignment`s. (Defaults to `GATEWAY_URL`)
+- `GRAPHQL_URL`: The url for the Arweave Gateway GraphQL server to be used by the CU. (Defaults to `${GATEWAY_URL}/graphql`)
 - `UPLOADER_URL`: The url of the uploader to use to upload Process `Checkpoints`
   to Arweave. (Defaults to `up.arweave.net`)
 - `WALLET`/`WALLET_FILE`: the JWK Interface stringified JSON that will be used by the CU, or a file to load it from

diff --git a/servers/cu/package-lock.json b/servers/cu/package-lock.json
diff --git a/servers/cu/package.json b/servers/cu/package.json
@@ -14,7 +14,7 @@
   "dependencies": {
     "@fastify/middie": "^8.3.0",
     "@permaweb/ao-loader": "^0.0.22",
-    "@permaweb/ao-scheduler-utils": "^0.0.15",
+    "@permaweb/ao-scheduler-utils": "^0.0.16",
     "arweave": "^1.14.4",
     "cors": "^2.8.5",
     "dataloader": "^2.2.2",

diff --git a/servers/cu/src/config.js b/servers/cu/src/config.js
@@ -2,10 +2,12 @@ import { existsSync, readFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import * as path from 'node:path'
 
+import { pipe } from 'ramda'
 import { z, ZodIssueCode } from 'zod'
 import ms from 'ms'
 
 import { domainConfigSchema, positiveIntSchema } from './domain/index.js'
+import { preprocessUrls } from './domain/utils.js'
 
 /**
  * Some frameworks will implicitly override NODE_ENV
@@ -32,46 +34,46 @@ const serverConfigSchema = domainConfigSchema.extend({
   DUMP_PATH: z.string().min(1)
 })
 
+/**
+ * An Either would be nice here, but just throwing a literal
+ * get's us what we need, for now.
+ */
+/* eslint-disable no-throw-literal */
+
 /**
  * If the WALLET_FILE env var is defined, load the contents from the file.
  * Refuse to boot the app if both or none of WALLET and WALLET_FILE are defined.
  */
-const preprocessedServerConfigSchema = z.preprocess(
-  (envConfig, zodRefinementContext) => {
-    const { WALLET, WALLET_FILE, ...theRestOfTheConfig } = envConfig
+export const preprocessWallet = (envConfig) => {
+  const { WALLET, WALLET_FILE, ...theRestOfTheConfig } = envConfig
 
-    const error = message => zodRefinementContext.addIssue({
-      code: ZodIssueCode.custom,
-      message
-    })
+  // nothing to do here
+  if (!!WALLET && !WALLET_FILE) return envConfig
 
-    if (!!WALLET && !WALLET_FILE) {
-      // nothing to do here
-      return envConfig
-    }
-    if (!WALLET && !WALLET_FILE) {
-      error('One of WALLET or WALLET_FILE is required')
-      return
-    }
-    if (!!WALLET && !!WALLET_FILE) {
-      error('Do not define both WALLET and WALLET_FILE')
-      return
-    }
+  if (!WALLET && !WALLET_FILE) throw 'One of WALLET or WALLET_FILE is required'
+  if (!!WALLET && !!WALLET_FILE) throw 'Do not define both WALLET and WALLET_FILE'
+
+  const walletPath = path.resolve(WALLET_FILE)
+  if (!existsSync(walletPath)) throw `WALLET_FILE does not exist: ${walletPath}`
 
-    const walletPath = path.resolve(WALLET_FILE)
-    if (!existsSync(walletPath)) {
-      error(`WALLET_FILE does not exist: ${walletPath}`)
-      return
+  try {
+    const walletFromFile = readFileSync(walletPath, 'utf8')
+    return {
+      WALLET: walletFromFile,
+      ...theRestOfTheConfig
     }
+  } catch (e) {
+    throw `An error occurred while reading WALLET_FILE from ${walletPath}\n${e}`
+  }
+}
+/* eslint-enable no-throw-literal */
 
+const preprocessedServerConfigSchema = z.preprocess(
+  (envConfig, zodRefinementContext) => {
     try {
-      const walletFromFile = readFileSync(walletPath, 'utf8')
-      return {
-        WALLET: walletFromFile,
-        ...theRestOfTheConfig
-      }
-    } catch (e) {
-      error(`An error occurred while reading WALLET_FILE from ${walletPath}\n${e}`)
+      return pipe(preprocessWallet, preprocessUrls)(envConfig)
+    } catch (message) {
+      zodRefinementContext.addIssue({ code: ZodIssueCode.custom, message })
     }
   },
   serverConfigSchema
@@ -88,6 +90,8 @@ const CONFIG_ENVS = {
     MODE,
     port: process.env.PORT || 6363,
     GATEWAY_URL: process.env.GATEWAY_URL || 'https://arweave.net',
+    GRAPHQL_URL: process.env.GRAPHQL_URL,
+    ARWEAVE_URL: process.env.ARWEAVE_URL,
     UPLOADER_URL: process.env.UPLOADER_URL || 'https://up.arweave.net',
     DB_MODE: process.env.DB_MODE || 'embedded',
     DB_URL: process.env.DB_URL || 'ao-cache',
@@ -115,6 +119,8 @@ const CONFIG_ENVS = {
     MODE,
     port: process.env.PORT || 6363,
     GATEWAY_URL: process.env.GATEWAY_URL || 'https://arweave.net',
+    GRAPHQL_URL: process.env.GRAPHQL_URL,
+    ARWEAVE_URL: process.env.ARWEAVE_URL,
     UPLOADER_URL: process.env.UPLOADER_URL || 'https://up.arweave.net',
     DB_MODE: process.env.DB_MODE || 'embedded',
     DB_URL: process.env.DB_URL || 'ao-cache',

diff --git a/servers/cu/src/domain/client/ao-block.js b/servers/cu/src/domain/client/ao-block.js
@@ -4,7 +4,6 @@ import { z } from 'zod'
 
 import { blockSchema } from '../model.js'
 import { BLOCKS_ASC_IDX } from './pouchdb.js'
-import { joinUrl } from '../utils.js'
 
 const blockDocSchema = z.object({
   _id: z.string().min(1),
@@ -74,7 +73,7 @@ export function findBlocksWith ({ pouchDb }) {
 /**
  * @typedef Env2
  * @property {fetch} fetch
- * @property {string} GATEWAY_URL
+ * @property {string} GRAPHQL_URL
  * @property {number} pageSize
  *
  * @callback LoadBlocksMeta
@@ -84,11 +83,8 @@ export function findBlocksWith ({ pouchDb }) {
  * @param {Env1} env
  * @returns {LoadBlocksMeta}
  */
-export function loadBlocksMetaWith ({ fetch, GATEWAY_URL, pageSize, logger }) {
+export function loadBlocksMetaWith ({ fetch, GRAPHQL_URL, pageSize, logger }) {
   // TODO: create a dataloader and use that to batch load contracts
-
-  const GRAPHQL = joinUrl({ url: GATEWAY_URL, path: '/graphql' })
-
   const GET_BLOCKS_QUERY = `
       query GetBlocks($min: Int!, $limit: Int!) {
         blocks(
@@ -129,7 +125,7 @@ export function loadBlocksMetaWith ({ fetch, GATEWAY_URL, pageSize, logger }) {
           return variables
         })
         .then((variables) =>
-          fetch(GRAPHQL, {
+          fetch(GRAPHQL_URL, {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({
@@ -141,8 +137,7 @@ export function loadBlocksMetaWith ({ fetch, GATEWAY_URL, pageSize, logger }) {
         .then(async (res) => {
           if (res.ok) return res.json()
           logger(
-            'Error Encountered when fetching page of block metadata from gateway \'%s\' with minBlock \'%s\' and maxTimestamp \'%s\'',
-            GATEWAY_URL,
+            'Error Encountered when fetching page of block metadata from gateway with minBlock \'%s\' and maxTimestamp \'%s\'',
             newMin,
             maxTimestamp
           )

diff --git a/servers/cu/src/domain/client/ao-block.test.js b/servers/cu/src/domain/client/ao-block.test.js
@@ -8,7 +8,7 @@ import { createLogger } from '../logger.js'
 import { loadBlocksMetaSchema } from '../dal.js'
 import { loadBlocksMetaWith } from './ao-block.js'
 
-const GATEWAY_URL = globalThis.GATEWAY || 'https://arweave.net'
+const GRAPHQL_URL = globalThis.GRAPHQL_URL || 'https://arweave.net/graphql'
 const logger = createLogger('ao-cu')
 
 describe('ao-block', () => {
@@ -19,7 +19,7 @@ describe('ao-block', () => {
     test('load the block data across multiple pages', async () => {
       const loadBlocksMeta = loadBlocksMetaSchema.implement(loadBlocksMetaWith({
         fetch,
-        GATEWAY_URL,
+        GRAPHQL_URL,
         /**
          * Weird page size, so we know we are chopping off the excess
          * from the last page, correctly

diff --git a/servers/cu/src/domain/client/arweave.js b/servers/cu/src/domain/client/arweave.js
@@ -41,7 +41,7 @@ export function buildAndSignDataItemWith ({ WALLET, createDataItem = createData
 /**
  * @typedef Env1
  * @property {fetch} fetch
- * @property {string} GATEWAY_URL
+ * @property {string} GRAPHQL_URL
  *
  * @callback LoadTransactionMeta
  * @param {string} id - the id of the process whose src is being loaded
@@ -50,7 +50,7 @@ export function buildAndSignDataItemWith ({ WALLET, createDataItem = createData
  * @param {Env1} env
  * @returns {LoadTransactionMeta}
  */
-export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) {
+export function loadTransactionMetaWith ({ fetch, GRAPHQL_URL, logger }) {
   // TODO: create a dataloader and use that to batch load contracts
 
   const GET_PROCESSES_QUERY = `
@@ -83,12 +83,10 @@ export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) {
     })
   })
 
-  const GRAPHQL = joinUrl({ url: GATEWAY_URL, path: '/graphql' })
-
   return (id) =>
     of(id)
       .chain(fromPromise((id) =>
-        fetch(GRAPHQL, {
+        fetch(GRAPHQL_URL, {
           method: 'POST',
           headers: { 'Content-Type': 'application/json' },
           body: JSON.stringify({
@@ -98,11 +96,7 @@ export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) {
         })
           .then(async (res) => {
             if (res.ok) return res.json()
-            logger(
-              'Error Encountered when fetching transaction \'%s\' from gateway \'%s\'',
-              id,
-              GATEWAY_URL
-            )
+            logger('Error Encountered when fetching transaction \'%s\' from gateway', id)
             throw new Error(`${res.status}: ${await res.text()}`)
           })
           .then(transactionConnectionSchema.parse)
@@ -114,7 +108,7 @@ export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) {
 /**
    * @typedef Env2
    * @property {fetch} fetch
-   * @property {string} GATEWAY_URL
+   * @property {string} ARWEAVE_URL
    *
    * @callback LoadTransactionData
    * @param {string} id - the id of the process whose src is being loaded
@@ -123,30 +117,24 @@ export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) {
    * @param {Env2} env
    * @returns {LoadTransactionData}
    */
-export function loadTransactionDataWith ({ fetch, GATEWAY_URL, logger }) {
+export function loadTransactionDataWith ({ fetch, ARWEAVE_URL, logger }) {
   // TODO: create a dataloader and use that to batch load processes
   return (id) =>
     of(id)
       .chain(fromPromise((id) =>
-        fetch(joinUrl({ url: GATEWAY_URL, path: `/raw/${id}` }))
+        fetch(joinUrl({ url: ARWEAVE_URL, path: `/raw/${id}` }))
           .then(async (res) => {
             if (res.ok) return res
-            logger(
-              'Error Encountered when fetching raw data for transaction \'%s\' from gateway \'%s\'',
-              id,
-              GATEWAY_URL
-            )
+            logger('Error Encountered when fetching raw data for transaction \'%s\'', id)
             throw new Error(`${res.status}: ${await res.text()}`)
           })
       ))
       .toPromise()
 }
 
-export function queryGatewayWith ({ fetch, GATEWAY_URL, logger }) {
-  const GRAPHQL = joinUrl({ url: GATEWAY_URL, path: '/graphql' })
-
+export function queryGatewayWith ({ fetch, GRAPHQL_URL, logger }) {
   return async ({ query, variables }) => {
-    return fetch(GRAPHQL, {
+    return fetch(GRAPHQL_URL, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify({ query, variables })

diff --git a/servers/cu/src/domain/client/arweave.test.js b/servers/cu/src/domain/client/arweave.test.js
@@ -4,7 +4,8 @@ import * as assert from 'node:assert'
 import { loadTransactionDataSchema, loadTransactionMetaSchema } from '../dal.js'
 import { loadTransactionDataWith, loadTransactionMetaWith } from './arweave.js'
 
-const GATEWAY_URL = globalThis.GATEWAY || 'https://arweave.net'
+const GRAPHQL_URL = globalThis.GRAPHQL_URL || 'https://arweave.net/graphql'
+const ARWEAVE_URL = globalThis.ARWEAVE_URL || 'https://arweave.net'
 const PROCESS = 'zc24Wpv_i6NNCEdxeKt7dcNrqL5w0hrShtSCcFGGL24'
 
 describe('arweave', () => {
@@ -13,7 +14,7 @@ describe('arweave', () => {
       const loadTransactionMeta = loadTransactionMetaSchema.implement(
         loadTransactionMetaWith({
           fetch,
-          GATEWAY_URL
+          GRAPHQL_URL
         })
       )
       const result = await loadTransactionMeta(PROCESS)
@@ -23,9 +24,9 @@ describe('arweave', () => {
     test('pass the correct variables', async () => {
       const loadTransactionMeta = loadTransactionMetaSchema.implement(
         loadTransactionMetaWith({
-          GATEWAY_URL,
+          GRAPHQL_URL,
           fetch: async (url, options) => {
-            assert.equal(url, `${GATEWAY_URL}/graphql`)
+            assert.equal(url, GRAPHQL_URL)
             const body = JSON.parse(options.body)
             assert.deepStrictEqual(body.variables, { processIds: [PROCESS] })
 
@@ -81,10 +82,10 @@ describe('arweave', () => {
       const loadTransactionData = loadTransactionDataSchema.implement(
         loadTransactionDataWith({
           fetch: (url, options) => {
-            assert.equal(url, `${GATEWAY_URL}/raw/${PROCESS}`)
+            assert.equal(url, `${ARWEAVE_URL}/raw/${PROCESS}`)
             return fetch(url, options)
           },
-          GATEWAY_URL
+          ARWEAVE_URL
         })
       )
       const result = await loadTransactionData(PROCESS)
@@ -98,10 +99,10 @@ describe('arweave', () => {
       const loadTransactionData = loadTransactionDataSchema.implement(
         loadTransactionDataWith({
           fetch: (url, options) => {
-            assert.equal(url, `${GATEWAY_URL}/raw/${PROCESS}?foo=bar`)
+            assert.equal(url, `${ARWEAVE_URL}/raw/${PROCESS}?foo=bar`)
             return fetch(url, options)
           },
-          GATEWAY_URL: `${GATEWAY_URL}?foo=bar`
+          ARWEAVE_URL: `${ARWEAVE_URL}?foo=bar`
         })
       )
       const result = await loadTransactionData(PROCESS)