diff --git a/servers/cu/README.md b/servers/cu/README.md index 1a1790aa7..b32bd6342 100644 --- a/servers/cu/README.md +++ b/servers/cu/README.md @@ -40,8 +40,13 @@ Either command will start a server listening on `PORT` (`6363` by default). There are a few environment variables that you can set. Besides `WALLET`/`WALLET_FILE`, they each have a default: -- `GATEWAY_URL`: The Arweave gateway for the CU to use fetch block metadata, - data on arweave, and Scheduler-Location data (defaults to `arweave.net`) +- `GATEWAY_URL`: The url of the Arweave gateway to use. (Defaults to `https://arweave.net`) + +> `GATEWAY_URL` is solely used as a fallback for both `ARWEAVE_URL` and `GRAPHQL_URL`, if not provided (see below). + +- `ARWEAVE_URL`: The url for the Arweave http API server, to be used by the CU to fetch +transaction data from Arweave, specifically ao `Modules`, and `Message` `Assignment`s. (Defaults to `GATEWAY_URL`) +- `GRAPHQL_URL`: The url for the Arweave Gateway GraphQL server to be used by the CU. (Defaults to `${GATEWAY_URL}/graphql`) - `UPLOADER_URL`: The url of the uploader to use to upload Process `Checkpoints` to Arweave. (Defaults to `up.arweave.net`) - `WALLET`/`WALLET_FILE`: the JWK Interface stringified JSON that will be used by the CU, or a file to load it from diff --git a/servers/cu/package-lock.json b/servers/cu/package-lock.json index f89e3df7c..a11019e01 100644 --- a/servers/cu/package-lock.json +++ b/servers/cu/package-lock.json @@ -10,7 +10,7 @@ "dependencies": { "@fastify/middie": "^8.3.0", "@permaweb/ao-loader": "^0.0.22", - "@permaweb/ao-scheduler-utils": "^0.0.15", + "@permaweb/ao-scheduler-utils": "^0.0.16", "arweave": "^1.14.4", "cors": "^2.8.5", "dataloader": "^2.2.2", @@ -135,9 +135,9 @@ } }, "node_modules/@permaweb/ao-scheduler-utils": { - "version": "0.0.15", - "resolved": "https://registry.npmjs.org/@permaweb/ao-scheduler-utils/-/ao-scheduler-utils-0.0.15.tgz", - "integrity": "sha512-Ef3IGP08E6KgcwX/AUq3k9gj7W/5GylEZRwXbXwC1JYl5FoHegdGAy0gDJYKHO1Nf4NVNFicb1xBaDeU0Ic7og==", + "version": "0.0.16", + "resolved": "https://registry.npmjs.org/@permaweb/ao-scheduler-utils/-/ao-scheduler-utils-0.0.16.tgz", + "integrity": "sha512-3VSuACbx9gUOl57TbnidVpoA/0yF2uQdm/+9pU+Wmp/oiV8J+D2ApEahlzk9gzzGX7zmzQ+RbRwc8oxKfW1v0Q==", "dependencies": { "lru-cache": "^10.2.0", "ramda": "^0.29.1" @@ -2190,9 +2190,9 @@ "integrity": "sha512-im4q+//pn+m+/YDoBzEov5MnCLcuPZQcTBxD2cCgzBDeRJ7zViVt2hfJNQRqpDY88diOvZOpFKlkpSx+iRMQzg==" }, "@permaweb/ao-scheduler-utils": { - "version": "0.0.15", - "resolved": "https://registry.npmjs.org/@permaweb/ao-scheduler-utils/-/ao-scheduler-utils-0.0.15.tgz", - "integrity": "sha512-Ef3IGP08E6KgcwX/AUq3k9gj7W/5GylEZRwXbXwC1JYl5FoHegdGAy0gDJYKHO1Nf4NVNFicb1xBaDeU0Ic7og==", + "version": "0.0.16", + "resolved": "https://registry.npmjs.org/@permaweb/ao-scheduler-utils/-/ao-scheduler-utils-0.0.16.tgz", + "integrity": "sha512-3VSuACbx9gUOl57TbnidVpoA/0yF2uQdm/+9pU+Wmp/oiV8J+D2ApEahlzk9gzzGX7zmzQ+RbRwc8oxKfW1v0Q==", "requires": { "lru-cache": "^10.2.0", "ramda": "^0.29.1" diff --git a/servers/cu/package.json b/servers/cu/package.json index 9dbe34c29..8743012d7 100644 --- a/servers/cu/package.json +++ b/servers/cu/package.json @@ -14,7 +14,7 @@ "dependencies": { "@fastify/middie": "^8.3.0", "@permaweb/ao-loader": "^0.0.22", - "@permaweb/ao-scheduler-utils": "^0.0.15", + "@permaweb/ao-scheduler-utils": "^0.0.16", "arweave": "^1.14.4", "cors": "^2.8.5", "dataloader": "^2.2.2", diff --git a/servers/cu/src/config.js b/servers/cu/src/config.js index 2dc6ca7b4..e530ff49b 100644 --- a/servers/cu/src/config.js +++ b/servers/cu/src/config.js @@ -2,10 +2,12 @@ import { existsSync, readFileSync } from 'node:fs' import { tmpdir } from 'node:os' import * as path from 'node:path' +import { pipe } from 'ramda' import { z, ZodIssueCode } from 'zod' import ms from 'ms' import { domainConfigSchema, positiveIntSchema } from './domain/index.js' +import { preprocessUrls } from './domain/utils.js' /** * Some frameworks will implicitly override NODE_ENV @@ -32,46 +34,46 @@ const serverConfigSchema = domainConfigSchema.extend({ DUMP_PATH: z.string().min(1) }) +/** + * An Either would be nice here, but just throwing a literal + * get's us what we need, for now. + */ +/* eslint-disable no-throw-literal */ + /** * If the WALLET_FILE env var is defined, load the contents from the file. * Refuse to boot the app if both or none of WALLET and WALLET_FILE are defined. */ -const preprocessedServerConfigSchema = z.preprocess( - (envConfig, zodRefinementContext) => { - const { WALLET, WALLET_FILE, ...theRestOfTheConfig } = envConfig +export const preprocessWallet = (envConfig) => { + const { WALLET, WALLET_FILE, ...theRestOfTheConfig } = envConfig - const error = message => zodRefinementContext.addIssue({ - code: ZodIssueCode.custom, - message - }) + // nothing to do here + if (!!WALLET && !WALLET_FILE) return envConfig - if (!!WALLET && !WALLET_FILE) { - // nothing to do here - return envConfig - } - if (!WALLET && !WALLET_FILE) { - error('One of WALLET or WALLET_FILE is required') - return - } - if (!!WALLET && !!WALLET_FILE) { - error('Do not define both WALLET and WALLET_FILE') - return - } + if (!WALLET && !WALLET_FILE) throw 'One of WALLET or WALLET_FILE is required' + if (!!WALLET && !!WALLET_FILE) throw 'Do not define both WALLET and WALLET_FILE' + + const walletPath = path.resolve(WALLET_FILE) + if (!existsSync(walletPath)) throw `WALLET_FILE does not exist: ${walletPath}` - const walletPath = path.resolve(WALLET_FILE) - if (!existsSync(walletPath)) { - error(`WALLET_FILE does not exist: ${walletPath}`) - return + try { + const walletFromFile = readFileSync(walletPath, 'utf8') + return { + WALLET: walletFromFile, + ...theRestOfTheConfig } + } catch (e) { + throw `An error occurred while reading WALLET_FILE from ${walletPath}\n${e}` + } +} +/* eslint-enable no-throw-literal */ +const preprocessedServerConfigSchema = z.preprocess( + (envConfig, zodRefinementContext) => { try { - const walletFromFile = readFileSync(walletPath, 'utf8') - return { - WALLET: walletFromFile, - ...theRestOfTheConfig - } - } catch (e) { - error(`An error occurred while reading WALLET_FILE from ${walletPath}\n${e}`) + return pipe(preprocessWallet, preprocessUrls)(envConfig) + } catch (message) { + zodRefinementContext.addIssue({ code: ZodIssueCode.custom, message }) } }, serverConfigSchema @@ -88,6 +90,8 @@ const CONFIG_ENVS = { MODE, port: process.env.PORT || 6363, GATEWAY_URL: process.env.GATEWAY_URL || 'https://arweave.net', + GRAPHQL_URL: process.env.GRAPHQL_URL, + ARWEAVE_URL: process.env.ARWEAVE_URL, UPLOADER_URL: process.env.UPLOADER_URL || 'https://up.arweave.net', DB_MODE: process.env.DB_MODE || 'embedded', DB_URL: process.env.DB_URL || 'ao-cache', @@ -115,6 +119,8 @@ const CONFIG_ENVS = { MODE, port: process.env.PORT || 6363, GATEWAY_URL: process.env.GATEWAY_URL || 'https://arweave.net', + GRAPHQL_URL: process.env.GRAPHQL_URL, + ARWEAVE_URL: process.env.ARWEAVE_URL, UPLOADER_URL: process.env.UPLOADER_URL || 'https://up.arweave.net', DB_MODE: process.env.DB_MODE || 'embedded', DB_URL: process.env.DB_URL || 'ao-cache', diff --git a/servers/cu/src/domain/client/ao-block.js b/servers/cu/src/domain/client/ao-block.js index b665f54a3..d5cc768a6 100644 --- a/servers/cu/src/domain/client/ao-block.js +++ b/servers/cu/src/domain/client/ao-block.js @@ -4,7 +4,6 @@ import { z } from 'zod' import { blockSchema } from '../model.js' import { BLOCKS_ASC_IDX } from './pouchdb.js' -import { joinUrl } from '../utils.js' const blockDocSchema = z.object({ _id: z.string().min(1), @@ -74,7 +73,7 @@ export function findBlocksWith ({ pouchDb }) { /** * @typedef Env2 * @property {fetch} fetch - * @property {string} GATEWAY_URL + * @property {string} GRAPHQL_URL * @property {number} pageSize * * @callback LoadBlocksMeta @@ -84,11 +83,8 @@ export function findBlocksWith ({ pouchDb }) { * @param {Env1} env * @returns {LoadBlocksMeta} */ -export function loadBlocksMetaWith ({ fetch, GATEWAY_URL, pageSize, logger }) { +export function loadBlocksMetaWith ({ fetch, GRAPHQL_URL, pageSize, logger }) { // TODO: create a dataloader and use that to batch load contracts - - const GRAPHQL = joinUrl({ url: GATEWAY_URL, path: '/graphql' }) - const GET_BLOCKS_QUERY = ` query GetBlocks($min: Int!, $limit: Int!) { blocks( @@ -129,7 +125,7 @@ export function loadBlocksMetaWith ({ fetch, GATEWAY_URL, pageSize, logger }) { return variables }) .then((variables) => - fetch(GRAPHQL, { + fetch(GRAPHQL_URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -141,8 +137,7 @@ export function loadBlocksMetaWith ({ fetch, GATEWAY_URL, pageSize, logger }) { .then(async (res) => { if (res.ok) return res.json() logger( - 'Error Encountered when fetching page of block metadata from gateway \'%s\' with minBlock \'%s\' and maxTimestamp \'%s\'', - GATEWAY_URL, + 'Error Encountered when fetching page of block metadata from gateway with minBlock \'%s\' and maxTimestamp \'%s\'', newMin, maxTimestamp ) diff --git a/servers/cu/src/domain/client/ao-block.test.js b/servers/cu/src/domain/client/ao-block.test.js index b56a2bc61..2c04a18a8 100644 --- a/servers/cu/src/domain/client/ao-block.test.js +++ b/servers/cu/src/domain/client/ao-block.test.js @@ -8,7 +8,7 @@ import { createLogger } from '../logger.js' import { loadBlocksMetaSchema } from '../dal.js' import { loadBlocksMetaWith } from './ao-block.js' -const GATEWAY_URL = globalThis.GATEWAY || 'https://arweave.net' +const GRAPHQL_URL = globalThis.GRAPHQL_URL || 'https://arweave.net/graphql' const logger = createLogger('ao-cu') describe('ao-block', () => { @@ -19,7 +19,7 @@ describe('ao-block', () => { test('load the block data across multiple pages', async () => { const loadBlocksMeta = loadBlocksMetaSchema.implement(loadBlocksMetaWith({ fetch, - GATEWAY_URL, + GRAPHQL_URL, /** * Weird page size, so we know we are chopping off the excess * from the last page, correctly diff --git a/servers/cu/src/domain/client/arweave.js b/servers/cu/src/domain/client/arweave.js index 31d9666d2..a3d98839b 100644 --- a/servers/cu/src/domain/client/arweave.js +++ b/servers/cu/src/domain/client/arweave.js @@ -41,7 +41,7 @@ export function buildAndSignDataItemWith ({ WALLET, createDataItem = createData /** * @typedef Env1 * @property {fetch} fetch - * @property {string} GATEWAY_URL + * @property {string} GRAPHQL_URL * * @callback LoadTransactionMeta * @param {string} id - the id of the process whose src is being loaded @@ -50,7 +50,7 @@ export function buildAndSignDataItemWith ({ WALLET, createDataItem = createData * @param {Env1} env * @returns {LoadTransactionMeta} */ -export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) { +export function loadTransactionMetaWith ({ fetch, GRAPHQL_URL, logger }) { // TODO: create a dataloader and use that to batch load contracts const GET_PROCESSES_QUERY = ` @@ -83,12 +83,10 @@ export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) { }) }) - const GRAPHQL = joinUrl({ url: GATEWAY_URL, path: '/graphql' }) - return (id) => of(id) .chain(fromPromise((id) => - fetch(GRAPHQL, { + fetch(GRAPHQL_URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -98,11 +96,7 @@ export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) { }) .then(async (res) => { if (res.ok) return res.json() - logger( - 'Error Encountered when fetching transaction \'%s\' from gateway \'%s\'', - id, - GATEWAY_URL - ) + logger('Error Encountered when fetching transaction \'%s\' from gateway', id) throw new Error(`${res.status}: ${await res.text()}`) }) .then(transactionConnectionSchema.parse) @@ -114,7 +108,7 @@ export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) { /** * @typedef Env2 * @property {fetch} fetch - * @property {string} GATEWAY_URL + * @property {string} ARWEAVE_URL * * @callback LoadTransactionData * @param {string} id - the id of the process whose src is being loaded @@ -123,30 +117,24 @@ export function loadTransactionMetaWith ({ fetch, GATEWAY_URL, logger }) { * @param {Env2} env * @returns {LoadTransactionData} */ -export function loadTransactionDataWith ({ fetch, GATEWAY_URL, logger }) { +export function loadTransactionDataWith ({ fetch, ARWEAVE_URL, logger }) { // TODO: create a dataloader and use that to batch load processes return (id) => of(id) .chain(fromPromise((id) => - fetch(joinUrl({ url: GATEWAY_URL, path: `/raw/${id}` })) + fetch(joinUrl({ url: ARWEAVE_URL, path: `/raw/${id}` })) .then(async (res) => { if (res.ok) return res - logger( - 'Error Encountered when fetching raw data for transaction \'%s\' from gateway \'%s\'', - id, - GATEWAY_URL - ) + logger('Error Encountered when fetching raw data for transaction \'%s\'', id) throw new Error(`${res.status}: ${await res.text()}`) }) )) .toPromise() } -export function queryGatewayWith ({ fetch, GATEWAY_URL, logger }) { - const GRAPHQL = joinUrl({ url: GATEWAY_URL, path: '/graphql' }) - +export function queryGatewayWith ({ fetch, GRAPHQL_URL, logger }) { return async ({ query, variables }) => { - return fetch(GRAPHQL, { + return fetch(GRAPHQL_URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ query, variables }) diff --git a/servers/cu/src/domain/client/arweave.test.js b/servers/cu/src/domain/client/arweave.test.js index e9d1f07f4..9ca558faa 100644 --- a/servers/cu/src/domain/client/arweave.test.js +++ b/servers/cu/src/domain/client/arweave.test.js @@ -4,7 +4,8 @@ import * as assert from 'node:assert' import { loadTransactionDataSchema, loadTransactionMetaSchema } from '../dal.js' import { loadTransactionDataWith, loadTransactionMetaWith } from './arweave.js' -const GATEWAY_URL = globalThis.GATEWAY || 'https://arweave.net' +const GRAPHQL_URL = globalThis.GRAPHQL_URL || 'https://arweave.net/graphql' +const ARWEAVE_URL = globalThis.ARWEAVE_URL || 'https://arweave.net' const PROCESS = 'zc24Wpv_i6NNCEdxeKt7dcNrqL5w0hrShtSCcFGGL24' describe('arweave', () => { @@ -13,7 +14,7 @@ describe('arweave', () => { const loadTransactionMeta = loadTransactionMetaSchema.implement( loadTransactionMetaWith({ fetch, - GATEWAY_URL + GRAPHQL_URL }) ) const result = await loadTransactionMeta(PROCESS) @@ -23,9 +24,9 @@ describe('arweave', () => { test('pass the correct variables', async () => { const loadTransactionMeta = loadTransactionMetaSchema.implement( loadTransactionMetaWith({ - GATEWAY_URL, + GRAPHQL_URL, fetch: async (url, options) => { - assert.equal(url, `${GATEWAY_URL}/graphql`) + assert.equal(url, GRAPHQL_URL) const body = JSON.parse(options.body) assert.deepStrictEqual(body.variables, { processIds: [PROCESS] }) @@ -81,10 +82,10 @@ describe('arweave', () => { const loadTransactionData = loadTransactionDataSchema.implement( loadTransactionDataWith({ fetch: (url, options) => { - assert.equal(url, `${GATEWAY_URL}/raw/${PROCESS}`) + assert.equal(url, `${ARWEAVE_URL}/raw/${PROCESS}`) return fetch(url, options) }, - GATEWAY_URL + ARWEAVE_URL }) ) const result = await loadTransactionData(PROCESS) @@ -98,10 +99,10 @@ describe('arweave', () => { const loadTransactionData = loadTransactionDataSchema.implement( loadTransactionDataWith({ fetch: (url, options) => { - assert.equal(url, `${GATEWAY_URL}/raw/${PROCESS}?foo=bar`) + assert.equal(url, `${ARWEAVE_URL}/raw/${PROCESS}?foo=bar`) return fetch(url, options) }, - GATEWAY_URL: `${GATEWAY_URL}?foo=bar` + ARWEAVE_URL: `${ARWEAVE_URL}?foo=bar` }) ) const result = await loadTransactionData(PROCESS) diff --git a/servers/cu/src/domain/client/worker.js b/servers/cu/src/domain/client/worker.js index fe2a971d9..0dd54bf39 100644 --- a/servers/cu/src/domain/client/worker.js +++ b/servers/cu/src/domain/client/worker.js @@ -61,18 +61,14 @@ function writeWasmFileWith ({ DIR, logger }) { * ####################### */ -function streamTransactionDataWith ({ fetch, GATEWAY_URL, logger }) { +function streamTransactionDataWith ({ fetch, ARWEAVE_URL, logger }) { return (id) => of(id) .chain(fromPromise((id) => - fetch(joinUrl({ url: GATEWAY_URL, path: `/raw/${id}` })) + fetch(joinUrl({ url: ARWEAVE_URL, path: `/raw/${id}` })) .then(async (res) => { if (res.ok) return res - logger( - 'Error Encountered when fetching raw data for transaction \'%s\' from gateway \'%s\'', - id, - GATEWAY_URL - ) + logger('Error Encountered when fetching raw data for transaction \'%s\' from gateway \'%s\'', id) throw new Error(`${res.status}: ${await res.text()}`) }) )) @@ -338,7 +334,7 @@ if (!process.env.NO_WORKER) { wasmInstanceCache: createWasmInstanceCache({ MAX_SIZE: workerData.WASM_INSTANCE_CACHE_MAX_SIZE }), readWasmFile: readWasmFileWith({ DIR: workerData.WASM_BINARY_FILE_DIRECTORY }), writeWasmFile: writeWasmFileWith({ DIR: workerData.WASM_BINARY_FILE_DIRECTORY, logger }), - streamTransactionData: streamTransactionDataWith({ fetch, GATEWAY_URL: workerData.GATEWAY_URL, logger }), + streamTransactionData: streamTransactionDataWith({ fetch, ARWEAVE_URL: workerData.ARWEAVE_URL, logger }), bootstrapWasmInstance: (wasmModule, gasLimit, memoryLimit) => AoLoader( (info, receiveInstance) => WebAssembly.instantiate(wasmModule, info).then(receiveInstance), gasLimit, diff --git a/servers/cu/src/domain/index.js b/servers/cu/src/domain/index.js index 7a215f3a3..bf2040867 100644 --- a/servers/cu/src/domain/index.js +++ b/servers/cu/src/domain/index.js @@ -37,7 +37,7 @@ export const createApis = async (ctx) => { const { locate } = schedulerUtilsConnect({ cacheSize: 100, - GATEWAY_URL: ctx.GATEWAY_URL, + GRAPHQL_URL: ctx.GRAPHQL_URL, followRedirects: true }) const locateDataloader = new Dataloader(async (params) => { @@ -75,7 +75,7 @@ export const createApis = async (ctx) => { WASM_MODULE_CACHE_MAX_SIZE: ctx.WASM_MODULE_CACHE_MAX_SIZE, WASM_INSTANCE_CACHE_MAX_SIZE: ctx.WASM_INSTANCE_CACHE_MAX_SIZE, WASM_BINARY_FILE_DIRECTORY: ctx.WASM_BINARY_FILE_DIRECTORY, - GATEWAY_URL: ctx.GATEWAY_URL, + ARWEAVE_URL: ctx.ARWEAVE_URL, id: workerId } } @@ -103,7 +103,7 @@ export const createApis = async (ctx) => { const saveCheckpoint = AoProcessClient.saveCheckpointWith({ address, - queryGateway: ArweaveClient.queryGatewayWith({ fetch: ctx.fetch, GATEWAY_URL: ctx.GATEWAY_URL, logger: ctx.logger }), + queryGateway: ArweaveClient.queryGatewayWith({ fetch: ctx.fetch, GRAPHQL_URL: ctx.GRAPHQL_URL, logger: ctx.logger }), hashWasmMemory: WasmClient.hashWasmMemory, buildAndSignDataItem: ArweaveClient.buildAndSignDataItemWith({ WALLET: ctx.WALLET }), uploadDataItem: ArweaveClient.uploadDataItemWith({ UPLOADER_URL: ctx.UPLOADER_URL, fetch: ctx.fetch, logger: ctx.logger }), @@ -154,12 +154,12 @@ export const createApis = async (ctx) => { }) const sharedDeps = (logger) => ({ - loadTransactionMeta: ArweaveClient.loadTransactionMetaWith({ fetch: ctx.fetch, GATEWAY_URL: ctx.GATEWAY_URL, logger }), - loadTransactionData: ArweaveClient.loadTransactionDataWith({ fetch: ctx.fetch, GATEWAY_URL: ctx.GATEWAY_URL, logger }), + loadTransactionMeta: ArweaveClient.loadTransactionMetaWith({ fetch: ctx.fetch, GRAPHQL_URL: ctx.GRAPHQL_URL, logger }), + loadTransactionData: ArweaveClient.loadTransactionDataWith({ fetch: ctx.fetch, ARWEAVE_URL: ctx.ARWEAVE_URL, logger }), findProcess: AoProcessClient.findProcessWith({ pouchDb, logger }), findProcessMemoryBefore: AoProcessClient.findProcessMemoryBeforeWith({ cache: wasmMemoryCache, - loadTransactionData: ArweaveClient.loadTransactionDataWith({ fetch: ctx.fetch, GATEWAY_URL: ctx.GATEWAY_URL, logger }), + loadTransactionData: ArweaveClient.loadTransactionDataWith({ fetch: ctx.fetch, ARWEAVE_URL: ctx.ARWEAVE_URL, logger }), findCheckpointFileBefore: AoProcessClient.findCheckpointFileBeforeWith({ DIR: ctx.PROCESS_CHECKPOINT_FILE_DIRECTORY, glob: fastGlob @@ -169,7 +169,7 @@ export const createApis = async (ctx) => { readFile }), address, - queryGateway: ArweaveClient.queryGatewayWith({ fetch: ctx.fetch, GATEWAY_URL: ctx.GATEWAY_URL, logger }), + queryGateway: ArweaveClient.queryGatewayWith({ fetch: ctx.fetch, GRAPHQL_URL: ctx.GRAPHQL_URL, logger }), PROCESS_IGNORE_ARWEAVE_CHECKPOINTS: ctx.PROCESS_IGNORE_ARWEAVE_CHECKPOINTS, logger }), @@ -184,7 +184,7 @@ export const createApis = async (ctx) => { saveEvaluation: AoEvaluationClient.saveEvaluationWith({ pouchDb, logger }), findBlocks: AoBlockClient.findBlocksWith({ pouchDb, logger }), saveBlocks: AoBlockClient.saveBlocksWith({ pouchDb, logger }), - loadBlocksMeta: AoBlockClient.loadBlocksMetaWith({ fetch: ctx.fetch, GATEWAY_URL: ctx.GATEWAY_URL, pageSize: 90, logger }), + loadBlocksMeta: AoBlockClient.loadBlocksMetaWith({ fetch: ctx.fetch, GRAPHQL_URL: ctx.GRAPHQL_URL, pageSize: 90, logger }), findModule: AoModuleClient.findModuleWith({ pouchDb, logger }), saveModule: AoModuleClient.saveModuleWith({ pouchDb, logger }), loadEvaluator: AoModuleClient.evaluatorWith({ diff --git a/servers/cu/src/domain/model.js b/servers/cu/src/domain/model.js index 351a4ac16..2c79e5cb4 100644 --- a/servers/cu/src/domain/model.js +++ b/servers/cu/src/domain/model.js @@ -20,12 +20,22 @@ export const domainConfigSchema = z.object({ */ PROCESS_WASM_COMPUTE_MAX_LIMIT: positiveIntSchema, /** - * The gateway for the CU to use fetch block metadata, data on arweave, - * and Scheduler-Location data + * The url for the graphql server to be used by the CU + * to query for metadata from an Arweave Gateway + * + * ie. https://arweave.net/graphql */ - GATEWAY_URL: z.string().url('GATEWAY_URL must be a a valid URL'), + GRAPHQL_URL: z.string().url('GRAPHQL_URL must be a valid URL'), + /** + * The url for the server that hosts the Arweave http API + * + * ie. https://arweave.net + */ + ARWEAVE_URL: z.string().url('ARWEAVE_URL must be a valid URL'), /** * The url of the uploader to use to upload Process Checkpoints to Arweave + * + * ie. https://up.arweave.net */ UPLOADER_URL: z.string().url('UPLOADER_URL must be a a valid URL'), /** diff --git a/servers/cu/src/domain/utils.js b/servers/cu/src/domain/utils.js index 2874a66eb..a1a772f93 100644 --- a/servers/cu/src/domain/utils.js +++ b/servers/cu/src/domain/utils.js @@ -1,5 +1,3 @@ -import { join as joinPath } from 'node:path/posix' - import { Rejected, Resolved } from 'hyper-async' import { F, T, __, allPass, always, append, assoc, chain, concat, cond, defaultTo, equals, @@ -9,15 +7,36 @@ import { ZodError, ZodIssueCode } from 'zod' export const joinUrl = ({ url, path }) => { if (!path) return url + if (path.startsWith('/')) return joinUrl({ url, path: path.slice(1) }) + url = new URL(url) - /** - * posix will correctly join the paths - * in a url compatible way - */ - url.pathname = joinPath(url.pathname, path) + url.pathname += path return url.toString() } +/* eslint-disable no-throw-literal */ +/** + * If either ARWEAVE_URL or GRAPHQL_URL is not defined, then set them to their defaults + * using GATEWAY_URL, which will always have a value. + */ +export const preprocessUrls = (envConfig) => { + let { GATEWAY_URL, ARWEAVE_URL, GRAPHQL_URL, ...theRestOfTheConfig } = envConfig + + if (ARWEAVE_URL && GRAPHQL_URL) return envConfig + + if (!GATEWAY_URL) { + if (!ARWEAVE_URL && !GRAPHQL_URL) throw 'GATEWAY_URL is required, if either ARWEAVE_URL or GRAPHQL_URL is not provided' + if (!ARWEAVE_URL) throw 'GATEWAY_URL is required if ARWEAVE_URL is not provided' + if (!GRAPHQL_URL) throw 'GATEWAY_URL is required if GRAPHQL_URL is not provided' + } + + if (!ARWEAVE_URL) ARWEAVE_URL = GATEWAY_URL + if (!GRAPHQL_URL) GRAPHQL_URL = joinUrl({ url: GATEWAY_URL, path: '/graphql' }) + + return { ARWEAVE_URL, GRAPHQL_URL, ...theRestOfTheConfig } +} +/* eslint-enable no-throw-literal */ + export const isNamed = has('name') export function errFrom (err) { diff --git a/servers/cu/src/domain/utils.test.js b/servers/cu/src/domain/utils.test.js index db47fa8cd..95e10e96f 100644 --- a/servers/cu/src/domain/utils.test.js +++ b/servers/cu/src/domain/utils.test.js @@ -3,7 +3,7 @@ import * as assert from 'node:assert' import { z } from 'zod' -import { busyIn, errFrom, evaluationToCursor, findPendingForProcessBeforeWith, maybeParseCursor, removeTagsByNameMaybeValue, joinUrl } from './utils.js' +import { busyIn, errFrom, evaluationToCursor, findPendingForProcessBeforeWith, maybeParseCursor, removeTagsByNameMaybeValue, joinUrl, preprocessUrls } from './utils.js' import ms from 'ms' describe('utils', () => { @@ -27,6 +27,32 @@ describe('utils', () => { }) }) + describe('preprocessUrls', () => { + const GATEWAY_URL = 'https://foo.bar' + const ARWEAVE_URL = 'https://arweave.net' + const GRAPHQL_URL = 'https://my.custom/graphql' + + test('should use the provided values', () => { + const config = preprocessUrls({ GATEWAY_URL, ARWEAVE_URL, GRAPHQL_URL }) + assert.equal(config.ARWEAVE_URL, ARWEAVE_URL) + assert.equal(config.GRAPHQL_URL, GRAPHQL_URL) + }) + + test('should use the provided GATEWAY_URL to default ARWEAVE_URL and GRAPHQL_URL', () => { + const noArweaveUrl = preprocessUrls({ GATEWAY_URL, GRAPHQL_URL }) + assert.equal(noArweaveUrl.ARWEAVE_URL, GATEWAY_URL) + assert.equal(noArweaveUrl.GRAPHQL_URL, GRAPHQL_URL) + + const noGraphQlUrl = preprocessUrls({ GATEWAY_URL, ARWEAVE_URL }) + assert.equal(noGraphQlUrl.GRAPHQL_URL, `${GATEWAY_URL}/graphql`) + assert.equal(noGraphQlUrl.ARWEAVE_URL, ARWEAVE_URL) + + const neither = preprocessUrls({ GATEWAY_URL }) + assert.equal(neither.GRAPHQL_URL, `${GATEWAY_URL}/graphql`) + assert.equal(neither.ARWEAVE_URL, GATEWAY_URL) + }) + }) + describe('errFrom', () => { test('should map ZodError to a friendly error', async () => { const schema = z.function().args(z.object({ name: z.string() })).returns(