feat(DB): Chunk SQL Inserts

defaults.env

@@ -12,6 +12,11 @@ CHAINGRAPH_POSTGRES_MAX_CONNECTIONS=
 # In real-world testing, this usually reduces the speed of Chaingraph's initial sync, so Chaingraph leaves "synchronous_commit = on" by default.
 CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT=true
 
+# The target size (in MB) for each INSERT query chunk sent to Postgres.
+# During initial sync, the transactions of each block will be inserted into the DB. This value specifies the target size of each INSERT query chunk to prevent excessivley large queries on large blocks.
+# Setting to a low value (e.g. 1MB) will allow Chaingraph to perform initial sync on memory constrained devices/servers.
+CHAINGRAPH_POSTGRES_INSERT_CHUNK_SIZE_MB=32
+
 # The target size (in MB) of the buffer which holds downloaded blocks waiting to be saved to the database. This primarily affects memory usage during the initial chain sync.
 # For best performance, this should be around `CHAINGRAPH_POSTGRES_MAX_CONNECTIONS * maximum block size`, while leaving enough memory available to the host machine. If not set, Chaingraph will measure free memory at startup and attempt to select a reasonable value.
 CHAINGRAPH_BLOCK_BUFFER_TARGET_SIZE_MB=

src/config.ts

@@ -44,6 +44,7 @@ const expectedOptions = [
   'CHAINGRAPH_POSTGRES_CONNECTION_STRING',
   'CHAINGRAPH_POSTGRES_MAX_CONNECTIONS',
   'CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT',
+  'CHAINGRAPH_POSTGRES_INSERT_CHUNK_SIZE_MB',
   'CHAINGRAPH_TRUSTED_NODES',
   'CHAINGRAPH_USER_AGENT',
   'NODE_ENV',
@@ -362,6 +363,17 @@ const chaingraphUserAgent =
 const postgresSynchronousCommit =
   configuration.CHAINGRAPH_POSTGRES_SYNCHRONOUS_COMMIT !== 'false';
 
+/**
+ * Set via the `CHAINGRAPH_POSTGRES_INSERT_CHUNK_SIZE_MB` environment variable.
+ */
+const postgresInsertChunkSizeMbValue = Number(
+  configuration.CHAINGRAPH_POSTGRES_INSERT_CHUNK_SIZE_MB
+);
+if (isNaN(postgresInsertChunkSizeMbValue)) {
+  // eslint-disable-next-line functional/no-throw-statement
+  throw new Error('CHAINGRAPH_POSTGRES_INSERT_CHUNK_SIZE_MB must be a number.');
+}
+
 /**
  * `true` if the `NODE_ENV` environment variable is `production`.
  */