Merge pull request #5 from subquery/sync-with-main-20250107

Sync with main and more bug fixes

packages/common-starknet/package.json

@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^5.1.1",
+    "@subql/common": "^5.2.2",
     "@subql/types-starknet": "workspace:*",
     "@typechain/starknet": "latest",
     "js-yaml": "^4.1.0",

packages/node/package.json

@@ -24,10 +24,10 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^5.1.1",
+    "@subql/common": "^5.2.2",
     "@subql/common-starknet": "workspace:*",
-    "@subql/node-core": "^14.1.2",
-    "@subql/testing": "^2.2.1",
+    "@subql/node-core": "^16.1.0",
+    "@subql/testing": "^2.2.2",
     "@subql/types-starknet": "workspace:*",
     "cacheable-lookup": "6",
     "eventemitter2": "^6.4.5",

packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts

@@ -5,7 +5,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
-  StoreCacheService,
   StoreService,
   IProjectService,
   BlockDispatcher,
@@ -14,13 +13,16 @@ import {
   IProjectUpgradeService,
   PoiSyncService,
   IBlock,
+  IStoreModelProvider,
 } from '@subql/node-core';
+import { StarknetBlock } from '@subql/types-starknet';
 import {
   StarknetProjectDs,
   SubqueryProject,
 } from '../../configure/SubqueryProject';
+import { isFullBlock } from '../../starknet/block.starknet';
 import { IndexerManager } from '../indexer.manager';
-import { BlockContent } from '../types';
+import { BlockContent, getBlockSize } from '../types';
 
 /**
  * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing
@@ -40,7 +42,7 @@ export class BlockDispatcherService
     @Inject('IProjectUpgradeService')
     projectUpgradeService: IProjectUpgradeService,
     storeService: StoreService,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
   ) {
@@ -50,7 +52,7 @@ export class BlockDispatcherService
       projectService,
       projectUpgradeService,
       storeService,
-      storeCacheService,
+      storeModelProvider,
       poiSyncService,
       project,
       apiService.fetchBlocks.bind(apiService),
@@ -65,4 +67,8 @@ export class BlockDispatcherService
       await this.projectService.getDataSources(block.getHeader().blockHeight),
     );
   }
+
+  protected getBlockSize(block: IBlock<BlockContent>): number {
+    return getBlockSize(block.block);
+  }
 }

packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts

@@ -7,7 +7,6 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
   StoreService,
-  StoreCacheService,
   IProjectService,
   WorkerBlockDispatcher,
   ConnectionPoolStateManager,
@@ -16,8 +15,10 @@ import {
   InMemoryCacheService,
   createIndexerWorker,
   MonitorServiceInterface,
+  IStoreModelProvider,
 } from '@subql/node-core';
 import { StarknetBlock } from '@subql/types-starknet';
+import { Header } from '../../../.yalc/@subql/node-core';
 import {
   StarknetProjectDs,
   SubqueryProject,
@@ -46,7 +47,7 @@ export class WorkerBlockDispatcherService
     projectUpgadeService: IProjectUpgradeService,
     cacheService: InMemoryCacheService,
     storeService: StoreService,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
@@ -60,7 +61,7 @@ export class WorkerBlockDispatcherService
       projectService,
       projectUpgadeService,
       storeService,
-      storeCacheService,
+      storeModelProvider,
       poiSyncService,
       project,
       () =>
@@ -88,7 +89,7 @@ export class WorkerBlockDispatcherService
   protected async fetchBlock(
     worker: IndexerWorker,
     height: number,
-  ): Promise<void> {
-    await worker.fetchBlock(height, 0 /* Unused with starknet*/);
+  ): Promise<Header> {
+    return worker.fetchBlock(height, 0 /* Unused with starknet*/);
   }
 }

packages/node/src/indexer/fetch.module.ts

@@ -9,12 +9,12 @@ import {
   NodeConfig,
   ConnectionPoolService,
   ConnectionPoolStateManager,
-  StoreCacheService,
   IProjectUpgradeService,
   PoiSyncService,
   InMemoryCacheService,
   MonitorService,
   CoreModule,
+  IStoreModelProvider,
 } from '@subql/node-core';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { StarknetApiConnection } from '../starknet/api.connection';
@@ -70,7 +70,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         indexerManager: IndexerManager,
         cacheService: InMemoryCacheService,
         storeService: StoreService,
-        storeCacheService: StoreCacheService,
+        storeModelProvider: IStoreModelProvider,
         poiSyncService: PoiSyncService,
         project: SubqueryProject,
         dynamicDsService: DynamicDsService,
@@ -86,7 +86,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectUpgradeService,
               cacheService,
               storeService,
-              storeCacheService,
+              storeModelProvider,
               poiSyncService,
               project,
               dynamicDsService,
@@ -102,7 +102,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectService,
               projectUpgradeService,
               storeService,
-              storeCacheService,
+              storeModelProvider,
               poiSyncService,
               project,
             ),
@@ -115,7 +115,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         IndexerManager,
         InMemoryCacheService,
         StoreService,
-        StoreCacheService,
+        'IStoreModelProvider',
         PoiSyncService,
         'ISubqueryProject',
         DynamicDsService,

packages/node/src/indexer/fetch.service.ts

@@ -12,7 +12,7 @@ import {
   ApiService,
   getModulos,
   Header,
-  StoreCacheService,
+  IStoreModelProvider,
 } from '@subql/node-core';
 import { StarknetBlock, SubqlDatasource } from '@subql/types-starknet';
 import { SubqueryProject } from '../configure/SubqueryProject';
@@ -40,25 +40,23 @@ export class FetchService extends BaseFetchService<
     private apiService: ApiService,
     nodeConfig: NodeConfig,
     @Inject('IProjectService') projectService: ProjectService,
-    @Inject('ISubqueryProject') project: SubqueryProject,
     @Inject('IBlockDispatcher')
     blockDispatcher: IStarknetBlockDispatcher,
     dictionaryService: StarknetDictionaryService,
     unfinalizedBlocksService: UnfinalizedBlocksService,
     eventEmitter: EventEmitter2,
     schedulerRegistry: SchedulerRegistry,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
   ) {
     super(
       nodeConfig,
       projectService,
-      project.network,
       blockDispatcher,
       dictionaryService,
       eventEmitter,
       schedulerRegistry,
       unfinalizedBlocksService,
-      storeCacheService,
+      storeModelProvider,
     );
   }
 

packages/node/src/indexer/types.ts

@@ -2,5 +2,11 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import { StarknetBlock, LightStarknetBlock } from '@subql/types-starknet';
+import { isFullBlock } from '../starknet/block.starknet';
 
 export type BlockContent = StarknetBlock | LightStarknetBlock;
+
+export function getBlockSize(block: BlockContent): number {
+  // TODO. not sure if this is the right way to determine the block size
+  return isFullBlock(block) ? (block as StarknetBlock).transactions.length : 0;
+}

packages/node/src/indexer/unfinalizedBlocks.service.ts

@@ -6,7 +6,7 @@ import {
   BaseUnfinalizedBlocksService,
   Header,
   NodeConfig,
-  StoreCacheService,
+  IStoreModelProvider,
 } from '@subql/node-core';
 import { StarknetNodeConfig } from '../configure/NodeConfig';
 import { StarknetApiService as ApiService } from '../starknet/api.service.starknet';
@@ -25,9 +25,9 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService<Block
   constructor(
     private readonly apiService: ApiService,
     nodeConfig: NodeConfig,
-    storeCache: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
   ) {
-    super(new StarknetNodeConfig(nodeConfig), storeCache);
+    super(new StarknetNodeConfig(nodeConfig), storeModelProvider);
   }
 
   protected async getFinalizedHead(): Promise<Header> {
@@ -41,7 +41,7 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService<Block
     return starknetBlockToHeader(formatBlock(block));
   }
 
-  protected async getHeaderForHeight(height: number): Promise<Header> {
+  async getHeaderForHeight(height: number): Promise<Header> {
     const block = await this.apiService.api.getBlockByHeightOrHash(height);
     return starknetBlockToHeader(formatBlock(block));
   }

packages/node/src/indexer/worker/worker.service.ts

@@ -11,14 +11,16 @@ import {
   BaseWorkerService,
   IProjectUpgradeService,
   IBlock,
+  Header,
 } from '@subql/node-core';
 import { StarknetProjectDs } from '../../configure/SubqueryProject';
 import { StarknetApi } from '../../starknet';
 import SafeStarknetProvider from '../../starknet/safe-api';
+import { starknetBlockToHeader } from '../../starknet/utils.starknet';
 import { IndexerManager } from '../indexer.manager';
-import { BlockContent } from '../types';
+import { BlockContent, getBlockSize } from '../types';
 
-export type FetchBlockResponse = { parentHash: string } | undefined;
+export type FetchBlockResponse = Header;
 
 export type WorkerStatusResponse = {
   threadId: number;
@@ -58,8 +60,9 @@ export class WorkerService extends BaseWorkerService<
     return block;
   }
 
-  protected toBlockResponse(block: BlockContent): { parentHash: string } {
+  protected toBlockResponse(block: BlockContent): Header {
     return {
+      ...starknetBlockToHeader(block),
       parentHash: block.parentHash,
     };
   }
@@ -70,4 +73,8 @@ export class WorkerService extends BaseWorkerService<
   ): Promise<ProcessBlockResponse> {
     return this.indexerManager.indexBlock(block, dataSources);
   }
+
+  getBlockSize(block: IBlock<BlockContent>): number {
+    return getBlockSize(block.block);
+  }
 }

packages/node/src/starknet/api.starknet.ts

@@ -257,8 +257,6 @@ export class StarknetApi implements ApiWrapper {
 
   // This follow method from official document https://starknetjs.com/docs/guides/events
   async fetchBlockLogs(blockNumber: number): Promise<StarknetLogRaw[]> {
-    logger.debug(`Fetch block ${blockNumber} events`);
-
     let continuationToken: string | undefined = '0';
     let chunkNum = 1;
     const allEvents: StarknetLogRaw[] = [];
@@ -288,15 +286,6 @@ export class StarknetApi implements ApiWrapper {
       }
       const nbEvents = eventsRes.events.length;
       continuationToken = eventsRes.continuation_token;
-      logger.debug(
-        'event chunk nb =',
-        chunkNum,
-        '.',
-        nbEvents,
-        'events recovered.',
-        'continuation_token =',
-        continuationToken,
-      );
       for (let i = 0; i < nbEvents; i++) {
         const event = eventsRes.events[i];
         allEvents.push(event);

packages/node/src/starknet/utils.starknet.ts

@@ -82,7 +82,7 @@ export function formatLog(
       return rawTransaction;
     },
   });
-  return formattedLog as unknown as StarknetLog<StarknetResult>;
+  return formattedLog as unknown as StarknetLog;
 }
 
 export function formatTransaction(
@@ -171,7 +171,7 @@ export function starknetBlockHeaderToHeader(block: SPEC.BLOCK_HEADER): Header {
 export function reverseToRawLog(log: StarknetLog): StarknetLogRaw {
   return {
     block_hash: log.blockHash,
-    keys: log.topics,
+    keys: [...log.topics],
     from_address: log.address,
     transaction_hash: log.transactionHash,
     block_number: log.blockNumber,

packages/node/src/subcommands/reindex.module.ts

@@ -2,25 +2,31 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import { Module } from '@nestjs/common';
-import { EventEmitterModule } from '@nestjs/event-emitter';
+import { EventEmitter2, EventEmitterModule } from '@nestjs/event-emitter';
 import { SchedulerRegistry } from '@nestjs/schedule';
 import {
   ApiService,
   DbModule,
   ForceCleanService,
-  StoreCacheService,
   StoreService,
   ReindexService,
   PoiService,
+  storeModelFactory,
+  NodeConfig,
 } from '@subql/node-core';
+import { Sequelize } from '@subql/x-sequelize';
 import { ConfigureModule } from '../configure/configure.module';
 import { DsProcessorService } from '../indexer/ds-processor.service';
 import { DynamicDsService } from '../indexer/dynamic-ds.service';
 import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 
 @Module({
   providers: [
-    StoreCacheService,
+    {
+      provide: 'IStoreModelProvider',
+      useFactory: storeModelFactory,
+      inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize],
+    },
     StoreService,
     ReindexService,
     PoiService,

packages/node/src/subcommands/testing.module.ts

@@ -13,11 +13,12 @@ import {
   NodeConfig,
   PoiService,
   PoiSyncService,
-  StoreCacheService,
   StoreService,
   TestRunner,
   SandboxService,
+  storeModelFactory,
 } from '@subql/node-core';
+import { Sequelize } from '@subql/x-sequelize';
 import { ConfigureModule } from '../configure/configure.module';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { DsProcessorService } from '../indexer/ds-processor.service';
@@ -32,7 +33,11 @@ import { StarknetApiConnection } from '../starknet/api.connection';
   providers: [
     InMemoryCacheService,
     StoreService,
-    StoreCacheService,
+    {
+      provide: 'IStoreModelProvider',
+      useFactory: storeModelFactory,
+      inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize],
+    },
     EventEmitter2,
     PoiService,
     PoiSyncService,