historical timestamp and db store (#298)

* historical timestamp and db store

* fix inject

packages/common-cosmos/CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- Bump `@subql/common` dependency
+
 ## [5.0.5] - 2024-10-23
 ### Changed
 - Bump `@subql/common` dependency

packages/common-cosmos/package.json

@@ -22,7 +22,7 @@
     "@protobufs/google": "^0.0.10",
     "@protobufs/ibc": "^0.1.0",
     "@protobufs/tendermint": "^0.0.10",
-    "@subql/common": "^5.1.4",
+    "@subql/common": "^5.2.1",
     "@subql/types-cosmos": "workspace:*",
     "@subql/x-cosmology-telescope": "^1.4.14",
     "fs-extra": "^11.1.1",

packages/node/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Support for historical indexing by timestamp as well as block height
+- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations.
+
 ## [4.1.4] - 2024-10-23
 ### Changed
 - Bump `@subql/common` dependency

packages/node/package.json

@@ -29,9 +29,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^5.1.4",
+    "@subql/common": "^5.2.1",
     "@subql/common-cosmos": "workspace:*",
-    "@subql/node-core": "^14.1.6",
+    "@subql/node-core": "^15.0.2",
     "@subql/types-cosmos": "workspace:*",
     "lodash": "^4.17.21",
     "protobufjs": "^6.11.4",

packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts

@@ -5,7 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
-  StoreCacheService,
+  IStoreModelProvider,
   StoreService,
   IProjectService,
   PoiSyncService,
@@ -38,7 +38,7 @@ export class BlockDispatcherService
     @Inject('IProjectUpgradeService')
     projectUpgradeService: IProjectUpgradeService,
     storeService: StoreService,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
   ) {
@@ -48,7 +48,7 @@ export class BlockDispatcherService
       projectService,
       projectUpgradeService,
       storeService,
-      storeCacheService,
+      storeModelProvider,
       poiSyncService,
       project,
       apiService.fetchBlocks.bind(apiService),

packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts

@@ -9,7 +9,7 @@ import {
   NodeConfig,
   StoreService,
   PoiSyncService,
-  StoreCacheService,
+  IStoreModelProvider,
   IProjectService,
   WorkerBlockDispatcher,
   ConnectionPoolStateManager,
@@ -24,6 +24,7 @@ import { DynamicDsService } from '../dynamic-ds.service';
 import { BlockContent } from '../types';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
 import { IIndexerWorker } from '../worker/worker';
+import { FetchBlockResponse } from '../worker/worker.service';
 
 type IndexerWorker = IIndexerWorker & {
   terminate: () => Promise<number>;
@@ -43,7 +44,7 @@ export class WorkerBlockDispatcherService
     projectUpgadeService: IProjectUpgradeService,
     cacheService: InMemoryCacheService,
     storeService: StoreService,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
     poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
@@ -57,7 +58,7 @@ export class WorkerBlockDispatcherService
       projectService,
       projectUpgadeService,
       storeService,
-      storeCacheService,
+      storeModelProvider,
       poiSyncService,
       project,
       () =>
@@ -92,9 +93,9 @@ export class WorkerBlockDispatcherService
   protected async fetchBlock(
     worker: IndexerWorker,
     height: number,
-  ): Promise<void> {
+  ): Promise<FetchBlockResponse> {
     // const start = new Date();
-    await worker.fetchBlock(height, 0 /* Value is not used with cosmos*/);
+    return worker.fetchBlock(height, 0 /* Value is not used with cosmos*/);
     // const end = new Date();
 
     // const waitTime = end.getTime() - start.getTime();

packages/node/src/indexer/fetch.module.ts

@@ -8,7 +8,7 @@ import {
   StoreService,
   PoiSyncService,
   NodeConfig,
-  StoreCacheService,
+  IStoreModelProvider,
   ConnectionPoolStateManager,
   IProjectUpgradeService,
   InMemoryCacheService,
@@ -55,7 +55,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         indexerManager: IndexerManager,
         cacheService: InMemoryCacheService,
         storeService: StoreService,
-        storeCacheService: StoreCacheService,
+        storeModelProvider: IStoreModelProvider,
         poiSyncService: PoiSyncService,
         project: SubqueryProject,
         dynamicDsService: DynamicDsService,
@@ -71,7 +71,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectUpgradeService,
               cacheService,
               storeService,
-              storeCacheService,
+              storeModelProvider,
               poiSyncService,
               project,
               dynamicDsService,
@@ -87,7 +87,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectService,
               projectUpgradeService,
               storeService,
-              storeCacheService,
+              storeModelProvider,
               poiSyncService,
               project,
             ),
@@ -100,7 +100,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         IndexerManager,
         InMemoryCacheService,
         StoreService,
-        StoreCacheService,
+        'IStoreModelProvider',
         PoiSyncService,
         'ISubqueryProject',
         DynamicDsService,

packages/node/src/indexer/fetch.service.ts

@@ -14,7 +14,7 @@ import {
   BaseFetchService,
   getModulos,
   Header,
-  StoreCacheService,
+  IStoreModelProvider,
 } from '@subql/node-core';
 import * as CosmosUtil from '../utils/cosmos';
 import { cosmosBlockToHeader } from '../utils/cosmos';
@@ -44,7 +44,7 @@ export class FetchService extends BaseFetchService<
     unfinalizedBlocksService: UnfinalizedBlocksService,
     eventEmitter: EventEmitter2,
     schedulerRegistry: SchedulerRegistry,
-    storeCacheService: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
   ) {
     super(
       nodeConfig,
@@ -54,7 +54,7 @@ export class FetchService extends BaseFetchService<
       eventEmitter,
       schedulerRegistry,
       unfinalizedBlocksService,
-      storeCacheService,
+      storeModelProvider,
     );
   }
 

packages/node/src/indexer/unfinalizedBlocks.service.ts

@@ -1,12 +1,12 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import {
   BaseUnfinalizedBlocksService,
   Header,
   NodeConfig,
-  StoreCacheService,
+  IStoreModelProvider,
   mainThreadOnly,
 } from '@subql/node-core';
 import { cosmosBlockToHeader } from '../utils/cosmos';
@@ -18,9 +18,9 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService<Block
   constructor(
     private readonly apiService: ApiService,
     nodeConfig: NodeConfig,
-    storeCache: StoreCacheService,
+    @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider,
   ) {
-    super(nodeConfig, storeCache);
+    super(nodeConfig, storeModelProvider);
   }
 
   @mainThreadOnly()
@@ -34,7 +34,7 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService<Block
   }
 
   @mainThreadOnly()
-  protected async getHeaderForHeight(height: number): Promise<Header> {
+  async getHeaderForHeight(height: number): Promise<Header> {
     return Promise.resolve(cosmosBlockToHeader(height));
   }
 }

packages/node/src/indexer/worker/worker.service.ts

@@ -11,15 +11,15 @@ import {
   BaseWorkerService,
   IProjectUpgradeService,
   IBlock,
+  Header,
 } from '@subql/node-core';
 import { CosmosDatasource } from '@subql/types-cosmos';
+import { cosmosBlockToHeader } from '../../utils/cosmos';
 import { ApiService } from '../api.service';
 import { IndexerManager } from '../indexer.manager';
 import { BlockContent } from '../types';
 
-export type FetchBlockResponse = {
-  parentHash: string | undefined;
-};
+export type FetchBlockResponse = Header;
 
 export type WorkerStatusResponse = {
   threadId: number;
@@ -58,9 +58,7 @@ export class WorkerService extends BaseWorkerService<
   }
 
   protected toBlockResponse(block: BlockContent): FetchBlockResponse {
-    return {
-      parentHash: block.block.header.lastBlockId?.hash.toString(),
-    };
+    return cosmosBlockToHeader(block.block.header.height);
   }
 
   protected async processFetchedBlock(

packages/node/src/subcommands/reindex.module.ts

@@ -2,16 +2,18 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import { Module } from '@nestjs/common';
-import { EventEmitterModule } from '@nestjs/event-emitter';
+import { EventEmitter2, EventEmitterModule } from '@nestjs/event-emitter';
 import { SchedulerRegistry } from '@nestjs/schedule';
 import {
   DbModule,
   ForceCleanService,
   ReindexService,
-  StoreCacheService,
   StoreService,
   PoiService,
+  storeModelFactory,
+  NodeConfig,
 } from '@subql/node-core';
+import { Sequelize } from '@subql/x-sequelize';
 import { ConfigureModule } from '../configure/configure.module';
 import { ApiService } from '../indexer/api.service';
 import { DsProcessorService } from '../indexer/ds-processor.service';
@@ -20,7 +22,11 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 
 @Module({
   providers: [
-    StoreCacheService,
+    {
+      provide: 'IStoreModelProvider',
+      useFactory: storeModelFactory,
+      inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize],
+    },
     StoreService,
     ReindexService,
     PoiService,

packages/node/src/subcommands/testing.module.ts

@@ -12,11 +12,12 @@ import {
   NodeConfig,
   PoiService,
   PoiSyncService,
-  StoreCacheService,
   StoreService,
   TestRunner,
   SandboxService,
+  storeModelFactory,
 } from '@subql/node-core';
+import { Sequelize } from '@subql/x-sequelize';
 import { ConfigureModule } from '../configure/configure.module';
 import { ApiService } from '../indexer/api.service';
 import { DsProcessorService } from '../indexer/ds-processor.service';
@@ -29,7 +30,11 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
   providers: [
     InMemoryCacheService,
     StoreService,
-    StoreCacheService,
+    {
+      provide: 'IStoreModelProvider',
+      useFactory: storeModelFactory,
+      inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize],
+    },
     EventEmitter2,
     PoiService,
     PoiSyncService,