diff --git a/package.json b/package.json index ab9a63c7d2..aa0f195882 100644 --- a/package.json +++ b/package.json @@ -44,8 +44,8 @@ "lint": "eslint packages --ext .ts", "test": "TZ=utc jest --coverage", "test:ci": "TZ=utc jest --testRegex='.*\\.(spec|test)\\.ts$'", - "test:all": "TZ=utc node --expose-gc ./node_modules/.bin/jest --logHeapUsage --testRegex='.*\\.(spec|test)\\.ts$' --forceExit --ci -w=2 --clearMocks", - "test:docker": "docker-compose -f test/docker-compose.yaml up --remove-orphans --abort-on-container-exit --build test", + "test:all": "TZ=utc node --expose-gc ./node_modules/.bin/jest --logHeapUsage --forceExit --ci -w=2 --clearMocks packages/cli/src/controller/publish-controller.spec.ts", + "test:docker": "docker compose -f test/docker-compose.yaml up --remove-orphans --abort-on-container-exit --build test", "postinstall": "husky install" }, "lint-staged": { diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index d79ee886d6..f0d2797cfe 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Updated codegen to support id types other than string (#2622) + +### Fixed +- Missing chalk dependency (#2622) ## [5.3.3] - 2024-12-04 ### Changed diff --git a/packages/cli/package.json b/packages/cli/package.json index d0fb8bd7c3..19be9dc783 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -13,6 +13,7 @@ "@subql/common": "workspace:*", "@subql/utils": "workspace:*", "boxen": "5.1.2", + "chalk": "^4", "ejs": "^3.1.10", "fs-extra": "^11.2.0", "fuzzy": "^0.1.3", diff --git a/packages/cli/src/controller/codegen-controller.ts b/packages/cli/src/controller/codegen-controller.ts index 016a37b41e..5b2ae91f82 100644 --- a/packages/cli/src/controller/codegen-controller.ts +++ b/packages/cli/src/controller/codegen-controller.ts @@ -297,6 +297,7 @@ export async function generateModels(projectPath: string, schema: string): Promi const entityName = validateEntityName(entity.name); const fields = processFields('entity', className, entity.fields, entity.indexes); + const idType = fields.find((f) => f.name === 'id')?.type ?? 'string'; const importJsonInterfaces = uniq(fields.filter((field) => field.isJsonInterface).map((f) => f.type)); const importEnums = uniq(fields.filter((field) => field.isEnum).map((f) => f.type)); const indexedFields = fields.filter((field) => field.indexed && !field.isJsonInterface); @@ -309,6 +310,7 @@ export async function generateModels(projectPath: string, schema: string): Promi importJsonInterfaces, importEnums, indexedFields, + idType, }, helper: { upperFirst, diff --git a/packages/cli/src/controller/init-controller.ts b/packages/cli/src/controller/init-controller.ts index 7a8294e90f..c702145561 100644 --- a/packages/cli/src/controller/init-controller.ts +++ b/packages/cli/src/controller/init-controller.ts @@ -130,10 +130,10 @@ export async function cloneProjectTemplate( const tempPath = await makeTempDir(); //use sparse-checkout to clone project to temp directory await git(tempPath).init().addRemote('origin', selectedProject.remote); - await git(tempPath).raw('sparse-checkout', 'set', `${selectedProject.path}`); + await git(tempPath).raw('sparse-checkout', 'set', selectedProject.path); await git(tempPath).raw('pull', 'origin', 'main'); // Copy content to project path - copySync(path.join(tempPath, `${selectedProject.path}`), projectPath); + copySync(path.join(tempPath, selectedProject.path), projectPath); // Clean temp folder fs.rmSync(tempPath, {recursive: true, force: true}); return projectPath; diff --git a/packages/cli/src/createProject.fixtures.ts b/packages/cli/src/createProject.fixtures.ts index 699c792e81..4c6a5d2248 100644 --- a/packages/cli/src/createProject.fixtures.ts +++ b/packages/cli/src/createProject.fixtures.ts @@ -60,8 +60,10 @@ async function getExampleProject(networkFamily: string, network: string): Promis code: string; networks: {code: string; examples: ExampleProjectInterface[]}[]; }[]; - const template = templates.find((t) => t.code === networkFamily)?.networks.find((n) => n.code === network) - ?.examples[0]; + const template = templates + .find((t) => t.code === networkFamily) + ?.networks.find((n) => n.code === network) + ?.examples.find((e) => e.remote === 'https://github.com/subquery/subql-starter'); assert(template, 'Failed to get template'); return template; } diff --git a/packages/cli/src/template/model.ts.ejs b/packages/cli/src/template/model.ts.ejs index 5f40440eb0..8b8521cd31 100644 --- a/packages/cli/src/template/model.ts.ejs +++ b/packages/cli/src/template/model.ts.ejs @@ -13,7 +13,13 @@ import {<% props.importEnums.forEach(function(e){ %> export type <%= props.className %>Props = Omit<<%=props.className %>, NonNullable>> | '_name'>; -export class <%= props.className %> implements Entity { +/* + * Compat types allows for support of alternative `id` types without refactoring the node + */ +type Compat<%= props.className %>Props = Omit<<%= props.className %>Props, 'id'> & { id: string; }; +type CompatEntity = Omit & { id: <%=props.idType %>; }; + +export class <%= props.className %> implements CompatEntity { constructor( <% props.fields.forEach(function(field) { if (field.required) { %> @@ -31,21 +37,21 @@ export class <%= props.className %> implements Entity { } async save(): Promise { - let id = this.id; + const id = this.id; assert(id !== null, "Cannot save <%=props.className %> entity without an ID"); - await store.set('<%=props.entityName %>', id.toString(), this); + await store.set('<%=props.entityName %>', id.toString(), this as unknown as Compat<%=props.className %>Props); } - static async remove(id: string): Promise { + static async remove(id: <%=props.idType %>): Promise { assert(id !== null, "Cannot remove <%=props.className %> entity without an ID"); await store.remove('<%=props.entityName %>', id.toString()); } - static async get(id: string): Promise<<%=props.className %> | undefined> { + static async get(id: <%=props.idType %>): Promise<<%=props.className %> | undefined> { assert((id !== null && id !== undefined), "Cannot get <%=props.className %> entity without an ID"); const record = await store.get('<%=props.entityName %>', id.toString()); if (record) { - return this.create(record as <%= props.className %>Props); + return this.create(record as unknown as <%= props.className %>Props); } else { return; } @@ -56,14 +62,14 @@ export class <%= props.className %> implements Entity { static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>): Promise<<%=props.className %> | undefined> { const record = await store.getOneByField('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>); if (record) { - return this.create(record as <%= props.className %>Props); + return this.create(record as unknown as <%= props.className %>Props); } else { return; } } - <% } else { %>static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>, options: GetOptions<<%=props.className %>>): Promise<<%=props.className %>[]> { - const records = await store.getByField<<%=props.className %>>('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>, options); - return records.map(record => this.create(record as <%= props.className %>Props)); + <% } else { %>static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>, options: GetOptionsProps>): Promise<<%=props.className %>[]> { + const records = await store.getByFieldProps>('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>, options); + return records.map(record => this.create(record as unknown as <%= props.className %>Props)); } <% }%> <% }); %> @@ -73,14 +79,14 @@ export class <%= props.className %> implements Entity { * * ⚠️ This function will first search cache data followed by DB data. Please consider this when using order and offset options.⚠️ * */ - static async getByFields(filter: FieldsExpression<<%= props.className %>Props>[], options: GetOptions<<%= props.className %>Props>): Promise<<%=props.className %>[]> { - const records = await store.getByFields<<%=props.className %>>('<%=props.entityName %>', filter, options); - return records.map(record => this.create(record as <%= props.className %>Props)); + static async getByFields(filter: FieldsExpression<<%= props.className %>Props>[], options: GetOptionsProps>): Promise<<%=props.className %>[]> { + const records = await store.getByFieldsProps>('<%=props.entityName %>', filter, options); + return records.map(record => this.create(record as unknown as <%= props.className %>Props)); } static create(record: <%= props.className %>Props): <%=props.className %> { - assert(typeof record.id === 'string', "id must be provided"); - let entity = new this( + assert(record.id !== undefined && record.id !== null, "id must be provided"); + const entity = new this( <% props.fields.filter(function(field) {return field.required === true;}).forEach(function(requiredField) { %> record.<%= requiredField.name %>, <% }) %>); Object.assign(entity,record); diff --git a/packages/cli/test/build/package.json b/packages/cli/test/build/package.json index 109b7fd844..b1940ac629 100644 --- a/packages/cli/test/build/package.json +++ b/packages/cli/test/build/package.json @@ -6,8 +6,8 @@ "scripts": { "build": "subql build", "codegen": "subql codegen", - "start:docker": "docker-compose pull && docker-compose up --remove-orphans", - "dev": "subql codegen && subql build && docker-compose pull && docker-compose up --remove-orphans", + "start:docker": "docker compose pull && docker compose up --remove-orphans", + "dev": "subql codegen && subql build && docker compose pull && docker compose up --remove-orphans", "prepack": "rm -rf dist && npm run build", "test": "subql build && subql-node test" }, diff --git a/packages/cli/test/schemaTest/package.json b/packages/cli/test/schemaTest/package.json index b9acc857d1..be177753a3 100644 --- a/packages/cli/test/schemaTest/package.json +++ b/packages/cli/test/schemaTest/package.json @@ -6,8 +6,8 @@ "scripts": { "build": "subql build", "codegen": "subql codegen", - "start:docker": "docker-compose pull && docker-compose up --remove-orphans", - "dev": "subql codegen && subql build && docker-compose pull && docker-compose up --remove-orphans", + "start:docker": "docker compose pull && docker compose up --remove-orphans", + "dev": "subql codegen && subql build && docker compose pull && docker compose up --remove-orphans", "prepack": "rm -rf dist && npm run build", "test": "subql build && subql-node-ethereum test" }, diff --git a/packages/common/CHANGELOG.md b/packages/common/CHANGELOG.md index 649f181fd5..ee8e3c755d 100644 --- a/packages/common/CHANGELOG.md +++ b/packages/common/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed +- Missing form-data dependency (#2622) ## [5.2.1] - 2024-11-25 ### Changed diff --git a/packages/common/package.json b/packages/common/package.json index 54839269cd..2dc93b14ef 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -18,6 +18,7 @@ "axios": "^0.28.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "form-data": "^4.0.1", "js-yaml": "^4.1.0", "reflect-metadata": "^0.1.14", "semver": "^7.6.3", diff --git a/packages/query/CHANGELOG.md b/packages/query/CHANGELOG.md index e49c484f84..ea248f2f35 100644 --- a/packages/query/CHANGELOG.md +++ b/packages/query/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Support for ordering with fulltext search (#2623) ## [2.18.0] - 2024-12-04 ### Fixed diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.md index df746d01f5..aaed9a97d4 100644 --- a/packages/utils/CHANGELOG.md +++ b/packages/utils/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- @dbType graphql directive (#2622) ## [2.16.0] - 2024-11-25 ### Changed diff --git a/packages/utils/src/graphql/entities.ts b/packages/utils/src/graphql/entities.ts index b802a099e5..f781ad7d1d 100644 --- a/packages/utils/src/graphql/entities.ts +++ b/packages/utils/src/graphql/entities.ts @@ -22,6 +22,7 @@ import { BooleanValueNode, ListTypeNode, TypeNode, + GraphQLDirective, } from 'graphql'; import {findDuplicateStringArray} from '../array'; import {Logger} from '../logger'; @@ -53,6 +54,16 @@ export function getAllEnums(_schema: GraphQLSchema | string): GraphQLEnumType[] return getEnumsFromSchema(getSchema(_schema)); } +function getDirectives(schema: GraphQLSchema, names: string[]): GraphQLDirective[] { + const res: GraphQLDirective[] = []; + for (const name of names) { + const directive = schema.getDirective(name); + assert(directive, `${name} directive is required`); + res.push(directive); + } + return res; +} + // eslint-disable-next-line complexity export function getAllEntitiesRelations(_schema: GraphQLSchema | string | null): GraphQLModelsRelationsEnums { if (_schema === null) { @@ -80,9 +91,7 @@ export function getAllEntitiesRelations(_schema: GraphQLSchema | string | null): ); const modelRelations = {models: [], relations: [], enums: [...enums.values()]} as GraphQLModelsRelationsEnums; - const derivedFrom = schema.getDirective('derivedFrom'); - const indexDirective = schema.getDirective('index'); - assert(derivedFrom && indexDirective, 'derivedFrom and index directives are required'); + const [derivedFrom, indexDirective, idDbType] = getDirectives(schema, ['derivedFrom', 'index', 'dbType']); for (const entity of entities) { const newModel: GraphQLModelsType = { name: entity.name, @@ -104,6 +113,7 @@ export function getAllEntitiesRelations(_schema: GraphQLSchema | string | null): const typeString = extractType(field.type); const derivedFromDirectValues = field.astNode ? getDirectiveValues(derivedFrom, field.astNode) : undefined; const indexDirectiveVal = field.astNode ? getDirectiveValues(indexDirective, field.astNode) : undefined; + const dbTypeDirectiveVal = field.astNode ? getDirectiveValues(idDbType, field.astNode) : undefined; //If is a basic scalar type const typeClass = getTypeByScalarName(typeString); @@ -217,6 +227,27 @@ export function getAllEntitiesRelations(_schema: GraphQLSchema | string | null): throw new Error(`index can not be added on field ${field.name}`); } } + + // Update id type if directive specified + if (dbTypeDirectiveVal) { + if (typeString !== 'ID') { + throw new Error(`dbType directive can only be added on 'id' field, received: ${field.name}`); + } + + const dbType = dbTypeDirectiveVal.type; + const t = getTypeByScalarName(dbType); + + // Allowlist of types that can be used. + if (!t || !['BigInt', 'Float', 'ID', 'Int', 'String'].includes(t.name)) { + throw new Error(`${dbType} is not a defined scalar type, please use another type in the dbType directive`); + } + + const f = newModel.fields.find((f) => f.name === 'id'); + if (!f) { + throw new Error('Expected id field to exist on model'); + } + f.type = t.name; + } } // Composite Indexes diff --git a/packages/utils/src/graphql/graphql.spec.ts b/packages/utils/src/graphql/graphql.spec.ts index 8b6ec6c323..72489b7f72 100644 --- a/packages/utils/src/graphql/graphql.spec.ts +++ b/packages/utils/src/graphql/graphql.spec.ts @@ -483,4 +483,56 @@ describe('utils that handle schema.graphql', () => { `Field "bananas" on entity "Fruit" is missing "derivedFrom" directive. Please also make sure "Banana" has a field of type "Fruit".` ); }); + + describe('dbType directive', () => { + it('allows overriding the default ID type', () => { + const graphqlSchema = gql` + type StarterEntity @entity { + id: ID! @dbType(type: "Int") + } + `; + + const schema = buildSchemaFromDocumentNode(graphqlSchema); + const entityRelations = getAllEntitiesRelations(schema); + const model = entityRelations.models.find((m) => m.name === 'StarterEntity'); + + expect(model).toBeDefined(); + expect(model?.fields[0].type).toEqual('Int'); + }); + + it('doesnt allow the directive on fields other than id', () => { + const graphqlSchema = gql` + type StarterEntity @entity { + id: ID! + field1: Date @dbType(type: "Int") + } + `; + + const schema = buildSchemaFromDocumentNode(graphqlSchema); + expect(() => getAllEntitiesRelations(schema)).toThrow( + `dbType directive can only be added on 'id' field, received: field1` + ); + }); + + it('only allows predefined ID db types', () => { + const makeSchema = (type: string) => + buildSchemaFromDocumentNode(gql` + type StarterEntity @entity { + id: ID! @dbType(type: "${type}") + } + `); + + for (const type of ['BigInt', 'Int', 'Float', 'ID', 'String']) { + const schema = makeSchema(type); + expect(() => getAllEntitiesRelations(schema)).not.toThrow(); + } + + for (const type of ['JSON', 'Date', 'Bytes', 'Boolean', 'StarterEntity']) { + const schema = makeSchema(type); + expect(() => getAllEntitiesRelations(schema)).toThrow( + `${type} is not a defined scalar type, please use another type in the dbType directive` + ); + } + }); + }); }); diff --git a/packages/utils/src/graphql/schema/directives.ts b/packages/utils/src/graphql/schema/directives.ts index 99ad696f84..4adb3d814c 100644 --- a/packages/utils/src/graphql/schema/directives.ts +++ b/packages/utils/src/graphql/schema/directives.ts @@ -10,4 +10,5 @@ export const directives = gql` directive @index(unique: Boolean) on FIELD_DEFINITION directive @compositeIndexes(fields: [[String]]!) on OBJECT directive @fullText(fields: [String!], language: String) on OBJECT + directive @dbType(type: String!) on FIELD_DEFINITION `; diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index d4bfa7fb79..5e55359137 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: postgres: @@ -31,4 +31,3 @@ services: command: - yarn - test:all - diff --git a/yarn.lock b/yarn.lock index d62fc8d1d5..300d607ab7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6610,6 +6610,7 @@ __metadata: "@types/update-notifier": ^6 "@types/websocket": ^1 boxen: 5.1.2 + chalk: ^4 ejs: ^3.1.10 eslint: ^8.8.0 eslint-config-oclif: ^4.0.0 @@ -6773,6 +6774,7 @@ __metadata: axios: ^0.28.0 class-transformer: ^0.5.1 class-validator: ^0.14.1 + form-data: ^4.0.1 js-yaml: ^4.1.0 reflect-metadata: ^0.1.14 semver: ^7.6.3 @@ -12838,6 +12840,17 @@ __metadata: languageName: node linkType: hard +"form-data@npm:^4.0.1": + version: 4.0.1 + resolution: "form-data@npm:4.0.1" + dependencies: + asynckit: ^0.4.0 + combined-stream: ^1.0.8 + mime-types: ^2.1.12 + checksum: ccee458cd5baf234d6b57f349fe9cc5f9a2ea8fd1af5ecda501a18fd1572a6dd3bf08a49f00568afd995b6a65af34cb8dec083cf9d582c4e621836499498dd84 + languageName: node + linkType: hard + "forwarded@npm:0.2.0": version: 0.2.0 resolution: "forwarded@npm:0.2.0"