feat: new design for design page (#1210)

* feat: new design for design page

* feat: add integrity check

* chore: remove depends

* feat: fix integrity

* fix: allowed sync copy paste

* fix: redis for preview env

* fix: baseId issue

* fix: redis template

* fix: sqlite error

.github/workflows/manual-preview.yml

@@ -84,6 +84,7 @@ jobs:
     if: needs.check-pr.outputs.should_deploy == 'true'
     runs-on: ubuntu-latest
     env:
+      NAMESPACE: 38puz7wo
       INSTANCE_NAME: pr-${{ github.event.pull_request.number }}
       INSTANCE_DOMAIN: pr-${{ github.event.pull_request.number }}
       DISPLAY_NAME: "teable-pr-${{ github.event.pull_request.number }}"
@@ -97,6 +98,7 @@ jobs:
     - name: Create deployment YAML
       run: |
         cp .github/workflows/templates/preview-template.yaml deploy.yaml
+        sed -i "s#__NAMESPACE__#${{ env.NAMESPACE }}#g" deploy.yaml
         sed -i "s#__INSTANCE_NAME__#${{ env.INSTANCE_NAME }}#g" deploy.yaml
         sed -i "s#__INSTANCE_DOMAIN__#${{ env.INSTANCE_DOMAIN }}#g" deploy.yaml
         sed -i "s#__MAIN_IMAGE_REPOSITORY__#${{ env.MAIN_IMAGE_REPOSITORY }}#g" deploy.yaml
@@ -133,7 +135,7 @@ jobs:
       with:
         script: |
           const success = process.env.JOB_STATUS === 'success';
-          const deploymentUrl = `https://${process.env.INSTANCE_DOMAIN}.sealosgzg.site`;
+          const deploymentUrl = `https://${process.env.INSTANCE_DOMAIN}.sealoshzh.site`;
           const status = success ? '✅ Success' : '❌ Failed';
           console.log(process.env.JOB_STATUS);
 

.github/workflows/preview-cleanup.yml

@@ -5,6 +5,7 @@ on:
     types: [closed]
 
 env:
+  NAMESPACE: 38puz7wo
   INSTANCE_NAME: pr-${{ github.event.pull_request.number }}
   INSTANCE_DOMAIN: pr-${{ github.event.pull_request.number }}
   DISPLAY_NAME: "teable-pr-${{ github.event.pull_request.number }}"
@@ -22,6 +23,7 @@ jobs:
       - name: Create deployment YAML
         run: |
           cp .github/workflows/templates/preview-template.yaml deploy.yaml
+          sed -i "s#__NAMESPACE__#${{ env.NAMESPACE }}#g" deploy.yaml
           sed -i "s#__INSTANCE_NAME__#${{ env.INSTANCE_NAME }}#g" deploy.yaml
           sed -i "s#__INSTANCE_DOMAIN__#${{ env.INSTANCE_DOMAIN }}#g" deploy.yaml
           sed -i "s#__MAIN_IMAGE_REPOSITORY__#${{ env.MAIN_IMAGE_REPOSITORY }}#g" deploy.yaml

.github/workflows/templates/preview-template.yaml

@@ -78,7 +78,7 @@ spec:
                   key: port
             - name: PRISMA_DATABASE_URL
               value: >-
-                postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.ns-rbcflj0l.svc:$(PG_PORT)/teable
+                postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.__NAMESPACE__.svc:$(PG_PORT)/teable
             - name: PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING
               value: '1'
           resources:
@@ -105,9 +105,9 @@ spec:
                   key: port
             - name: PRISMA_DATABASE_URL
               value: >-
-                postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.ns-rbcflj0l.svc:$(PG_PORT)/teable
+                postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.__NAMESPACE__.svc:$(PG_PORT)/teable
             - name: PUBLIC_ORIGIN
-              value: https://__INSTANCE_DOMAIN__.sealosgzg.site
+              value: https://__INSTANCE_DOMAIN__.sealoshzh.site
             - name: BACKEND_JWT_SECRET
               value: exdpbfxmlqhjnqxu
             - name: BACKEND_SESSION_SECRET
@@ -176,7 +176,7 @@ spec:
                   key: password
             - name: BACKEND_CACHE_REDIS_URI
               value: >-
-                redis://$(REDIS_USERNAME):$(REDIS_PASSWORD)@$(REDIS_HOST).ns-rbcflj0l.svc:$(REDIS_PORT)/1
+                redis://$(REDIS_USERNAME):$(REDIS_PASSWORD)@$(REDIS_HOST).__NAMESPACE__.svc:$(REDIS_PORT)/1
             - name: NEXT_ENV_IMAGES_ALL_REMOTE
               value: 'true'
           resources:
@@ -236,7 +236,7 @@ metadata:
     nginx.ingress.kubernetes.io/proxy-read-timeout: '300'
 spec:
   rules:
-    - host: __INSTANCE_DOMAIN__.sealosgzg.site
+    - host: __INSTANCE_DOMAIN__.sealoshzh.site
       http:
         paths:
           - pathType: Prefix
@@ -248,7 +248,7 @@ spec:
                   number: 3000
   tls:
     - hosts:
-        - __INSTANCE_DOMAIN__.sealosgzg.site
+        - __INSTANCE_DOMAIN__.sealoshzh.site
       secretName: wildcard-cert
 
 ---
@@ -306,7 +306,6 @@ spec:
             resources:
               requests:
                 storage: 1Gi
-            storageClassName: openebs-backup
   terminationPolicy: Delete
   tolerations: []
 
@@ -368,7 +367,7 @@ spec:
                   key: password
             - name: DATABASE_URL
               value: >-
-                postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.ns-rbcflj0l.svc:5432
+                postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.__NAMESPACE__.svc:5432
           command:
             - /bin/sh
             - '-c'
@@ -418,8 +417,8 @@ spec:
       replicas: 1
       resources:
         limits:
-          cpu: 300m
-          memory: 384Mi
+          cpu: 500m
+          memory: 512Mi
         requests:
           cpu: 100m
           memory: 102Mi
@@ -434,7 +433,6 @@ spec:
             resources:
               requests:
                 storage: 1Gi
-            storageClassName: openebs-backup
     - componentDefRef: redis-sentinel
       monitor: true
       name: redis-sentinel
@@ -485,7 +483,7 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: teable-__INSTANCE_NAME__-redis
-    namespace: ns-rbcflj0l
+    namespace: __NAMESPACE__
 
 ---
 apiVersion: objectstorage.sealos.io/v1
@@ -517,7 +515,7 @@ metadata:
     cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__
 spec:
   data:
-    url: https://__INSTANCE_DOMAIN__.sealosgzg.site
+    url: https://__INSTANCE_DOMAIN__.sealoshzh.site
   displayType: normal
   icon: https://framerusercontent.com/images/x9gZmjwbtvaGd95qbfUmsZ8Jc.png
   name: __DISPLAY_NAME__

apps/nestjs-backend/src/app.module.ts

@@ -19,6 +19,7 @@ import { ExportOpenApiModule } from './features/export/open-api/export-open-api.
 import { FieldOpenApiModule } from './features/field/open-api/field-open-api.module';
 import { HealthModule } from './features/health/health.module';
 import { ImportOpenApiModule } from './features/import/open-api/import-open-api.module';
+import { IntegrityModule } from './features/integrity/integrity.module';
 import { InvitationModule } from './features/invitation/invitation.module';
 import { NextModule } from './features/next/next.module';
 import { NotificationModule } from './features/notification/notification.module';
@@ -45,6 +46,7 @@ export const appModules = {
     NextModule,
     FieldOpenApiModule,
     BaseModule,
+    IntegrityModule,
     ChatModule,
     AttachmentsModule,
     WsModule,

apps/nestjs-backend/src/db-provider/db.provider.interface.ts

@@ -1,4 +1,4 @@
-import type { DriverClient, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
+import type { DriverClient, FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
 import type { Prisma } from '@teable/db-main-prisma';
 import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
 import type { Knex } from 'knex';
@@ -72,6 +72,8 @@ export interface IDbProvider {
     prisma: Prisma.TransactionClient
   ): Promise<boolean>;
 
+  checkTableExist(tableName: string): string;
+
   dropColumnAndIndex(tableName: string, columnName: string, indexName: string): string[];
 
   modifyColumnSchema(tableName: string, columnName: string, schemaType: SchemaType): string[];
@@ -165,5 +167,5 @@ export interface IDbProvider {
 
   lookupOptionsQuery(optionsKey: keyof ILookupOptionsVo, value: string): string;
 
-  optionsQuery(optionsKey: string, value: string): string;
+  optionsQuery(type: FieldType, optionsKey: string, value: string): string;
 }

apps/nestjs-backend/src/db-provider/postgres.provider.ts

@@ -1,6 +1,6 @@
 /* eslint-disable sonarjs/no-duplicate-string */
 import { Logger } from '@nestjs/common';
-import type { IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
+import type { FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
 import { DriverClient } from '@teable/core';
 import type { PrismaClient } from '@teable/db-main-prisma';
 import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
@@ -75,6 +75,16 @@ export class PostgresProvider implements IDbProvider {
     return res[0].exists;
   }
 
+  checkTableExist(tableName: string): string {
+    const [schemaName, dbTableName] = this.splitTableName(tableName);
+    return this.knex
+      .raw(
+        'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema = ? AND table_name = ?) AS exists',
+        [schemaName, dbTableName]
+      )
+      .toQuery();
+  }
+
   renameColumn(tableName: string, oldName: string, newName: string): string[] {
     return this.knex.schema
       .alterTable(tableName, (table) => {
@@ -424,20 +434,29 @@ export class PostgresProvider implements IDbProvider {
   lookupOptionsQuery(optionsKey: keyof ILookupOptionsVo, value: string): string {
     return this.knex('field')
       .select({
+        tableId: 'table_id',
         id: 'id',
+        type: 'type',
+        name: 'name',
         lookupOptions: 'lookup_options',
       })
+      .whereNull('deleted_time')
       .whereRaw(`lookup_options::json->>'${optionsKey}' = ?`, [value])
       .toQuery();
   }
 
-  optionsQuery(optionsKey: string, value: string): string {
+  optionsQuery(type: FieldType, optionsKey: string, value: string): string {
     return this.knex('field')
       .select({
+        tableId: 'table_id',
         id: 'id',
+        type: 'type',
+        name: 'name',
         options: 'options',
       })
+      .whereNull('deleted_time')
       .whereRaw(`options::json->>'${optionsKey}' = ?`, [value])
+      .where('type', type)
       .toQuery();
   }
 }

apps/nestjs-backend/src/db-provider/sqlite.provider.ts

@@ -1,6 +1,6 @@
 /* eslint-disable sonarjs/no-duplicate-string */
 import { Logger } from '@nestjs/common';
-import type { IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
+import type { FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core';
 import { DriverClient } from '@teable/core';
 import type { PrismaClient } from '@teable/db-main-prisma';
 import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi';
@@ -65,6 +65,18 @@ export class SqliteProvider implements IDbProvider {
     return columns.some((column) => column.name === columnName);
   }
 
+  checkTableExist(tableName: string): string {
+    return this.knex
+      .raw(
+        `SELECT EXISTS (
+          SELECT 1 FROM sqlite_master 
+          WHERE type='table' AND name = ?
+        ) as "exists"`,
+        [tableName]
+      )
+      .toQuery();
+  }
+
   renameColumn(tableName: string, oldName: string, newName: string): string[] {
     return [
       this.knex
@@ -382,18 +394,25 @@ export class SqliteProvider implements IDbProvider {
     return this.knex('field')
       .select({
         id: 'id',
+        type: 'type',
+        name: 'name',
         lookupOptions: 'lookup_options',
       })
+      .whereNull('deleted_time')
       .whereRaw(`json_extract(lookup_options, '$."${optionsKey}"') = ?`, [value])
       .toQuery();
   }
 
-  optionsQuery(optionsKey: string, value: string): string {
+  optionsQuery(type: FieldType, optionsKey: string, value: string): string {
     return this.knex('field')
       .select({
         id: 'id',
+        type: 'type',
+        name: 'name',
         options: 'options',
       })
+      .where('type', type)
+      .whereNull('deleted_time')
       .whereRaw(`json_extract(options, '$."${optionsKey}"') = ?`, [value])
       .toQuery();
   }

apps/nestjs-backend/src/features/base/base.controller.ts

@@ -140,7 +140,7 @@ export class BaseController {
 
   @Permissions('base|db_connection')
   @Post(':baseId/connection')
-  async createDbConnection(@Param('baseId') baseId: string): Promise<IDbConnectionVo> {
+  async createDbConnection(@Param('baseId') baseId: string): Promise<IDbConnectionVo | null> {
     return await this.dbConnectionService.create(baseId);
   }
 

apps/nestjs-backend/src/features/base/db-connection.service.ts

@@ -2,7 +2,7 @@ import {
   BadRequestException,
   Injectable,
   InternalServerErrorException,
-  NotFoundException,
+  Logger,
 } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import type { IDsn } from '@teable/core';
@@ -12,17 +12,16 @@ import type { IDbConnectionVo } from '@teable/openapi';
 import { Knex } from 'knex';
 import { nanoid } from 'nanoid';
 import { InjectModel } from 'nest-knexjs';
-import { ClsService } from 'nestjs-cls';
 import { BaseConfig, type IBaseConfig } from '../../configs/base.config';
 import { InjectDbProvider } from '../../db-provider/db.provider';
 import { IDbProvider } from '../../db-provider/db.provider.interface';
-import type { IClsStore } from '../../types/cls';
 
 @Injectable()
 export class DbConnectionService {
+  private readonly logger = new Logger(DbConnectionService.name);
+
   constructor(
     private readonly prismaService: PrismaService,
-    private readonly cls: ClsService<IClsStore>,
     private readonly configService: ConfigService,
     @InjectDbProvider() private readonly dbProvider: IDbProvider,
     @InjectModel('CUSTOM_KNEX') private readonly knex: Knex,
@@ -118,7 +117,8 @@ export class DbConnectionService {
     const readOnlyRole = `read_only_role_${baseId}`;
     const publicDatabaseProxy = this.baseConfig.publicDatabaseProxy;
     if (!publicDatabaseProxy) {
-      throw new NotFoundException('PUBLIC_DATABASE_PROXY is not found in env');
+      this.logger.error('PUBLIC_DATABASE_PROXY is not found in env');
+      return null;
     }
 
     const { hostname: dbHostProxy, port: dbPortProxy } = new URL(`https://${publicDatabaseProxy}`);
@@ -185,7 +185,8 @@ export class DbConnectionService {
       const password = nanoid();
       const publicDatabaseProxy = this.baseConfig.publicDatabaseProxy;
       if (!publicDatabaseProxy) {
-        throw new NotFoundException('PUBLIC_DATABASE_PROXY is not found in env');
+        this.logger.error('PUBLIC_DATABASE_PROXY is not found in env');
+        return null;
       }
 
       const { hostname: dbHostProxy, port: dbPortProxy } = new URL(

apps/nestjs-backend/src/features/calculation/reference.service.ts

@@ -343,6 +343,7 @@ export class ReferenceService {
       if (!fromRecordIds?.length && !toRecordIds?.length) {
         continue;
       }
+
       const relatedRecordItems = await this.getAffectedRecordItems({
         fieldId,
         fieldMap,
@@ -583,6 +584,13 @@ export class ReferenceService {
       : (field.options as ILinkFieldOptions);
     const { relationship } = lookupOptions;
     const linkFieldId = field.lookupOptions ? field.lookupOptions.linkFieldId : field.id;
+
+    if (!recordItem.record?.fields) {
+      console.log('recordItem', JSON.stringify(recordItem, null, 2));
+      console.log('recordItem.field', field);
+      throw new InternalServerErrorException('record fields is undefined');
+    }
+
     const cellValue = recordItem.record.fields[linkFieldId];
     const dependenciesIndexed = keyBy(dependencies, 'id');
 

apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts

@@ -276,8 +276,22 @@ export class FieldSupplementService {
   }
 
   private async prepareLinkField(tableId: string, field: IFieldRo) {
-    const options = field.options as ILinkFieldOptionsRo;
-    const { relationship, foreignTableId } = options;
+    let options = field.options as ILinkFieldOptionsRo;
+    const { baseId, relationship, foreignTableId } = options;
+
+    // if link target is in the same base, we should not set baseId
+    if (baseId) {
+      const tableMeta = await this.prismaService.tableMeta.findFirstOrThrow({
+        where: { id: tableId, deletedTime: null },
+        select: { id: true, baseId: true },
+      });
+      if (tableMeta.baseId === baseId) {
+        options = {
+          ...options,
+          baseId: undefined,
+        };
+      }
+    }
 
     const fieldId = field.id ?? generateFieldId();
     const optionsVo = await this.generateNewLinkOptionsVo(tableId, fieldId, options);