Port formula support from OnDemand tables to megagrist

ext/app/megagrist/lib/DataEngine.ts

@@ -1,18 +1,18 @@
 import {ActionSet, ApplyResultSet, Query, QueryResult, QueryResultStreaming, QuerySubId} from './types';
 import {CellValue} from './DocActions';
 import {IDataEngine, QueryStreamingOptions, QuerySubCallback} from './IDataEngine';
-import {BindParams, sqlSelectFromQuery} from './sqlConstruct';
+import {BindParams, ExpandedQuery, sqlSelectFromQuery} from './sqlConstruct';
 import {StoreDocAction} from './StoreDocAction';
 import SqliteDatabase from 'better-sqlite3';
 
 abstract class BaseDataEngine implements IDataEngine {
   private _querySubs = new Map<number, Query>();
   private _nextQuerySub = 1;
 
-  public async fetchQuery(query: Query): Promise<QueryResult> {
+  public async fetchQuery(query: ExpandedQuery): Promise<QueryResult> {
     const bindParams = new BindParams();
     const sql = sqlSelectFromQuery(query, bindParams);
-    // console.warn("RUNNING SQL", sql, bindParams.getParams());
+    // console.warn("fetchQuery", sql, bindParams.getParams());
     return this.withDB((db) => db.transaction(() => {
       const stmt = db.prepare(sql);
       const rows = stmt.raw().all(bindParams.getParams()) as CellValue[][];
@@ -30,11 +30,11 @@ abstract class BaseDataEngine implements IDataEngine {
   }
 
   public async fetchQueryStreaming(
-    query: Query, options: QueryStreamingOptions, abortSignal?: AbortSignal
+    query: ExpandedQuery, options: QueryStreamingOptions, abortSignal?: AbortSignal
   ): Promise<QueryResultStreaming> {
     const bindParams = new BindParams();
     const sql = sqlSelectFromQuery(query, bindParams);
-    // console.warn("RUNNING SQL", sql, bindParams.getParams());
+    // console.warn("fetchQueryStreaming", sql, bindParams.getParams());
 
     // Note the convoluted flow here: we are returning an object, which includes a generator.
     // Caller is expected to iterate through the generator. This iteration happens inside a DB

ext/app/megagrist/lib/sqlConstruct.ts

@@ -3,6 +3,16 @@ import {OrderByClause, ParsedPredicateFormula} from './types';
 import {CellValue} from './DocActions';
 import {quoteIdent} from './sqlUtil';
 
+// See ExpandedQuery in core/app/server/lib/ExpandedQuery.ts
+export interface ExpandedQuery extends Query {
+  // A list of join clauses to bring in data from other tables.
+  joins?: string[];
+
+  // A list of selections for regular data and data computed via formulas.
+  // If query.selects is present, then query.columns is ignored.
+  selects?: string[];
+}
+
 /**
  * When constructing a query, and values to include in the query are included as placeholders,
  * with the actual values collected in this BindParams object. They should be retrieved using
@@ -32,32 +42,37 @@ export class BindParams {
 /**
  * Construct SQL from the given query.
  */
-export function sqlSelectFromQuery(query: Query, params: BindParams): string {
-  const conditions = sqlSelectConditionsFromQuery('', query, params);
-  const columns = query.columns ? query.columns.map(c => quoteIdent(c)).join(", ") : '*';
-  return `SELECT ${columns} FROM ${quoteIdent(query.tableId)} ${conditions}`;
+export function sqlSelectFromQuery(query: ExpandedQuery, params: BindParams): string {
+  const namePrefix = `${quoteIdent(query.tableId)}.`;
+  const conditions = sqlSelectConditionsFromQuery(namePrefix, query, params);
+  const joinClauses = query.joins ? query.joins.join(' ') : '';
+  const selects = (
+    query.selects ? query.selects.join(', ') :
+    query.columns ? query.columns.map(c => quoteIdent(c)).join(", ") :
+    '*');
+  return `SELECT ${selects} FROM ${quoteIdent(query.tableId)} ${joinClauses} ${conditions}`;
 }
 
 /**
  * Construct just the portion of SQL starting with WHERE, i.e. all conditions including ORDER BY
  * and LIMIT. It also prefixes each mention of the column with namePrefix (like `quotedTableName.`
  * or '' to omit the prefix). This helps for using the SQL in JOINs.
  */
-export function sqlSelectConditionsFromQuery(namePrefix: string, query: Query, params: BindParams): string {
+export function sqlSelectConditionsFromQuery(namePrefix: string, query: ExpandedQuery, params: BindParams): string {
   const filterExpr = query.filters ? sqlExprFromFilters(namePrefix, query.filters, params) : null;
   const cursorExpr = query.cursor ? sqlExprFromCursor(namePrefix, query.sort, query.cursor, params) : null;
-  const rowsExpr = query.rowIds ? sqlExprFromRowIds(query.rowIds) : null;
+  const rowsExpr = query.rowIds ? sqlExprFromRowIds(namePrefix, query.rowIds) : null;
   const whereExpr = [filterExpr, cursorExpr, rowsExpr].filter(Boolean).map(expr => `(${expr})`).join(' AND ') || '1';
   const orderBy = `ORDER BY ${sqlOrderByFromSort(namePrefix, query.sort)}`;
   const limit = typeof query.limit === 'number' ? `LIMIT ${query.limit}` : '';
   return `WHERE ${whereExpr} ${orderBy} ${limit}`;
 }
 
-function sqlExprFromRowIds(rowIds: number[]) {
+function sqlExprFromRowIds(namePrefix: string, rowIds: number[]) {
   if (!rowIds.every(Number.isInteger)) {
     throw new Error("Expected all rowIds to be integers");
   }
-  return `id in (${rowIds})`;
+  return `${namePrefix}id in (${rowIds})`;
 }
 
 function sqlExprFromFilters(namePrefix: string, filters: ParsedPredicateFormula, params: BindParams): string {
@@ -110,7 +125,7 @@ function sqlOrderByFromSort(namePrefix: string, sort: OrderByClause|undefined):
       parts.push(`${fullColId} ${isDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'}`);
     }
   }
-  parts.push('id');
+  parts.push(namePrefix + 'id');
   return parts.join(', ');
 }
 

ext/app/server/lib/MegaDataEngine.ts

@@ -1,4 +1,5 @@
 import {CellValue} from 'app/common/DocActions';
+import {DocData} from 'app/common/DocData';
 import {EngineCode} from 'app/common/DocumentSettings';
 import {isListType} from 'app/common/gristTypes';
 import * as marshal from 'app/common/marshal';
@@ -9,7 +10,9 @@ import {DataEnginePooled} from 'app/megagrist/lib/DataEngine';
 import {createDataEngineServer} from 'app/megagrist/lib/DataEngineServer';
 import {QueryStreamingOptions} from 'app/megagrist/lib/IDataEngine';
 import {Query, QueryResultStreaming} from 'app/megagrist/lib/types';
+import {ExpandedQuery} from 'app/megagrist/lib/sqlConstruct';
 import {appSettings} from 'app/server/lib/AppSettings';
+import {expandQuery} from 'app/server/lib/ExpandedQuery';
 import * as ICreate from 'app/server/lib/ICreate';
 
 
@@ -27,15 +30,16 @@ export function getSupportedEngineChoices(): EngineCode[]|undefined {
 }
 
 export class MegaDataEngine {
-  public static maybeCreate(dbPath: string, engine?: string): MegaDataEngine|null {
+  public static maybeCreate(dbPath: string, docData: DocData): MegaDataEngine|null {
+    const engine = docData.docSettings().engine;
     const isEnabled = enableMegaDataEngine && engine === MEGA_ENGINE;
-    return isEnabled ? new MegaDataEngine(dbPath) : null;
+    return isEnabled ? new MegaDataEngine(dbPath, docData) : null;
   }
 
   private _dataEngine: DataEnginePooled;
 
-  constructor(dbPath: string) {
-    this._dataEngine = new UnmarshallingDataEngine(dbPath, {verbose: console.log});
+  constructor(dbPath: string, docData: DocData) {
+    this._dataEngine = new UnmarshallingDataEngine(docData, dbPath, {verbose: console.log});
     // db.exec("PRAGMA journal_mode=WAL");  <-- TODO we want this, but leaving for later.
   }
 
@@ -46,24 +50,23 @@ export class MegaDataEngine {
 }
 
 class UnmarshallingDataEngine extends DataEnginePooled {
+  constructor(private _docData: DocData, ...args: ConstructorParameters<typeof DataEnginePooled>) {
+    super(...args);
+  }
+
   public async fetchQueryStreaming(
     query: Query, options: QueryStreamingOptions, abortSignal?: AbortSignal
   ): Promise<QueryResultStreaming> {
 
-    // Get the grist types of columns, which determine how to decode values.
-    // TODO This is poor. If we do two queries, they need to be in a transaction, to ensure no
-    // changes to DB can happen in between. And also column types is something Grist already knows
-    // how to maintain in memory; seems silly to query (but fine, save for the transaction point).
-    let columnTypes: Map<string, string>;
-    this.withDB((db) => db.transaction(() => {
-      const stmt = db.prepare('SELECT colId, type'
-        + ' FROM _grist_Tables_column c LEFT JOIN _grist_Tables t ON c.parentID = t.id'
-        + ' WHERE t.tableId=?');
-      columnTypes = new Map(stmt.raw().all(query.tableId) as Array<[string, string]>);
-    })());
+    // We use the metadata about the table to get the grist types of columns, which determine how
+    // to decode values.
+    const tableData = this._docData.getTable(query.tableId)
+
+    const expanded = expandQuery({tableId: query.tableId, filters: {}}, this._docData, true);
+    const expandedQuery: ExpandedQuery = {...query, joins: expanded.joins, selects: expanded.selects};
 
-    const queryResult = await super.fetchQueryStreaming(query, options, abortSignal);
-    const decoders = queryResult.value.colIds.map(c => getDecoder(columnTypes.get(c)));
+    const queryResult = await super.fetchQueryStreaming(expandedQuery, options, abortSignal);
+    const decoders = queryResult.value.colIds.map(c => getDecoder(tableData?.getColType(c)));
 
     async function *generateRows() {
       for await (const chunk of queryResult.chunks) {