Fix slow hierarchy filtering with large number of filtered paths (#711)

* Add performance test * Remove FilteredChildrenPaths from query * Run extract-api and update extractions * Add changeset * Update changeset comment * Add filterPathsIdentifierPositions * Remove added test * Fix tests and cleanup * Run extract-api, update test name * Fix comments * Add back new lines * Update changeset * Change eCInstanceIdCondition * Adjust Filtering.test.ts * Adjust import order * Update packages/hierarchies/src/hierarchies/imodel/FilteringHierarchyDefinition.ts Co-authored-by: Grigas <[email protected]> * Adjust import order & change ECSQL_COLUMN_NAME_FilterECInstanceId to return id64 string * Fix comments * Update .changeset/tall-comics-compare.md Co-authored-by: Grigas <[email protected]> * Add watch to performance-tests, fix benchmark * Add tests * Fix filtering * Add unit test * Cleanup integration test `filters through instance nodes that are in multiple paths` * Update packages/hierarchies/src/hierarchies/imodel/FilteringHierarchyDefinition.ts Co-authored-by: Grigas <[email protected]> * Create a separate class for handling positions * Remove unnecessarily exported class * Add parentNode to parseNode * Run extract api * Fix comments * Update changeset --------- Co-authored-by: Grigas <[email protected]>
iTwin · Oct 11, 2024 · 99a76ec · 99a76ec
1 parent ba2ae5e
commit 99a76ec
Show file tree

Hide file tree

Showing 15 changed files with 625 additions and 92 deletions.
diff --git a/.changeset/tall-comics-compare.md b/.changeset/tall-comics-compare.md
@@ -0,0 +1,17 @@
+---
+"@itwin/presentation-hierarchies": minor
+---
+
+Increased the speed of hierarchy filtering with large number of filtered paths.
+
+| Amount of paths | Before the change | After the change |
+| --- | ---------- | --------- |
+| 500 | 960.18 ms  | 233.65 ms |
+| 1k  | 2.29 s     | 336.81 ms |
+| 10k | 232.55 s   | 2.17 s    |
+| 50k | not tested | 13.45 s   |
+
+In addition, changed `NodeParser` (return type of `HierarchyDefinition.parseNode`):
+
+- It now can return a promise, so instead of just `SourceInstanceHierarchyNode` it can now also return `Promise<SourceInstanceHierarchyNode>`.
+- Additionally, it now accepts an optional `parentNode` argument of `HierarchyDefinitionParentNode` type.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -140,6 +140,7 @@
       "args": [
         "--config",
         "./.mocharc.json",
+        "./lib/**/*.test.js",
         "--no-timeouts",
         "--parallel=false"
       ],

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -27,7 +27,7 @@
     "source.fixAll": "explicit"
   },
 
-  "eslint.experimental.useFlatConfig": true,
+  "eslint.useFlatConfig": true,
   "eslint.workingDirectories": [
     {
       "mode": "auto"

diff --git a/apps/full-stack-tests/src/hierarchies/HierarchyFiltering.test.ts b/apps/full-stack-tests/src/hierarchies/HierarchyFiltering.test.ts
@@ -223,6 +223,97 @@ describe("Hierarchies", () => {
         });
       });
     });
+    describe("instance nodes", () => {
+      it("filters through instance nodes that are in multiple paths", async function () {
+        const { imodel, ...keys } = await buildIModel(this, async (builder) => {
+          const rootSubject = { className: subjectClassName, id: IModel.rootSubjectId };
+          const childSubject1 = insertSubject({ builder, codeValue: "test subject 1", parentId: rootSubject.id });
+          const childSubject2 = insertSubject({ builder, codeValue: "test subject 2", parentId: rootSubject.id });
+          const childSubject3 = insertSubject({ builder, codeValue: "test subject 3", parentId: rootSubject.id });
+          const childSubject4 = insertSubject({ builder, codeValue: "test subject 4", parentId: rootSubject.id });
+          return { rootSubject, childSubject1, childSubject2, childSubject3, childSubject4 };
+        });
+        const imodelAccess = createIModelAccess(imodel);
+        const selectQueryFactory = createNodesQueryClauseFactory({
+          imodelAccess,
+          instanceLabelSelectClauseFactory: createBisInstanceLabelSelectClauseFactory({ classHierarchyInspector: imodelAccess }),
+        });
+        const createHierarchyLevelDefinition = async (whereClause: (alias: string) => string) => {
+          return [
+            {
+              fullClassName: subjectClassName,
+              query: {
+                ecsql: `
+                  SELECT ${await selectQueryFactory.createSelectClause({
+                    ecClassId: { selector: `this.ECClassId` },
+                    ecInstanceId: { selector: `this.ECInstanceId` },
+                    nodeLabel: { selector: `this.CodeValue` },
+                  })}
+                  FROM ${subjectClassName} AS this
+                  ${whereClause("this")}
+                `,
+              },
+            },
+          ];
+        };
+
+        const hierarchy: HierarchyDefinition = {
+          async defineHierarchyLevel({ parentNode }) {
+            if (!parentNode) {
+              return createHierarchyLevelDefinition((alias: string) => `WHERE ${alias}.ECInstanceId IN (${keys.childSubject1.id}, ${keys.childSubject4.id})`);
+            }
+            if (HierarchyNode.isInstancesNode(parentNode) && parentNode.label === "test subject 1" && parentNode.parentKeys.length === 0) {
+              return createHierarchyLevelDefinition((alias: string) => `WHERE ${alias}.ECInstanceId IN (${keys.childSubject2.id}, ${keys.childSubject3.id})`);
+            }
+            if (HierarchyNode.isInstancesNode(parentNode) && parentNode.label === "test subject 4") {
+              return createHierarchyLevelDefinition((alias: string) => `WHERE ${alias}.ECInstanceId = ${keys.childSubject1.id}`);
+            }
+            if (HierarchyNode.isInstancesNode(parentNode) && parentNode.label === "test subject 1" && parentNode.parentKeys.length === 1) {
+              return createHierarchyLevelDefinition((alias: string) => `WHERE ${alias}.ECInstanceId = ${keys.childSubject2.id}`);
+            }
+            return [];
+          },
+        };
+
+        await validateHierarchy({
+          provider: createProvider({
+            imodel,
+            hierarchy,
+            filteredNodePaths: [
+              { path: [keys.childSubject1, keys.childSubject3], options: { autoExpand: true } },
+              { path: [keys.childSubject4, keys.childSubject1, keys.childSubject2], options: { autoExpand: true } },
+            ],
+          }),
+          expect: [
+            NodeValidators.createForInstanceNode({
+              instanceKeys: [keys.childSubject1],
+              children: [
+                NodeValidators.createForInstanceNode({
+                  instanceKeys: [keys.childSubject3],
+                  isFilterTarget: true,
+                  children: false,
+                }),
+              ],
+            }),
+            NodeValidators.createForInstanceNode({
+              instanceKeys: [keys.childSubject4],
+              children: [
+                NodeValidators.createForInstanceNode({
+                  instanceKeys: [keys.childSubject1],
+                  children: [
+                    NodeValidators.createForInstanceNode({
+                      instanceKeys: [keys.childSubject2],
+                      isFilterTarget: true,
+                      children: false,
+                    }),
+                  ],
+                }),
+              ],
+            }),
+          ],
+        });
+      });
+    });
 
     describe("when filtering through hidden nodes", () => {
       it("filters through hidden generic nodes", async function () {

diff --git a/apps/performance-tests/package.json b/apps/performance-tests/package.json
@@ -6,6 +6,7 @@
     "benchmark:hierarchies": "npm run test:hierarchies -- -O BENCHMARK_OUTPUT_PATH=./hierarchies-benchmark.json",
     "benchmark:unified-selection": "npm run test:unified-selection -- -O BENCHMARK_OUTPUT_PATH=./unified-selection-benchmark.json",
     "build": "tsc",
+    "build:watch": "tsc -w",
     "clean": "rimraf lib temp",
     "docs": "betools extract --fileExt=ts --extractFrom=./src --recursive --out=./build/docs/extract",
     "lint": "eslint \"./src/**/*.ts\"",

diff --git a/apps/performance-tests/src/hierarchies/Filtering.test.ts b/apps/performance-tests/src/hierarchies/Filtering.test.ts
@@ -0,0 +1,121 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+
+import { expect } from "chai";
+import { IModelDb, PhysicalElement, SnapshotDb } from "@itwin/core-backend";
+import { Id64 } from "@itwin/core-bentley";
+import { createNodesQueryClauseFactory, HierarchyFilteringPath, HierarchyNode } from "@itwin/presentation-hierarchies";
+import { createBisInstanceLabelSelectClauseFactory, ECClassHierarchyInspector, ECSchemaProvider } from "@itwin/presentation-shared";
+import { Datasets } from "../util/Datasets";
+import { run } from "../util/TestUtilities";
+import { ProviderOptions, StatelessHierarchyProvider } from "./StatelessHierarchyProvider";
+
+describe("filtering", () => {
+  const totalNumberOfFilteringPaths = 50000;
+  const physicalElementsSmallestDecimalId = 20;
+
+  run({
+    testName: `filters with ${totalNumberOfFilteringPaths} paths`,
+    setup: (): ProviderOptions => {
+      const { schemaName, itemsPerGroup, defaultClassName } = Datasets.CUSTOM_SCHEMA;
+
+      const filtering = {
+        paths: new Array<HierarchyFilteringPath>(),
+      };
+      const parentIdsArr = new Array<number>();
+      for (let i = 1; i <= 100; ++i) {
+        parentIdsArr.push(i + physicalElementsSmallestDecimalId);
+        for (let j = (i - 1) * 500; j < i * 500; ++j) {
+          filtering.paths.push([
+            { className: `${schemaName}.${defaultClassName}_0`, id: `0x${physicalElementsSmallestDecimalId.toString(16)}` },
+            {
+              className: `${schemaName}.${defaultClassName}_${Math.floor(i / itemsPerGroup)}`,
+              id: `0x${(i + physicalElementsSmallestDecimalId).toString(16)}`,
+            },
+            {
+              className: `${schemaName}.${defaultClassName}_${Math.floor(j / itemsPerGroup)}`,
+              id: `0x${(j + physicalElementsSmallestDecimalId).toString(16)}`,
+            },
+          ]);
+        }
+      }
+
+      const iModel = SnapshotDb.openFile(Datasets.getIModelPath("50k flat elements"));
+      const fullClassName = PhysicalElement.classFullName.replace(":", ".");
+      const createHierarchyLevelDefinition = async (imodelAccess: ECSchemaProvider & ECClassHierarchyInspector, whereClause: (alias: string) => string) => {
+        const query = createNodesQueryClauseFactory({
+          imodelAccess,
+          instanceLabelSelectClauseFactory: createBisInstanceLabelSelectClauseFactory({ classHierarchyInspector: imodelAccess }),
+        });
+        return [
+          {
+            fullClassName,
+            query: {
+              ecsql: `
+                SELECT ${await query.createSelectClause({
+                  ecClassId: { selector: `this.ECClassId` },
+                  ecInstanceId: { selector: `this.ECInstanceId` },
+                  nodeLabel: { selector: `this.UserLabel` },
+                })}
+                FROM ${fullClassName} AS this
+                ${whereClause("this")}
+              `,
+            },
+          },
+        ];
+      };
+      return {
+        iModel,
+        rowLimit: "unbounded",
+        getHierarchyFactory: (imodelAccess) => ({
+          async defineHierarchyLevel(props) {
+            // A hierarchy with this structure is created:
+            //
+            //        id:21 -> all other BisCore.PhysicalElement
+            //       /  .
+            //  id:20   .
+            //       \  .
+            //        id:120 -> all other BisCore.PhysicalElement
+            //
+            // We need to split the hierarchy in 100 parts, because we are using 50000 paths and there is a limit of 500 filtering paths for a single parent.
+
+            if (!props.parentNode) {
+              return createHierarchyLevelDefinition(imodelAccess, (alias) => `WHERE ${alias}.ECInstanceId = ${physicalElementsSmallestDecimalId}`);
+            }
+            if (
+              props.parentNode &&
+              HierarchyNode.isInstancesNode(props.parentNode) &&
+              props.parentNode.key.instanceKeys.some(({ id }) => Id64.getLocalId(id) === physicalElementsSmallestDecimalId)
+            ) {
+              return createHierarchyLevelDefinition(imodelAccess, (alias) => `WHERE ${alias}.ECInstanceId IN (${parentIdsArr.join(", ")})`);
+            }
+
+            if (
+              props.parentNode &&
+              HierarchyNode.isInstancesNode(props.parentNode) &&
+              props.parentNode.key.instanceKeys.some(({ id }) => parentIdsArr.includes(Id64.getLocalId(id)))
+            ) {
+              return createHierarchyLevelDefinition(
+                imodelAccess,
+                (alias) => `WHERE ${alias}.ECInstanceId NOT IN (${physicalElementsSmallestDecimalId}, ${parentIdsArr.join(", ")})`,
+              );
+            }
+
+            return [];
+          },
+        }),
+        filtering,
+      };
+    },
+    cleanup: (props: { iModel: IModelDb }) => {
+      props.iModel.close();
+    },
+    test: async (props) => {
+      const provider = new StatelessHierarchyProvider(props);
+      const nodeCount = await provider.loadHierarchy();
+      expect(nodeCount).to.eq(totalNumberOfFilteringPaths);
+    },
+  });
+});
diff --git a/apps/performance-tests/src/hierarchies/StatelessHierarchyProvider.ts b/apps/performance-tests/src/hierarchies/StatelessHierarchyProvider.ts
@@ -12,6 +12,7 @@ import {
   createIModelHierarchyProvider,
   createLimitingECSqlQueryExecutor,
   HierarchyDefinition,
+  HierarchyFilteringPath,
   HierarchyNode,
   HierarchyProvider,
 } from "@itwin/presentation-hierarchies";
@@ -22,6 +23,9 @@ export interface ProviderOptions {
   iModel: IModelDb;
   rowLimit?: number | "unbounded";
   getHierarchyFactory(imodelAccess: ECSchemaProvider & ECClassHierarchyInspector): HierarchyDefinition;
+  filtering?: {
+    paths: HierarchyFilteringPath[];
+  };
 }
 
 const LOG_CATEGORY = "Presentation.PerformanceTests.StatelessHierarchyProvider";
@@ -88,6 +92,7 @@ export class StatelessHierarchyProvider {
       imodelAccess,
       hierarchyDefinition: this._props.getHierarchyFactory(imodelAccess),
       queryCacheSize: 0,
+      filtering: this._props.filtering,
     });
   }
 }

diff --git a/packages/hierarchies/api/presentation-hierarchies.api.md b/packages/hierarchies/api/presentation-hierarchies.api.md
@@ -540,7 +540,7 @@ export function mergeProviders({ providers }: MergeHierarchyProvidersProps): Hie
 // @beta
 export type NodeParser = (row: {
     [columnName: string]: any;
-}) => SourceInstanceHierarchyNode;
+}, parentNode?: HierarchyDefinitionParentNode) => SourceInstanceHierarchyNode | Promise<SourceInstanceHierarchyNode>;
 
 // @beta
 export type NodePostProcessor = (node: ProcessedHierarchyNode) => Promise<ProcessedHierarchyNode>;