Skip to content

Commit

Permalink
Add performance tests (#461)
Browse files Browse the repository at this point in the history
* Initial refactor

* Add arguments aliases and a debugger option

* Fix merge issue

* remove extra parameters

* refactor

* Move changes to a new project

* Setup test reporter

* Fix performance tests

* Remove unnecessary stuff

* Set up global benchmark script

* Update readme

* Adjust pipeline

* Fix output path

* Remove test results file

* Fix formatting issue

* Fix eslint config

* Save main benchmark results only when run on master branch

* Fix condition

* Fix eslint problems

* Remove start script

* Change file structure

* Use public query executor

* Refactor blocking stats

* Add table dependency

* Remove logging from hierarchy provider

* Improve test reporter console output

* Remove check leaks option

* Remove pid from profile name

* Remove pre-caching of iModels

* Simplify float rounding

* Fix p95

* Add additional entries for benchmark

* Fix import being on top of header

* Fix debugger crashing on startup

* Fix issue of TestReporter measuring time from `beforeEach` to test end.

* Add undefined check

* Improve comment

* Rename itMeasures to run

* Improve text for blocking benchmark entries

---------

Co-authored-by: Dmitrij Kuzmiciov <[email protected]>
  • Loading branch information
Yato333 and Yato333 authored Mar 12, 2024
1 parent 1794a83 commit 0ab2c5c
Show file tree
Hide file tree
Showing 20 changed files with 1,215 additions and 799 deletions.
7 changes: 3 additions & 4 deletions .github/workflows/benchmark-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@ on:
- master
- stable
paths:
- apps/load-tests/**
- apps/performance-tests/**
- packages/core-interop/**
- packages/hierarchy-builder/**
- packages/models-tree/**
- pnpm-lock.yaml
- scripts/runBenchmarkTests.js
- .github/workflows/benchmark-pr.yaml
types: [opened, synchronize, reopened, ready_for_review]

Expand Down Expand Up @@ -39,7 +38,7 @@ jobs:
run: pnpm install

- name: Build benchmark tests
run: pnpm lage build --to @load-tests/backend @load-tests/frontend
run: pnpm lage build --to presentation-performance-tests

- name: Run benchmark tests
run: pnpm benchmark
Expand All @@ -48,7 +47,7 @@ jobs:
uses: jalextowle/github-action-benchmark@35e54a2ea34188ca73481c2a0ce5919907978f4f
with:
tool: 'customSmallerIsBetter'
output-file-path: ./apps/load-tests/tests/benchmark.json
output-file-path: ./apps/performance-tests/benchmark.json
github-token: ${{ secrets.GITHUB_TOKEN }}
save-data-file: false
comment-on-pull-request: true
7 changes: 4 additions & 3 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ on:
branches:
- master
paths:
- apps/load-tests/**
- apps/performance-tests/**
- packages/core-interop/**
- packages/hierarchy-builder/**
- packages/models-tree/**
Expand Down Expand Up @@ -39,16 +39,17 @@ jobs:
run: pnpm install

- name: Build benchmark tests
run: pnpm lage build --to @load-tests/backend @load-tests/frontend
run: pnpm lage build --to presentation-performance-tests

- name: Run benchmark tests
run: pnpm benchmark

- name: Store benchmark result
uses: benchmark-action/github-action-benchmark@v1
if: ${{ github.ref == 'refs/heads/master' }}
with:
tool: 'customSmallerIsBetter'
output-file-path: ./apps/load-tests/tests/benchmark.json
output-file-path: ./apps/performance-tests/benchmark.json
# Access token to deploy GitHub Pages branch
github-token: ${{ secrets.IMJS_ADMIN_GH_TOKEN }}
# Push and deploy GitHub pages branch automatically
Expand Down
11 changes: 11 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,17 @@
"NODE_ENV": "development"
}
},
{
"name": "Tests: Performance",
"cwd": "${workspaceFolder}/apps/performance-tests",
"type": "node",
"request": "launch",
"runtimeExecutable": "npx",
"runtimeArgs": ["mocha", "--config", "./.mocharc.json", "./lib/*test.js"],
"env": {
"NODE_OPTIONS": "--enable-source-maps"
}
},
{
"name": "Tests: Hierarchy builder",
"cwd": "${workspaceFolder}/packages/hierarchy-builder",
Expand Down
3 changes: 3 additions & 0 deletions apps/performance-tests/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
datasets/*
benchmark.json
test-results.xml
5 changes: 5 additions & 0 deletions apps/performance-tests/.mocharc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"timeout": 60000,
"file": "./lib/main.js",
"reporter": ["./lib/util/TestReporter.js"]
}
25 changes: 25 additions & 0 deletions apps/performance-tests/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Presentation performance tests

## Tests

The tests are supposed to represent various scenarios that we want to profile.

## Test reporter

Additionally, we want to measure how much time the main thread is being blocked.
Also, these tests have a different purpose - to provide a benchmark that will be used by GitHub actions and can be useful for the developers.
The simplest way to accommodate that is to use a custom test reporter (defined in `TestReporter.ts`).
The reporter gathers test durations and information about main thread blocking and saves it to a file if an output path is provided.

Example: `mocha -R ./lib/TestReporter.js -O BENCHMARK_OUTPUT_PATH="./results.json"`

### iModels

The tests may use iModels that are managed in `Datasets.ts` module. The iModels are stored locally in the `./datasets` folder.

## Usage

- In order to run all performance tests type:
`pnpm test`
- In order to run performance tests and save the results to `./benchmark.json` enter:
`pnpm benchmark`.
20 changes: 20 additions & 0 deletions apps/performance-tests/eslint.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Bentley Systems, Incorporated. All rights reserved.
* See LICENSE.md in the project root for license terms and full copyright notice.
*--------------------------------------------------------------------------------------------*/
const iTwinPlugin = require("@itwin/eslint-plugin");
const eslintBaseConfig = require("../../eslint.base.config");

module.exports = [
{
files: ["**/*.ts"],
...iTwinPlugin.configs.iTwinjsRecommendedConfig,
},
...eslintBaseConfig,
{
files: ["**/*.ts"],
rules: {
"no-console": "off",
},
},
];
34 changes: 34 additions & 0 deletions apps/performance-tests/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"name": "presentation-performance-tests",
"version": "0.0.0",
"private": true,
"scripts": {
"test": "NODE_OPTIONS=\"--enable-source-maps\" mocha --config ./.mocharc.json ./lib/*test.js",
"benchmark": "npm run test -- -O BENCHMARK_OUTPUT_PATH=./benchmark.json",
"build": "tsc",
"clean": "rimraf lib temp",
"lint": "eslint \"./src/**/*.ts\""
},
"dependencies": {
"@itwin/core-backend": "^4.4.0",
"@itwin/core-bentley": "^4.4.0",
"@itwin/core-common": "^4.4.0",
"@itwin/ecschema-metadata": "^4.4.0",
"@itwin/presentation-hierarchy-builder": "workspace:*",
"@itwin/presentation-core-interop": "workspace:*",
"@itwin/presentation-models-tree": "workspace:*",
"as-table": "^1.0.55",
"mocha": "^10.3.0",
"blocked": "^1.3.0",
"rxjs": "^7.8.1"
},
"devDependencies": {
"@itwin/eslint-plugin": "4.0.0-dev.48",
"@types/blocked": "^1.3.4",
"@types/mocha": "^10.0.6",
"@types/node": "^18.17.7",
"eslint": "^8.56.0",
"rimraf": "^5.0.5",
"typescript": "~5.0.4"
}
}
42 changes: 42 additions & 0 deletions apps/performance-tests/src/Datasets.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Bentley Systems, Incorporated. All rights reserved.
* See LICENSE.md in the project root for license terms and full copyright notice.
*--------------------------------------------------------------------------------------------*/
import fs from "fs";
import path from "path";

async function downloadDataset(name: string, downloadUrl: string, localPath: string): Promise<void> {
console.log(`Downloading "${name}" iModel from "${downloadUrl}"...`);
const response = await fetch(downloadUrl);
if (!response.ok) {
throw new Error(`Failed to fetch ${name} iModel: ${response.statusText}`);
}

await response.body!.pipeTo(fs.WriteStream.toWeb(fs.createWriteStream(localPath)));
}

/** Paths to downloaded iModels. */
export const iModelPaths = new Array<string>();

/** Loads iModels into cache for the tests to use. */
export async function loadDataSets(datasetsDirPath: string) {
await fs.promises.mkdir(datasetsDirPath, { recursive: true });

const datasets = [["Baytown", "https://github.com/imodeljs/desktop-starter/raw/master/assets/Baytown.bim"]].map((entry) => [
...entry,
path.join(datasetsDirPath, `${entry[0]}.bim`),
]);

const datasetPaths = await Promise.all(
datasets.map(async ([name, url, localPath]) => {
try {
await fs.promises.access(localPath, fs.constants.F_OK);
} catch {
await downloadDataset(name, url, localPath);
}
return path.resolve(localPath);
}),
);

iModelPaths.push(...datasetPaths);
}
84 changes: 84 additions & 0 deletions apps/performance-tests/src/StatelessHierarchyProvider.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Bentley Systems, Incorporated. All rights reserved.
* See LICENSE.md in the project root for license terms and full copyright notice.
*--------------------------------------------------------------------------------------------*/
import { expand, filter, from, mergeAll, of } from "rxjs";
import { IModelDb } from "@itwin/core-backend";
import { ISchemaLocater, Schema, SchemaContext, SchemaInfo, SchemaKey, SchemaMatchType } from "@itwin/ecschema-metadata";
import { createECSqlQueryExecutor, createMetadataProvider } from "@itwin/presentation-core-interop";
import { createLimitingECSqlQueryExecutor, HierarchyNode, HierarchyProvider } from "@itwin/presentation-hierarchy-builder";
import { ModelsTreeDefinition } from "@itwin/presentation-models-tree";

export class StatelessHierarchyProvider {
private readonly _provider: HierarchyProvider;

constructor(
iModelDb: IModelDb,
private readonly _nodeRequestLimit = 10,
) {
this._provider = createProvider(iModelDb);
}

public async loadInitialHierarchy(): Promise<void> {
await this.loadNodes((node) => node.children && !!node.autoExpand);
}

public async loadFullHierarchy(): Promise<void> {
await this.loadNodes((node) => node.children);
}

private async loadNodes(nodeHasChildren: (node: HierarchyNode) => boolean) {
await new Promise<void>((resolve, reject) => {
const nodesObservable = of<HierarchyNode | undefined>(undefined).pipe(
expand((parentNode) => {
return from(this._provider.getNodes({ parentNode })).pipe(
mergeAll(),
filter((node) => nodeHasChildren(node)),
);
}, this._nodeRequestLimit),
);
nodesObservable.subscribe({
complete: resolve,
error: reject,
});
});
}
}

function createProvider(iModelDb: IModelDb) {
const schemas = new SchemaContext();
const locater = new SchedulingSchemaLocater(iModelDb);
schemas.addLocater(locater);
const metadataProvider = createMetadataProvider(schemas);

return new HierarchyProvider({
metadataProvider,
hierarchyDefinition: new ModelsTreeDefinition({ metadataProvider }),
queryExecutor: createLimitingECSqlQueryExecutor(createECSqlQueryExecutor(iModelDb), 1000),
});
}

class SchedulingSchemaLocater implements ISchemaLocater {
constructor(private readonly _iModelDb: IModelDb) {}

public getSchemaSync<T extends Schema>(_schemaKey: Readonly<SchemaKey>, _matchType: SchemaMatchType, _schemaContext: SchemaContext): T | undefined {
console.error(`getSchemaSync not implemented`);
return undefined;
}

public async getSchemaInfo(schemaKey: Readonly<SchemaKey>, matchType: SchemaMatchType, schemaContext: SchemaContext): Promise<SchemaInfo | undefined> {
const schemaJson = this._iModelDb.getSchemaProps(schemaKey.name);
const schemaInfo = await Schema.startLoadingFromJson(schemaJson, schemaContext);
if (schemaInfo !== undefined && schemaInfo.schemaKey.matches(schemaKey, matchType)) {
return schemaInfo;
}
return undefined;
}

public async getSchema<T extends Schema>(schemaKey: Readonly<SchemaKey>, matchType: SchemaMatchType, schemaContext: SchemaContext): Promise<T | undefined> {
await this.getSchemaInfo(schemaKey, matchType, schemaContext);
// eslint-disable-next-line @itwin/no-internal
const schema = await schemaContext.getCachedSchema(schemaKey, matchType);
return schema as T;
}
}
17 changes: 17 additions & 0 deletions apps/performance-tests/src/main.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Bentley Systems, Incorporated. All rights reserved.
* See LICENSE.md in the project root for license terms and full copyright notice.
*--------------------------------------------------------------------------------------------*/
import { IModelHost } from "@itwin/core-backend";
import { loadDataSets } from "./Datasets";

before(async () => {
await IModelHost.startup({
profileName: "presentation-performance-tests",
});
await loadDataSets("./datasets");
});

after(async () => {
await IModelHost.shutdown();
});
30 changes: 30 additions & 0 deletions apps/performance-tests/src/stateless.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Bentley Systems, Incorporated. All rights reserved.
* See LICENSE.md in the project root for license terms and full copyright notice.
*--------------------------------------------------------------------------------------------*/
import { SnapshotDb } from "@itwin/core-backend";
import { iModelPaths } from "./Datasets";
import { StatelessHierarchyProvider } from "./StatelessHierarchyProvider";
import { run } from "./util/TestUtilities";

describe("stateless hierarchy", () => {
let iModel: SnapshotDb;

beforeEach(() => {
iModel = SnapshotDb.openFile(iModelPaths[0]);
});

afterEach(() => {
iModel.close();
});

run("loads initial hierarchy", async () => {
const provider = new StatelessHierarchyProvider(iModel);
await provider.loadInitialHierarchy();
});

run("loads full hierarchy", async () => {
const provider = new StatelessHierarchyProvider(iModel);
await provider.loadFullHierarchy();
});
});
Loading

0 comments on commit 0ab2c5c

Please sign in to comment.