From a214fd9b0d28bbc3719fcf9977298a65339257cc Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 1 Sep 2023 12:09:53 -0400 Subject: [PATCH 01/10] test phase 1 --- .../test/standalone/IModelTransformer.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/transformer/src/test/standalone/IModelTransformer.test.ts b/packages/transformer/src/test/standalone/IModelTransformer.test.ts index ad424d3a..9bb718f7 100644 --- a/packages/transformer/src/test/standalone/IModelTransformer.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformer.test.ts @@ -5,6 +5,7 @@ import { assert, expect } from "chai"; import * as fs from "fs"; +import * as child_process from "child_process"; import * as path from "path"; import * as Semver from "semver"; import * as sinon from "sinon"; @@ -2635,6 +2636,41 @@ describe("IModelTransformer", () => { targetDb.close(); }); + it.only("prunes unnecessary geometry parts", async function () { + const sourceDbFile = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "PruneGeomParts.bim"); + const sourceDb = SnapshotDb.createFrom(await ReusedSnapshots.extensiveTestScenario, sourceDbFile); + + const targetDbFile: string = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "PruneGeomParts-Target.bim"); + const targetDb = SnapshotDb.createEmpty(targetDbFile, { rootSubject: { name: "PruneGeomParts" } }); + targetDb.saveChanges(); + + const transformer = new IModelTransformer(sourceDb, targetDb); + // expect this to not reject, adding chai as promised makes the error less readable + await transformer.processSchemas(); + const targetModelId = PhysicalModel.insert(targetDb, IModel.rootSubjectId, "Physical"); + + const sourceElemId = sourceDb.queryEntityIds({from: "bis.PhysicalElement", limit: 1})[Symbol.iterator]().next().value; + expect(sourceElemId).not.to.be.undefined; + expect(sourceElemId).not.to.equal(Id64.invalid); + const sourceElem = sourceDb.elements.getElement(sourceElemId); + + transformer.context.remapElement(sourceElem.model, targetModelId); + await transformer.processElement(sourceElemId); + + function printSize(p: string) { + // eslint-disable-next-line + console.log(`Size of ${p}\n`, child_process.execSync(`du -h ${p}`, {encoding: "utf-8"})); + } + + printSize(sourceDbFile); + printSize(targetDbFile); + + // clean up + transformer.dispose(); + sourceDb.close(); + targetDb.close(); + }); + /** unskip to generate a javascript CPU profile on just the processAll portion of an iModel */ it.skip("should profile an IModel transformation", async function () { const sourceDbFile = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "ProfileTransformation.bim"); From 8bb9ba89ab7c56125252276161f44b3c80cb0fd8 Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 1 Sep 2023 12:23:55 -0400 Subject: [PATCH 02/10] test 2 --- .../test/standalone/IModelTransformer.test.ts | 68 ++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/packages/transformer/src/test/standalone/IModelTransformer.test.ts b/packages/transformer/src/test/standalone/IModelTransformer.test.ts index 9bb718f7..71f44da8 100644 --- a/packages/transformer/src/test/standalone/IModelTransformer.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformer.test.ts @@ -12,7 +12,7 @@ import * as sinon from "sinon"; import { CategorySelector, DisplayStyle3d, DocumentListModel, Drawing, DrawingCategory, DrawingGraphic, DrawingModel, ECSqlStatement, Element, ElementMultiAspect, ElementOwnsChildElements, ElementOwnsExternalSourceAspects, ElementOwnsMultiAspects, ElementOwnsUniqueAspect, ElementRefersToElements, - ElementUniqueAspect, ExternalSourceAspect, GenericPhysicalMaterial, GeometricElement, IModelDb, IModelElementCloneContext, IModelHost, IModelJsFs, + ElementUniqueAspect, ExternalSourceAspect, GenericPhysicalMaterial, GeometricElement, GeometryPart, IModelDb, IModelElementCloneContext, IModelHost, IModelJsFs, InformationRecordModel, InformationRecordPartition, LinkElement, Model, ModelSelector, OrthographicViewDefinition, PhysicalModel, PhysicalObject, PhysicalPartition, PhysicalType, Relationship, RenderMaterialElement, RepositoryLink, Schema, SnapshotDb, SpatialCategory, StandaloneDb, SubCategory, Subject, Texture, @@ -23,9 +23,9 @@ import * as TestUtils from "../TestUtils"; import { DbResult, Guid, Id64, Id64String, Logger, LogLevel, OpenMode } from "@itwin/core-bentley"; import { AxisAlignedBox3d, BriefcaseIdValue, Code, CodeScopeSpec, CodeSpec, ColorDef, CreateIModelProps, DefinitionElementProps, ElementAspectProps, ElementProps, - ExternalSourceAspectProps, GeometricElement2dProps, ImageSourceFormat, IModel, IModelError, InformationPartitionElementProps, ModelProps, PhysicalElementProps, Placement3d, ProfileOptions, QueryRowFormat, RelatedElement, RelationshipProps, RepositoryLinkProps, + ExternalSourceAspectProps, GeometricElement2dProps, GeometryPartProps, GeometryStreamBuilder, GeometryStreamProps, ImageSourceFormat, IModel, IModelError, InformationPartitionElementProps, ModelProps, PhysicalElementProps, Placement3d, ProfileOptions, QueryRowFormat, RelatedElement, RelationshipProps, RepositoryLinkProps, } from "@itwin/core-common"; -import { Point3d, Range3d, StandardViewIndex, Transform, YawPitchRollAngles } from "@itwin/core-geometry"; +import { Box, Point3d, Range3d, StandardViewIndex, Transform, Vector3d, YawPitchRollAngles } from "@itwin/core-geometry"; import { IModelExporter, IModelExportHandler, IModelTransformer, IModelTransformOptions, TransformerLoggerCategory } from "../../transformer"; import { AspectTrackingImporter, @@ -2636,9 +2636,57 @@ describe("IModelTransformer", () => { targetDb.close(); }); + function createBigGeomPart(): GeometryStreamProps { + const geometryStreamBuilder = new GeometryStreamBuilder(); + for (let i = 1; i < 2_00; ++i) { + geometryStreamBuilder.appendGeometry(Box.createDgnBox( + Point3d.createZero(), Vector3d.unitX(), Vector3d.unitY(), new Point3d(0, 0, i), + i, i, i, i, true, + )!); + } + return geometryStreamBuilder.geometryStream; + } + + function createBoxWithGeomParts(geometryPartId: Id64String): GeometryStreamProps { + const geometryStreamBuilder = new GeometryStreamBuilder(); + geometryStreamBuilder.appendGeometry(Box.createDgnBox( + Point3d.createZero(), Vector3d.unitX(), Vector3d.unitY(), new Point3d(0, 0, 0), + 0, 0, 0, 0, true, + )!); + geometryStreamBuilder.appendGeometryPart3d(geometryPartId); + geometryStreamBuilder.appendGeometryPart3d(geometryPartId); + return geometryStreamBuilder.geometryStream; + } + it.only("prunes unnecessary geometry parts", async function () { const sourceDbFile = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "PruneGeomParts.bim"); - const sourceDb = SnapshotDb.createFrom(await ReusedSnapshots.extensiveTestScenario, sourceDbFile); + const sourceDb = SnapshotDb.createEmpty(sourceDbFile, { rootSubject: { name: "PruneGeomPartsSrc" } }); + + const sourceModelId = PhysicalModel.insert(sourceDb, IModel.rootSubjectId, "Physical"); + const categoryId = SpatialCategory.insert(sourceDb, IModel.dictionaryId, "SpatialCategory", { color: ColorDef.green.toJSON() }); + const [geometryPart1Id, geometryPart2Id] = [1, 2].map((i) => { + const geometryPartProps: GeometryPartProps = { + classFullName: GeometryPart.classFullName, + model: IModelDb.dictionaryId, + code: GeometryPart.createCode(sourceDb, IModelDb.dictionaryId, `GeometryPart${i}`), + geom: createBigGeomPart(), + }; + return sourceDb.elements.insertElement(geometryPartProps); + }); + + const physicalObject1Props: PhysicalElementProps = { + classFullName: PhysicalObject.classFullName, + model: sourceModelId, + category: categoryId, + code: Code.createEmpty(), + userLabel: "PhysicalObject1", + geom: createBoxWithGeomParts(geometryPart1Id), + placement: { + origin: Point3d.create(1, 1, 1), + angles: YawPitchRollAngles.createDegrees(0, 0, 0), + }, + }; + const physicalObject1Id = sourceDb.elements.insertElement(physicalObject1Props); const targetDbFile: string = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "PruneGeomParts-Target.bim"); const targetDb = SnapshotDb.createEmpty(targetDbFile, { rootSubject: { name: "PruneGeomParts" } }); @@ -2649,13 +2697,10 @@ describe("IModelTransformer", () => { await transformer.processSchemas(); const targetModelId = PhysicalModel.insert(targetDb, IModel.rootSubjectId, "Physical"); - const sourceElemId = sourceDb.queryEntityIds({from: "bis.PhysicalElement", limit: 1})[Symbol.iterator]().next().value; - expect(sourceElemId).not.to.be.undefined; - expect(sourceElemId).not.to.equal(Id64.invalid); - const sourceElem = sourceDb.elements.getElement(sourceElemId); + const physicalObject1 = sourceDb.elements.getElement(physicalObject1Id); - transformer.context.remapElement(sourceElem.model, targetModelId); - await transformer.processElement(sourceElemId); + transformer.context.remapElement(physicalObject1.model, targetModelId); + await transformer.processElement(physicalObject1Id); function printSize(p: string) { // eslint-disable-next-line @@ -2665,6 +2710,9 @@ describe("IModelTransformer", () => { printSize(sourceDbFile); printSize(targetDbFile); + expect(count(sourceDb, GeometryPart.classFullName)).to.equal(2); + expect(count(targetDb, GeometryPart.classFullName)).to.equal(1); + // clean up transformer.dispose(); sourceDb.close(); From 5f9c41f66b5cc2722f759cdd3e3344170465cab9 Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 1 Sep 2023 14:14:49 -0400 Subject: [PATCH 03/10] test processAll instead --- .../test/standalone/IModelTransformer.test.ts | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/packages/transformer/src/test/standalone/IModelTransformer.test.ts b/packages/transformer/src/test/standalone/IModelTransformer.test.ts index 71f44da8..46a121b1 100644 --- a/packages/transformer/src/test/standalone/IModelTransformer.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformer.test.ts @@ -2658,35 +2658,40 @@ describe("IModelTransformer", () => { return geometryStreamBuilder.geometryStream; } - it.only("prunes unnecessary geometry parts", async function () { + it.only("processAll prunes unnecessary geometry parts", async function () { const sourceDbFile = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "PruneGeomParts.bim"); const sourceDb = SnapshotDb.createEmpty(sourceDbFile, { rootSubject: { name: "PruneGeomPartsSrc" } }); const sourceModelId = PhysicalModel.insert(sourceDb, IModel.rootSubjectId, "Physical"); const categoryId = SpatialCategory.insert(sourceDb, IModel.dictionaryId, "SpatialCategory", { color: ColorDef.green.toJSON() }); - const [geometryPart1Id, geometryPart2Id] = [1, 2].map((i) => { + + const [physObj1, physObj2] = [1, 2].map((i) => { const geometryPartProps: GeometryPartProps = { classFullName: GeometryPart.classFullName, model: IModelDb.dictionaryId, code: GeometryPart.createCode(sourceDb, IModelDb.dictionaryId, `GeometryPart${i}`), geom: createBigGeomPart(), }; - return sourceDb.elements.insertElement(geometryPartProps); - }); - const physicalObject1Props: PhysicalElementProps = { - classFullName: PhysicalObject.classFullName, - model: sourceModelId, - category: categoryId, - code: Code.createEmpty(), - userLabel: "PhysicalObject1", - geom: createBoxWithGeomParts(geometryPart1Id), - placement: { - origin: Point3d.create(1, 1, 1), - angles: YawPitchRollAngles.createDegrees(0, 0, 0), - }, - }; - const physicalObject1Id = sourceDb.elements.insertElement(physicalObject1Props); + const geomPartId = sourceDb.elements.insertElement(geometryPartProps); + + const physObjProps: PhysicalElementProps = { + classFullName: PhysicalObject.classFullName, + model: sourceModelId, + category: categoryId, + code: Code.createEmpty(), + userLabel: "PhysicalObject1", + geom: createBoxWithGeomParts(geomPartId), + placement: { + origin: Point3d.create(1, 1, 1), + angles: YawPitchRollAngles.createDegrees(0, 0, 0), + }, + }; + + const physObjId = sourceDb.elements.insertElement(physObjProps); + + return { geomPartId, id: physObjId }; + }); const targetDbFile: string = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "PruneGeomParts-Target.bim"); const targetDb = SnapshotDb.createEmpty(targetDbFile, { rootSubject: { name: "PruneGeomParts" } }); @@ -2697,10 +2702,11 @@ describe("IModelTransformer", () => { await transformer.processSchemas(); const targetModelId = PhysicalModel.insert(targetDb, IModel.rootSubjectId, "Physical"); - const physicalObject1 = sourceDb.elements.getElement(physicalObject1Id); + const physObj1Elem = sourceDb.elements.getElement(physObj1.id); - transformer.context.remapElement(physicalObject1.model, targetModelId); - await transformer.processElement(physicalObject1Id); + transformer.context.remapElement(physObj1Elem.model, targetModelId); + transformer.shouldExportElement = (elem) => elem.id !== physObj2.id; + await transformer.processAll(); function printSize(p: string) { // eslint-disable-next-line @@ -2710,6 +2716,8 @@ describe("IModelTransformer", () => { printSize(sourceDbFile); printSize(targetDbFile); + assert(Id64.isValidId64(transformer.context.findTargetElementId(physObj1.id))); + assert(!Id64.isValidId64(transformer.context.findTargetElementId(physObj2.id))); expect(count(sourceDb, GeometryPart.classFullName)).to.equal(2); expect(count(targetDb, GeometryPart.classFullName)).to.equal(1); From 1e208e05367c09956d90bd36c637e809800ca137 Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 1 Sep 2023 14:53:20 -0400 Subject: [PATCH 04/10] add primitive geometry cleanup routine --- .../src/CleanupUnusedGeometryParts.ts | 39 +++++++++++++++++++ packages/transformer/src/IModelTransformer.ts | 25 ++++++++---- .../test/standalone/IModelTransformer.test.ts | 2 +- 3 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 packages/transformer/src/CleanupUnusedGeometryParts.ts diff --git a/packages/transformer/src/CleanupUnusedGeometryParts.ts b/packages/transformer/src/CleanupUnusedGeometryParts.ts new file mode 100644 index 00000000..b8711132 --- /dev/null +++ b/packages/transformer/src/CleanupUnusedGeometryParts.ts @@ -0,0 +1,39 @@ +import { IModelDb } from "@itwin/core-backend"; +import { DbResult, Id64String } from "@itwin/core-bentley"; +import { ElementGeometry } from "@itwin/core-common"; + +/** + * delete all geometry parts that are not referenced by any geometric elements. + * This will be replaced by a more integrated approach + * @internal + */ +export function cleanupUnusedGeometryParts(db: IModelDb) { + const usedGeomParts = new Set(); + + const allGeomElemIdsQuery = ` + SELECT ECInstanceId + FROM bis.GeometricElement + `; + db.withPreparedStatement(allGeomElemIdsQuery, (geomElemIdStmt) => { + while (geomElemIdStmt.step() === DbResult.BE_SQLITE_ROW) { + const geomElemId = geomElemIdStmt.getValue(0).getId(); + db.elementGeometryRequest({ + elementId: geomElemId, + skipBReps: true, // breps contain no references to geometry parts + onGeometry(geomInfo) { + for (const entry of new ElementGeometry.Iterator(geomInfo)) { + const maybeGeomPart = entry.toGeometryPart(); + if (maybeGeomPart) + usedGeomParts.add(maybeGeomPart); + } + }, + }); + } + }); + + // NOTE: maybe (negligbly?) faster to do custom query with `NOT InVirtualSet()` + const unusedGeomPartIds = db.queryEntityIds({ from: "bis.GeometryPart" }); + for (const usedGeomPartId of usedGeomParts) + unusedGeomPartIds.delete(usedGeomPartId); + db.elements.deleteDefinitionElements([...unusedGeomPartIds]); +} diff --git a/packages/transformer/src/IModelTransformer.ts b/packages/transformer/src/IModelTransformer.ts index 5cf2fc77..ff3afe3c 100644 --- a/packages/transformer/src/IModelTransformer.ts +++ b/packages/transformer/src/IModelTransformer.ts @@ -31,6 +31,7 @@ import { IModelImporter, IModelImporterState, OptimizeGeometryOptions } from "./ import { TransformerLoggerCategory } from "./TransformerLoggerCategory"; import { PendingReference, PendingReferenceMap } from "./PendingReferenceMap"; import { EntityKey, EntityMap } from "./EntityMap"; +import { cleanupUnusedGeometryParts } from "./CleanupUnusedGeometryParts"; import { IModelCloneContext } from "./IModelCloneContext"; import { EntityUnifier } from "./EntityUnifier"; @@ -148,6 +149,12 @@ export interface IModelTransformOptions { * @beta */ optimizeGeometry?: OptimizeGeometryOptions; + + /** internal option, will definitely be replaced with a more complete API + * remove unused geometry parts during `finalizeTransformation` + * @internal + */ + cleanupUnusedGeometryParts?: boolean; } /** @@ -1129,6 +1136,16 @@ export class IModelTransformer extends IModelExportHandler { partiallyCommittedElem.forceComplete(); } } + + if (this._options.cleanupUnusedGeometryParts) + // FIXME: move to importer + cleanupUnusedGeometryParts(this.targetDb); + + if (this._options.optimizeGeometry) + this.importer.optimizeGeometry(this._options.optimizeGeometry); + + this.importer.computeProjectExtents(); + // this internal is guaranteed stable for just transformer usage /* eslint-disable @itwin/no-internal */ if ("codeValueBehavior" in this.sourceDb as any) { @@ -1450,10 +1467,6 @@ export class IModelTransformer extends IModelExportHandler { await this.detectRelationshipDeletes(); } - if (this._options.optimizeGeometry) - this.importer.optimizeGeometry(this._options.optimizeGeometry); - - this.importer.computeProjectExtents(); this.finalizeTransformation(); } @@ -1685,10 +1698,6 @@ export class IModelTransformer extends IModelExportHandler { await this.exporter.exportChanges(options); await this.processDeferredElements(); // eslint-disable-line deprecation/deprecation - if (this._options.optimizeGeometry) - this.importer.optimizeGeometry(this._options.optimizeGeometry); - - this.importer.computeProjectExtents(); this.finalizeTransformation(); } } diff --git a/packages/transformer/src/test/standalone/IModelTransformer.test.ts b/packages/transformer/src/test/standalone/IModelTransformer.test.ts index 46a121b1..4829be41 100644 --- a/packages/transformer/src/test/standalone/IModelTransformer.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformer.test.ts @@ -2697,7 +2697,7 @@ describe("IModelTransformer", () => { const targetDb = SnapshotDb.createEmpty(targetDbFile, { rootSubject: { name: "PruneGeomParts" } }); targetDb.saveChanges(); - const transformer = new IModelTransformer(sourceDb, targetDb); + const transformer = new IModelTransformer(sourceDb, targetDb, { cleanupUnusedGeometryParts: true }); // expect this to not reject, adding chai as promised makes the error less readable await transformer.processSchemas(); const targetModelId = PhysicalModel.insert(targetDb, IModel.rootSubjectId, "Physical"); From b76b047443388a98b100376ac956ff9176067386 Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 1 Sep 2023 15:08:20 -0400 Subject: [PATCH 05/10] remove it.only --- .../transformer/src/test/standalone/IModelTransformer.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/transformer/src/test/standalone/IModelTransformer.test.ts b/packages/transformer/src/test/standalone/IModelTransformer.test.ts index 4829be41..28c0f889 100644 --- a/packages/transformer/src/test/standalone/IModelTransformer.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformer.test.ts @@ -2658,7 +2658,7 @@ describe("IModelTransformer", () => { return geometryStreamBuilder.geometryStream; } - it.only("processAll prunes unnecessary geometry parts", async function () { + it("processAll prunes unnecessary geometry parts", async function () { const sourceDbFile = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "PruneGeomParts.bim"); const sourceDb = SnapshotDb.createEmpty(sourceDbFile, { rootSubject: { name: "PruneGeomPartsSrc" } }); From 47958046c1a2a35ee5a7449cf5285a50c885df00 Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Fri, 1 Sep 2023 15:13:59 -0400 Subject: [PATCH 06/10] Change files --- ...l-transformer-d56dd0fb-4a09-43f6-9ce5-9bc6a92c7880.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 change/@itwin-imodel-transformer-d56dd0fb-4a09-43f6-9ce5-9bc6a92c7880.json diff --git a/change/@itwin-imodel-transformer-d56dd0fb-4a09-43f6-9ce5-9bc6a92c7880.json b/change/@itwin-imodel-transformer-d56dd0fb-4a09-43f6-9ce5-9bc6a92c7880.json new file mode 100644 index 00000000..e7a2dfb8 --- /dev/null +++ b/change/@itwin-imodel-transformer-d56dd0fb-4a09-43f6-9ce5-9bc6a92c7880.json @@ -0,0 +1,7 @@ +{ + "type": "patch", + "comment": "add primitive geometry cleanup routine", + "packageName": "@itwin/imodel-transformer", + "email": "mike.belousov@bentley.com", + "dependentChangeType": "patch" +} From a3026edb819a9741294a364333dfa67badf0e5c3 Mon Sep 17 00:00:00 2001 From: imodeljs-admin Date: Fri, 1 Sep 2023 19:15:40 +0000 Subject: [PATCH 07/10] Version bump [skip actions] --- packages/transformer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/transformer/package.json b/packages/transformer/package.json index 392d8667..2c8ef266 100644 --- a/packages/transformer/package.json +++ b/packages/transformer/package.json @@ -1,6 +1,6 @@ { "name": "@itwin/imodel-transformer", - "version": "0.3.2", + "version": "0.3.3-prune-geom-parts.0", "description": "API for exporting an iModel's parts and also importing them into another iModel", "main": "lib/cjs/transformer.js", "typings": "lib/cjs/transformer", From 03b792a07bd3140f36b3d98e27ce12b30bff2c4c Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Thu, 7 Sep 2023 13:31:15 -0400 Subject: [PATCH 08/10] unlimited element id query --- .../src/CleanupUnusedGeometryParts.ts | 32 ++++++++++++++++--- .../test/standalone/IModelTransformer.test.ts | 31 +++++++++++------- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/packages/transformer/src/CleanupUnusedGeometryParts.ts b/packages/transformer/src/CleanupUnusedGeometryParts.ts index b8711132..72fb1464 100644 --- a/packages/transformer/src/CleanupUnusedGeometryParts.ts +++ b/packages/transformer/src/CleanupUnusedGeometryParts.ts @@ -8,12 +8,23 @@ import { ElementGeometry } from "@itwin/core-common"; * @internal */ export function cleanupUnusedGeometryParts(db: IModelDb) { + const unusedGeomPartIds = queryUnusedGeomParts(db); + + db.elements.deleteDefinitionElements([...unusedGeomPartIds]); +} + +/** + * queryEntityIds maxes out at 10K, we may need everything during this temporary solution + * since geometry parts may reference each other + */ +function queryUnusedGeomParts(db: IModelDb) { const usedGeomParts = new Set(); const allGeomElemIdsQuery = ` SELECT ECInstanceId FROM bis.GeometricElement `; + db.withPreparedStatement(allGeomElemIdsQuery, (geomElemIdStmt) => { while (geomElemIdStmt.step() === DbResult.BE_SQLITE_ROW) { const geomElemId = geomElemIdStmt.getValue(0).getId(); @@ -31,9 +42,20 @@ export function cleanupUnusedGeometryParts(db: IModelDb) { } }); - // NOTE: maybe (negligbly?) faster to do custom query with `NOT InVirtualSet()` - const unusedGeomPartIds = db.queryEntityIds({ from: "bis.GeometryPart" }); - for (const usedGeomPartId of usedGeomParts) - unusedGeomPartIds.delete(usedGeomPartId); - db.elements.deleteDefinitionElements([...unusedGeomPartIds]); + const unusedGeomPartIds = db.withPreparedStatement(` + SELECT ECInstanceId + FROM bis.GeometryPart + WHERE NOT InVirtualSet(?, ECInstanceId) + `, + (stmt) => { + const ids = new Set(); + stmt.bindIdSet(1, [...usedGeomParts]); + while (stmt.step() === DbResult.BE_SQLITE_ROW) { + const id = stmt.getValue(0).getId(); + ids.add(id); + } + return ids; + }); + + return unusedGeomPartIds; } diff --git a/packages/transformer/src/test/standalone/IModelTransformer.test.ts b/packages/transformer/src/test/standalone/IModelTransformer.test.ts index 28c0f889..c580ba1c 100644 --- a/packages/transformer/src/test/standalone/IModelTransformer.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformer.test.ts @@ -14,7 +14,7 @@ import { ElementMultiAspect, ElementOwnsChildElements, ElementOwnsExternalSourceAspects, ElementOwnsMultiAspects, ElementOwnsUniqueAspect, ElementRefersToElements, ElementUniqueAspect, ExternalSourceAspect, GenericPhysicalMaterial, GeometricElement, GeometryPart, IModelDb, IModelElementCloneContext, IModelHost, IModelJsFs, InformationRecordModel, InformationRecordPartition, LinkElement, Model, ModelSelector, OrthographicViewDefinition, - PhysicalModel, PhysicalObject, PhysicalPartition, PhysicalType, Relationship, RenderMaterialElement, RepositoryLink, Schema, SnapshotDb, SpatialCategory, StandaloneDb, + PhysicalModel, PhysicalObject, PhysicalPartition, PhysicalType, Relationship, RenderMaterialElement, RepositoryLink, SQLiteDb, Schema, SnapshotDb, SpatialCategory, StandaloneDb, SubCategory, Subject, Texture, } from "@itwin/core-backend"; import * as coreBackendPkgJson from "@itwin/core-backend/package.json"; @@ -2636,12 +2636,12 @@ describe("IModelTransformer", () => { targetDb.close(); }); - function createBigGeomPart(): GeometryStreamProps { + function createBigGeomPart(offset = 0): GeometryStreamProps { const geometryStreamBuilder = new GeometryStreamBuilder(); - for (let i = 1; i < 2_00; ++i) { + for (let i = 1; i < 100_000; ++i) { geometryStreamBuilder.appendGeometry(Box.createDgnBox( - Point3d.createZero(), Vector3d.unitX(), Vector3d.unitY(), new Point3d(0, 0, i), - i, i, i, i, true, + Point3d.createZero(), Vector3d.unitX(), Vector3d.unitY(), new Point3d(0, 0, i + offset), + i + offset, i + offset, i + offset, i + offset, true, )!); } return geometryStreamBuilder.geometryStream; @@ -2670,7 +2670,7 @@ describe("IModelTransformer", () => { classFullName: GeometryPart.classFullName, model: IModelDb.dictionaryId, code: GeometryPart.createCode(sourceDb, IModelDb.dictionaryId, `GeometryPart${i}`), - geom: createBigGeomPart(), + geom: createBigGeomPart(i), }; const geomPartId = sourceDb.elements.insertElement(geometryPartProps); @@ -2693,8 +2693,10 @@ describe("IModelTransformer", () => { return { geomPartId, id: physObjId }; }); + sourceDb.saveChanges(); + const targetDbFile: string = IModelTransformerTestUtils.prepareOutputFile("IModelTransformer", "PruneGeomParts-Target.bim"); - const targetDb = SnapshotDb.createEmpty(targetDbFile, { rootSubject: { name: "PruneGeomParts" } }); + const targetDb = StandaloneDb.createEmpty(targetDbFile, { rootSubject: { name: "PruneGeomParts" } }); targetDb.saveChanges(); const transformer = new IModelTransformer(sourceDb, targetDb, { cleanupUnusedGeometryParts: true }); @@ -2708,6 +2710,16 @@ describe("IModelTransformer", () => { transformer.shouldExportElement = (elem) => elem.id !== physObj2.id; await transformer.processAll(); + targetDb.saveChanges(); + + assert(Id64.isValidId64(transformer.context.findTargetElementId(physObj1.id))); + assert(!Id64.isValidId64(transformer.context.findTargetElementId(physObj2.id))); + expect(count(sourceDb, GeometryPart.classFullName)).to.equal(2); + expect(count(targetDb, GeometryPart.classFullName)).to.equal(1); + + targetDb.nativeDb.vacuum(); + targetDb.saveChanges(); + function printSize(p: string) { // eslint-disable-next-line console.log(`Size of ${p}\n`, child_process.execSync(`du -h ${p}`, {encoding: "utf-8"})); @@ -2716,11 +2728,6 @@ describe("IModelTransformer", () => { printSize(sourceDbFile); printSize(targetDbFile); - assert(Id64.isValidId64(transformer.context.findTargetElementId(physObj1.id))); - assert(!Id64.isValidId64(transformer.context.findTargetElementId(physObj2.id))); - expect(count(sourceDb, GeometryPart.classFullName)).to.equal(2); - expect(count(targetDb, GeometryPart.classFullName)).to.equal(1); - // clean up transformer.dispose(); sourceDb.close(); From f8bbc7cd5f38e7f04c6ca79e1cac0e06956089be Mon Sep 17 00:00:00 2001 From: Michael Belousov Date: Thu, 7 Sep 2023 13:33:40 -0400 Subject: [PATCH 09/10] add cleanup option to test-app --- packages/test-app/src/Main.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/test-app/src/Main.ts b/packages/test-app/src/Main.ts index 20c78679..9bb8337d 100644 --- a/packages/test-app/src/Main.ts +++ b/packages/test-app/src/Main.ts @@ -133,6 +133,7 @@ void (async () => { type: "boolean", default: false, }, + combinePhysicalModels: { desc: "Combine all source PhysicalModels into a single PhysicalModel in the target iModel", type: "boolean", @@ -188,6 +189,11 @@ void (async () => { default: "reject" as const, choices: ["reject", "ignore"] as const, }, + cleanupUnusedGeometryParts: { + desc: "cleans up unused geometry parts after the transformation is finished", + type: "boolean", + default: false, + }, }) .parseSync(); From b4878d3e007608bfc31abc812e0650d13bd514db Mon Sep 17 00:00:00 2001 From: imodeljs-admin Date: Thu, 7 Sep 2023 17:37:24 +0000 Subject: [PATCH 10/10] Version bump [skip actions] --- packages/transformer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/transformer/package.json b/packages/transformer/package.json index 2c8ef266..bf9d8c92 100644 --- a/packages/transformer/package.json +++ b/packages/transformer/package.json @@ -1,6 +1,6 @@ { "name": "@itwin/imodel-transformer", - "version": "0.3.3-prune-geom-parts.0", + "version": "0.3.3-prune-geom-parts.1", "description": "API for exporting an iModel's parts and also importing them into another iModel", "main": "lib/cjs/transformer.js", "typings": "lib/cjs/transformer",