diff --git a/change/@itwin-imodel-transformer-e69e7651-40c7-4476-9e9e-6026f430c36c.json b/change/@itwin-imodel-transformer-e69e7651-40c7-4476-9e9e-6026f430c36c.json new file mode 100644 index 00000000..4a82ea95 --- /dev/null +++ b/change/@itwin-imodel-transformer-e69e7651-40c7-4476-9e9e-6026f430c36c.json @@ -0,0 +1,7 @@ +{ + "type": "prerelease", + "comment": "Add APIs 'addCustomChange' and 'addCustomRelationshipChange' to class 'ChangedInstanceIds' to support providing custom changes ( not found in a changeset ) to the transformer", + "packageName": "@itwin/imodel-transformer", + "email": "22119573+nick4598@users.noreply.github.com", + "dependentChangeType": "patch" +} diff --git a/common/api/imodel-transformer.api.md b/common/api/imodel-transformer.api.md index 5bf12471..540c21dc 100644 --- a/common/api/imodel-transformer.api.md +++ b/common/api/imodel-transformer.api.md @@ -22,6 +22,7 @@ import { EntityReference } from '@itwin/core-common'; import { ExternalSourceAspect } from '@itwin/core-backend'; import { ExternalSourceAspectProps } from '@itwin/core-common'; import { FontProps } from '@itwin/core-common'; +import { Id64Arg } from '@itwin/core-bentley'; import { Id64Array } from '@itwin/core-bentley'; import { Id64Set } from '@itwin/core-bentley'; import { Id64String } from '@itwin/core-bentley'; @@ -36,11 +37,34 @@ import { Relationship } from '@itwin/core-backend'; import { RelationshipProps } from '@itwin/core-backend'; import { Schema } from '@itwin/ecschema-metadata'; import { SchemaKey } from '@itwin/ecschema-metadata'; +import { SqliteChangeOp } from '@itwin/core-backend'; + +// @beta +export interface ChangedInstanceCustomRelationshipData { + // (undocumented) + classFullName: string; + // (undocumented) + ecClassId: Id64String; + // (undocumented) + sourceIdOfRelationship: Id64String; + // (undocumented) + targetIdOfRelationship: Id64String; +} // @public export class ChangedInstanceIds { constructor(db: IModelDb); addChange(change: ChangedECInstance): Promise; + // @beta + addCustomAspectChange(changeType: SqliteChangeOp, ids: Id64Arg): void; + // @beta + addCustomCodeSpecChange(changeType: SqliteChangeOp, ids: Id64Arg): void; + // @beta + addCustomElementChange(changeType: SqliteChangeOp, ids: Id64Arg): void; + // @beta + addCustomModelChange(changeType: SqliteChangeOp, ids: Id64Arg): void; + // @beta + addCustomRelationshipChange(ecClassId: string, changeType: SqliteChangeOp, id: Id64String, sourceECInstanceId: Id64String, targetECInstanceId: Id64String): Promise; // (undocumented) aspect: ChangedInstanceOps; // (undocumented) @@ -49,8 +73,14 @@ export class ChangedInstanceIds { element: ChangedInstanceOps; // (undocumented) font: ChangedInstanceOps; + // @beta + getCustomRelationshipDataFromId(id: Id64String): ChangedInstanceCustomRelationshipData | undefined; + // (undocumented) + get hasCustomChanges(): boolean; static initialize(opts: ChangedInstanceIdsInitOptions): Promise; // (undocumented) + get isEmpty(): boolean; + // (undocumented) model: ChangedInstanceOps; // (undocumented) relationship: ChangedInstanceOps; @@ -69,6 +99,8 @@ export class ChangedInstanceOps { // (undocumented) insertIds: Set; // (undocumented) + get isEmpty(): boolean; + // (undocumented) updateIds: Set; } @@ -124,6 +156,7 @@ export function hasEntityChanged(entity: Entity, entityProps: EntityProps, names // @beta export class IModelExporter { constructor(sourceDb: IModelDb, elementAspectsStrategy?: new (source: IModelDb, handler: ElementAspectsHandler) => ExportElementAspectsStrategy); + addCustomChanges(): void; excludeCodeSpec(codeSpecName: string): void; excludeElement(elementId: Id64String): void; excludeElementAspectClass(classFullName: string): void; diff --git a/common/api/summary/imodel-transformer.exports.csv b/common/api/summary/imodel-transformer.exports.csv index d8cb54ca..88a439e6 100644 --- a/common/api/summary/imodel-transformer.exports.csv +++ b/common/api/summary/imodel-transformer.exports.csv @@ -1,5 +1,6 @@ sep=; Release Tag;API Item +beta;ChangedInstanceCustomRelationshipData public;ChangedInstanceIds public;ChangedInstanceIdsInitOptions = ExportChangesOptions & public;ChangedInstanceOps diff --git a/package.json b/package.json index 10908a4b..ca74d5a2 100644 --- a/package.json +++ b/package.json @@ -46,5 +46,5 @@ "overrides": { "semver": "^7.5.2" } -} + } } diff --git a/packages/test-app/src/IModelHubUtils.ts b/packages/test-app/src/IModelHubUtils.ts index f7706722..ab6da6db 100644 --- a/packages/test-app/src/IModelHubUtils.ts +++ b/packages/test-app/src/IModelHubUtils.ts @@ -97,12 +97,14 @@ export namespace IModelHubUtils { ): Promise { return ( // eslint-disable-next-line @itwin/no-internal - await IModelHost.hubAccess.queryChangeset({ - accessToken, - iModelId, - changeset: { index: changesetIndex }, - }) - ).id; + ( + await IModelHost.hubAccess.queryChangeset({ + accessToken, + iModelId, + changeset: { index: changesetIndex }, + }) + ).id + ); } /** Temporarily needed to convert from the legacy ChangesetId to the now preferred ChangeSetIndex. @@ -201,8 +203,8 @@ export namespace IModelHubUtils { return BriefcaseDb.open({ fileName: briefcaseProps.fileName, readonly: briefcaseArg.briefcaseId - // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison - ? briefcaseArg.briefcaseId === BriefcaseIdValue.Unassigned + ? // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison + briefcaseArg.briefcaseId === BriefcaseIdValue.Unassigned : false, }); } diff --git a/packages/transformer/src/IModelExporter.ts b/packages/transformer/src/IModelExporter.ts index 5bda0d8f..0ed0799b 100644 --- a/packages/transformer/src/IModelExporter.ts +++ b/packages/transformer/src/IModelExporter.ts @@ -19,6 +19,7 @@ import { ElementMultiAspect, ElementRefersToElements, ElementUniqueAspect, + EntityReferences, GeometricElement, IModelDb, IModelHost, @@ -33,6 +34,8 @@ import { import { assert, DbResult, + Id64, + Id64Arg, Id64String, IModelStatus, Logger, @@ -41,6 +44,8 @@ import { import { ChangesetFileProps, CodeSpec, + ConcreteEntityTypes, + EntityReference, FontProps, IModel, IModelError, @@ -289,9 +294,10 @@ export class IModelExporter { private _progressCounter: number = 0; /** Optionally cached entity change information */ private _sourceDbChanges?: ChangedInstanceIds; + /** * Retrieve the cached entity change information. - * @note This will only be initialized after [IModelExporter.exportChanges] is invoked. + * @note This will only be initialized after [IModelExporter.exportChanges] is invoked or [IModelExporter.initialize] is called. */ public get sourceDbChanges(): ChangedInstanceIds | undefined { return this._sourceDbChanges; @@ -425,6 +431,7 @@ export class IModelExporter { * range and open the source iModel as of the end (inclusive) of the desired range. * @note the changedInstanceIds are just for this call to exportChanges, so you must continue to pass it in * for consecutive calls + * @note Passing {} or undefined to exportChanges will result in the current changeset of the source iModel being exported. */ public async exportChanges(args?: ExportChangesOptions): Promise { if (!this.sourceDb.isBriefcaseDb()) @@ -438,14 +445,21 @@ export class IModelExporter { return; } - const startChangeset = - args && "startChangeset" in args ? args.startChangeset : undefined; + const isEmptyObject = (obj: object): boolean => + Object.keys(obj).length === 0; - const initOpts: ExporterInitOptions = { - startChangeset: { id: startChangeset?.id }, - }; + let initOpts: ExporterInitOptions; + if (args === undefined || isEmptyObject(args)) { + // Fallback behavior for exportChanges with no args / empty object, this.initialize will process the current changeset of the source iModel being exported when startChangeset.id is undefined. + initOpts = { + startChangeset: { id: undefined }, + }; + } else { + initOpts = args; + } await this.initialize(initOpts); + // _sourceDbChanges are initialized in this.initialize nodeAssert( this._sourceDbChanges !== undefined, @@ -580,7 +594,7 @@ export class IModelExporter { public async exportCodeSpecByName(codeSpecName: string): Promise { const codeSpec: CodeSpec = this.sourceDb.codeSpecs.getByName(codeSpecName); let isUpdate: boolean | undefined; - if (undefined !== this._sourceDbChanges) { + if (this._sourceDbChanges !== undefined) { // is changeset information available? if (this._sourceDbChanges.codeSpec.insertIds.has(codeSpec.id)) { isUpdate = false; @@ -680,7 +694,7 @@ export class IModelExporter { /** Export the model (the container only) from the source iModel. */ private async exportModelContainer(model: Model): Promise { let isUpdate: boolean | undefined; - if (undefined !== this._sourceDbChanges) { + if (this._sourceDbChanges !== undefined) { // is changeset information available? if (this._sourceDbChanges.model.insertIds.has(model.id)) { isUpdate = false; @@ -719,7 +733,7 @@ export class IModelExporter { ); return; } - if (undefined !== this._sourceDbChanges) { + if (this._sourceDbChanges !== undefined) { // is changeset information available? if ( !this._sourceDbChanges.model.insertIds.has(modelId) && @@ -950,7 +964,7 @@ export class IModelExporter { return; } let isUpdate: boolean | undefined; - if (undefined !== this._sourceDbChanges) { + if (this._sourceDbChanges !== undefined) { // is changeset information available? if (this._sourceDbChanges.relationship.insertIds.has(relInstanceId)) { isUpdate = false; @@ -1027,6 +1041,26 @@ export class ChangedInstanceOps { val.delete.forEach((id: Id64String) => this.deleteIds.add(id)); } } + + public get isEmpty(): boolean { + return ( + 0 === this.insertIds.size && + 0 === this.updateIds.size && + 0 === this.deleteIds.size + ); + } +} + +/** + * Interface to describe a 'custom' change. A custom change is one which isn't found by reading changesets, but instead added by a user calling the 'addCustomChange' API on the ChangedInstanceIds instance. + * The purpose a custom change would serve is to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @beta + */ +export interface ChangedInstanceCustomRelationshipData { + sourceIdOfRelationship: Id64String; + targetIdOfRelationship: Id64String; + ecClassId: Id64String; + classFullName: string; } /** @@ -1045,9 +1079,23 @@ export class ChangedInstanceIds { private _elementSubclassIds?: Set; private _aspectSubclassIds?: Set; private _relationshipSubclassIds?: Set; + private _relationshipSubclassIdsToSkip?: Set; + private _ecClassIdsToClassFullNames?: Map; + /** c${string} is used to represent codeSpecs since they do not currently have a representation in the EntityReference class. This map holds information passed to the 'addCustom' functions. */ + private _entityReferenceToCustomDataMap: Map< + EntityReference | `c${string}`, + ChangedInstanceCustomRelationshipData + >; + private _hasCustomRelationshipChanges: boolean; + private _db: IModelDb; public constructor(db: IModelDb) { this._db = db; + this._hasCustomRelationshipChanges = false; + this._entityReferenceToCustomDataMap = new Map< + EntityReference, + ChangedInstanceCustomRelationshipData + >(); } private async setupECClassIds(): Promise { @@ -1056,15 +1104,21 @@ export class ChangedInstanceIds { this._elementSubclassIds = new Set(); this._aspectSubclassIds = new Set(); this._relationshipSubclassIds = new Set(); + this._relationshipSubclassIdsToSkip = new Set(); + this._ecClassIdsToClassFullNames = new Map(); const addECClassIdsToSet = async ( setToModify: Set, baseClass: string ) => { for await (const row of this._db.createQueryReader( - `SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (${baseClass})` + `SELECT c.ECInstanceId ECClassId, c.Name className, s.Name schemaName FROM ECDbMeta.ECClassDef c JOIN ECDbMeta.ECSchemaDef s ON s.ECInstanceId = c.Schema.Id WHERE c.ECInstanceId IS (${baseClass})` )) { - setToModify.add(row.ECInstanceId); + setToModify.add(row.ECClassId); + this._ecClassIdsToClassFullNames?.set( + row.ECClassId, + `${row.schemaName}:${row.className}` + ); } }; const promises = [ @@ -1080,6 +1134,10 @@ export class ChangedInstanceIds { this._relationshipSubclassIds, "BisCore.ElementRefersToElements" ), + addECClassIdsToSet( + this._relationshipSubclassIdsToSkip, + "BisCore.ElementDrivesElement" + ), ]; await Promise.all(promises); } @@ -1090,7 +1148,8 @@ export class ChangedInstanceIds { this._modelSubclassIds && this._elementSubclassIds && this._aspectSubclassIds && - this._relationshipSubclassIds + this._relationshipSubclassIds && + this._relationshipSubclassIdsToSkip ); } @@ -1114,6 +1173,21 @@ export class ChangedInstanceIds { return this._elementSubclassIds?.has(ecClassId); } + public get hasCustomRelationshipChanges(): boolean { + return this._hasCustomRelationshipChanges; + } + + public get hasChanges(): boolean { + return ( + !this.codeSpec.isEmpty || + !this.model.isEmpty || + !this.element.isEmpty || + !this.aspect.isEmpty || + !this.relationship.isEmpty || + !this.font.isEmpty + ); + } + /** * Adds the provided [[ChangedECInstance]] to the appropriate set of changes by class type (codeSpec, model, element, aspect, or relationship) maintained by this instance of ChangedInstanceIds. * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' @@ -1132,6 +1206,7 @@ export class ChangedInstanceIds { throw new Error( `ChangeType was undefined for id: ${change.ECInstanceId}.` ); + if (this._relationshipSubclassIdsToSkip?.has(ecClassId)) return; if (this.isRelationship(ecClassId)) this.handleChange(this.relationship, changeType, change.ECInstanceId); @@ -1145,6 +1220,142 @@ export class ChangedInstanceIds { this.handleChange(this.element, changeType, change.ECInstanceId); } + /** + * Adds the provided change to the element changes maintained by this instance of ChangedInstanceIds + * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' + * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type. + * @note element changes will also cause the element's model to be marked as updated in [[ChangedInstanceIds.model]], so that the element does not get skipped by the transformer. + * @note It is the responsibility of the caller to ensure that the provided id is, in fact an element. + * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @beta + */ + public addCustomElementChange( + changeType: SqliteChangeOp, + ids: Id64Arg + ): void { + // if delete unnecessary? + for (const id of Id64.iterable(ids)) { + this.addModelToUpdated(id); + this.handleChange(this.element, changeType, id); + } + } + + /** + * Adds the provided change to the codespec changes maintained by this instance of ChangedInstanceIds + * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' + * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type. + * @note It is the responsibility of the caller to ensure that the provided id is, in fact a codespec. + * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @beta + */ + public addCustomCodeSpecChange( + changeType: SqliteChangeOp, + ids: Id64Arg + ): void { + for (const id of Id64.iterable(ids)) { + this.handleChange(this.codeSpec, changeType, id); + } + } + + /** + * Adds the provided change to the model changes maintained by this instance of ChangedInstanceIds. + * Also adds the model's modeledElement to the element changes. This is to ensure the changes from the model and its modeledElement get exported together. + * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' + * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type. + * @note It is the responsibility of the caller to ensure that the provided id is, in fact a model. + * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @beta + */ + public addCustomModelChange(changeType: SqliteChangeOp, ids: Id64Arg): void { + for (const id of Id64.iterable(ids)) { + this.handleChange(this.model, changeType, id); + // Also add the model's modeledElement to the element changes. The modeledElement and model go hand in hand. + this.handleChange(this.element, changeType, id); + } + } + + /** + * Adds the provided change to the aspect changes maintained by this instance of ChangedInstanceIds + * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' + * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type. + * @note It is the responsibility of the caller to ensure that the provided id is, in fact an aspect. + * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @beta + */ + public addCustomAspectChange(changeType: SqliteChangeOp, ids: Id64Arg): void { + for (const id of Id64.iterable(ids)) { + this.handleChange(this.aspect, changeType, id); + } + } + + /** + * TODO: Think more about permutations of model updated / inserted / deleted. Can you delete a model without deleting its elements? + * What if model delete but custom change si to insert element into target? + * // It is possible and apparently occasionally sensical to delete a model without deleting its underlying element. + // - If only the model is deleted, [[initFromExternalSourceAspects]] will have already remapped the underlying element since it still exists. + // - If both were deleted, [[remapDeletedSourceEntities]] will find and remap the deleted element making this operation valid + * TODO: If the element is a custom delete we probably shouldnt be calling this? + * There is an optimization in [IModelExporter.exportModelContents] which doesn't try to export elements within a model unless the model itself is part of + * the sourceDbChanges. This method is used in addCustomChange to add the model to the updatedIds set so that the custom element changes are exported. + */ + private addModelToUpdated(elementId: Id64String) { + const modelId = this._db.elements.getElement(elementId).model; + this.handleChange(this.model, "Updated", modelId); + } + + /** TODO: Maybe relationships only? maybe not. + * @beta + */ + public getCustomRelationshipDataFromId( + id: Id64String + ): ChangedInstanceCustomRelationshipData | undefined { + return this._entityReferenceToCustomDataMap.get( + EntityReferences.fromEntityType(id, ConcreteEntityTypes.Relationship) + ); + } + + /** + * Adds the provided change to the set of relationship changes maintained by this instance of ChangedInstanceIds. + * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x' + * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type. + * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship. + * @throws if the ecClassId is NOT a relationship classId + * @param ecClassId class id of the custom change + * @param changeType insert, update or delete + * @param id ECInstanceID of the custom change + * @param sourceECInstanceId source ECInstanceId of the relationship + * @param targetECInstanceId target ECInstanceId of the relationship + * @beta + */ + public async addCustomRelationshipChange( + ecClassId: string, + changeType: SqliteChangeOp, + id: Id64String, + sourceECInstanceId: Id64String, + targetECInstanceId: Id64String + ): Promise { + if (!this._ecClassIdsInitialized) await this.setupECClassIds(); + if (this._relationshipSubclassIdsToSkip?.has(ecClassId)) return; + if (!this._relationshipSubclassIds?.has(ecClassId)) + throw new Error( + `Misuse. id: ${id}, ecClassId: ${ecClassId} is not a relationship class. Use 'addCustomChange' instead.` + ); + + this._hasCustomRelationshipChanges = true; + const classFullName = this._ecClassIdsToClassFullNames?.get(ecClassId); + assert(classFullName !== undefined); // setupECClassIds adds an entry to the above map for every single ECClassId. + this._entityReferenceToCustomDataMap.set( + EntityReferences.fromEntityType(id, ConcreteEntityTypes.Relationship), + { + sourceIdOfRelationship: sourceECInstanceId, + targetIdOfRelationship: targetECInstanceId, + ecClassId, + classFullName, + } + ); + this.handleChange(this.relationship, changeType, id); + } + private handleChange( changedInstanceOps: ChangedInstanceOps, changeType: SqliteChangeOp, @@ -1228,12 +1439,6 @@ export class ChangedInstanceIds { if (csFileProps === undefined) return undefined; const changedInstanceIds = new ChangedInstanceIds(opts.iModel); - const relationshipECClassIdsToSkip = new Set(); - for await (const row of opts.iModel.createQueryReader( - "SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (BisCore.ElementDrivesElement)" - )) { - relationshipECClassIdsToSkip.add(row.ECInstanceId); - } for (const csFile of csFileProps) { const csReader = SqliteChangesetReader.openFile({ @@ -1249,11 +1454,6 @@ export class ChangedInstanceIds { const changes: ChangedECInstance[] = [...ecChangeUnifier.instances]; for (const change of changes) { - if ( - change.ECClassId !== undefined && - relationshipECClassIdsToSkip.has(change.ECClassId) - ) - continue; await changedInstanceIds.addChange(change); } csReader.close(); diff --git a/packages/transformer/src/IModelTransformer.ts b/packages/transformer/src/IModelTransformer.ts index 60a8144e..d35f60c3 100644 --- a/packages/transformer/src/IModelTransformer.ts +++ b/packages/transformer/src/IModelTransformer.ts @@ -93,6 +93,7 @@ import { SourceAndTarget, } from "@itwin/core-common"; import { + ChangedInstanceIds, ExportChangesOptions, ExporterInitOptions, ExportSchemaResult, @@ -2776,6 +2777,56 @@ export class IModelTransformer extends IModelExportHandler { this._initialized = true; } + private async handleCustomChanges( + hasElementChangedCache: Set, + deleteIdsProcessed: Set + ): Promise { + // The hasElementChangedCache gets populated by changes from this._csFileProps. + // Because there is a possibility that someone could manually add ids to exporter.sourceDbChanges, we must separately process exporter.sourceDbChanges and add them to our hasElementChangedCache. + // Without this change we risk onExportElement returning early because we use hasElementChangedCache to decide if an element has changed or not. + this.exporter.sourceDbChanges?.element.updateIds.forEach((id) => + hasElementChangedCache.add(id) + ); + this.exporter.sourceDbChanges?.element.insertIds.forEach((id) => + hasElementChangedCache.add(id) + ); + + // This loop is to process all custom deleteIds. Unclear if the special logic is still necessary for relationships or not (TODO!!). For all other entities, we assume that the element is still present in the sourceDb because it is not + // a real delete and instead a simulated delete to update filtering criteria between source and target. Since the element is still present, we do not need to call processDeletedOp to find the corresponding targetId. + // We can instead rely on `forEachTrackedElement` at the top of processChangesets to find the corresponding targetId. + // Note this also assumes we don't need to handle entity recreation for these custom deletes. I.e. a caller of API would not be able to add a custom delete for an entity that was recreated. + // a delete followed by an insert. + // ASSUME: If a changeset has a deleteId then custom change will never reference it. Is this still true if it was re-inserted? (TODO!!) + if (this.exporter.sourceDbChanges?.hasCustomRelationshipChanges) { + for (const id of this.exporter.sourceDbChanges?.relationship.deleteIds.keys() ?? + []) { + if (deleteIdsProcessed?.has(id)) continue; + + const customData = + this.exporter.sourceDbChanges?.getCustomRelationshipDataFromId(id); + if (customData === undefined) { + Logger.logError( + loggerCategory, + "Custom data not found for relationship.", + { id } + ); + continue; + } + const classFullName = customData.classFullName; + const sourceIdOfRelationshipInSource = + customData?.sourceIdOfRelationship; + const targetIdOfRelationshipInSource = + customData?.targetIdOfRelationship; + await this.processRelationshipDeleteOp( + id, + classFullName, + sourceIdOfRelationshipInSource, + targetIdOfRelationshipInSource + ); + } + } + } + /** * Reads all the changeset files in the private member of the transformer: _csFileProps and does two things with these changesets. * Finds the corresponding target entity for any deleted source entities and remaps the sourceId to the targetId. @@ -2789,8 +2840,18 @@ export class IModelTransformer extends IModelExportHandler { this.context.remapElement(sourceElementId, targetElementId); } ); - if (this._csFileProps === undefined || this._csFileProps.length === 0) - return; + await this.addCustomChanges(this.exporter.sourceDbChanges); + + if (this._csFileProps === undefined || this._csFileProps.length === 0) { + if ( + this.exporter.sourceDbChanges === undefined || + !this.exporter.sourceDbChanges.hasChanges + ) + return; + // our sourcedbChanges aren't empty (probably due to someone adding custom changes), change our sourceChangeDataState to has-changes + if (this._sourceChangeDataState === "no-changes") + this._sourceChangeDataState = "has-changes"; + } const hasElementChangedCache = new Set(); const relationshipECClassIdsToSkip = new Set(); @@ -2833,9 +2894,12 @@ export class IModelTransformer extends IModelExportHandler { alreadyImportedModelInserts.add(targetModelId); } ); - this._deletedSourceRelationshipData = new Map(); - for (const csFile of this._csFileProps) { + this._deletedSourceRelationshipData = new Map(); + /** a map of element ids to this transformation scope's ESA data for that element, in case the ESA is deleted in the target */ + const elemIdToScopeEsa = new Map(); + const deleteIdsProcessed = new Set(); + for (const csFile of this._csFileProps ?? []) { const csReader = SqliteChangesetReader.openFile({ fileName: csFile.pathname, db: this.sourceDb, @@ -2848,8 +2912,6 @@ export class IModelTransformer extends IModelExportHandler { } const changes: ChangedECInstance[] = [...ecChangeUnifier.instances]; - /** a map of element ids to this transformation scope's ESA data for that element, in case the ESA is deleted in the target */ - const elemIdToScopeEsa = new Map(); for (const change of changes) { if ( change.ECClassId !== undefined && @@ -2889,145 +2951,227 @@ export class IModelTransformer extends IModelExportHandler { relationshipECClassIdsToSkip.has(ecClassId) ) continue; - await this.processDeletedOp( - change, - elemIdToScopeEsa, - relationshipECClassIds.has(ecClassId ?? ""), - alreadyImportedElementInserts, - alreadyImportedModelInserts - ); + if (relationshipECClassIds.has(ecClassId)) { + if (change.$meta?.classFullName === undefined) { + Logger.logError( + loggerCategory, + "ClassFullName was not found for relationship when reading changes. Relationship delete will not propagate.", + { relationshipId: change.ECInstanceId, ecClassId } + ); + continue; + } + if ( + change.SourceECInstanceId === undefined || + change.TargetECInstanceId === undefined + ) { + Logger.logError( + loggerCategory, + "SourceECInstanceId or TargetECInstanceId was not found for relationship when reading changes. Relationship delete will not propagate.", + { + relationshipId: change.ECInstanceId, + ecClassId, + classFullName: change.$meta.classFullName, + } + ); + continue; + } + await this.processRelationshipDeleteOp( + change.ECInstanceId, + change.$meta.classFullName, + change.SourceECInstanceId, + change.TargetECInstanceId + ); + } else { + await this.processElementDeleteOp( + change.ECInstanceId, + alreadyImportedElementInserts, + alreadyImportedModelInserts, + elemIdToScopeEsa, + change.FederationGuid + ); + } + deleteIdsProcessed.add(change.ECInstanceId); } csReader.close(); } + + await this.handleCustomChanges(hasElementChangedCache, deleteIdsProcessed); + this._hasElementChangedCache = hasElementChangedCache; return; } + + /** + * Helper function for processChangesets. + * Populates the '_deletedSourceRelationshipData' map, whose key is the id of the relationship in the source and the value is an object used to find that relationship in the target. + * @param changedInstanceId The id of the relationship that was deleted + * @param classFullName classFullName of relationship + * @param sourceIdOfRelationshipInSource the element Id acting as the source of the relationship in the sourceDb + * @param targetIdOfRelationshipInSource the element Id acting as the target of the relationship in the sourceDb + * @returns + */ + private async processRelationshipDeleteOp( + changedInstanceId: Id64String, + classFullName: string, + sourceIdOfRelationshipInSource: Id64String, + targetIdOfRelationshipInSource: Id64String + ) { + // we need a connected iModel with changes to remap elements with deletions + const notConnectedModel = this.sourceDb.iTwinId === undefined; + const noChanges = + this.synchronizationVersion.index === this.sourceDb.changeset.index && + (this.exporter.sourceDbChanges === undefined || + !this.exporter.sourceDbChanges.hasChanges); + if (notConnectedModel || noChanges) return; + + const sourceIdOfRelationshipInTarget = await this.getTargetIdFromSourceId( + sourceIdOfRelationshipInSource, + true + ); + const targetIdOfRelationshipInTarget = await this.getTargetIdFromSourceId( + targetIdOfRelationshipInSource, + true + ); + if (sourceIdOfRelationshipInTarget && targetIdOfRelationshipInTarget) { + this._deletedSourceRelationshipData!.set(changedInstanceId, { + classFullName, + sourceIdInTarget: sourceIdOfRelationshipInTarget, + targetIdInTarget: targetIdOfRelationshipInTarget, + }); + } else if (this.sourceDb === this.provenanceSourceDb) { + const relProvenance = this._queryProvenanceForRelationship( + changedInstanceId, + { + classFullName, + sourceId: sourceIdOfRelationshipInSource, + targetId: targetIdOfRelationshipInSource, + } + ); + if (relProvenance && relProvenance.relationshipId) + this._deletedSourceRelationshipData!.set(changedInstanceId, { + classFullName, + relId: relProvenance.relationshipId, + provenanceAspectId: relProvenance.aspectId, + }); + } + } + + /** + * This function is called by the transformer as it is about to process the changesets passed to it in [[IModelTransformOptions.argsForProcessChanges]]. + * This would be after the exporter has already processed the same set of changesets passed to the transformer in [[IModelTransformOptions.argsForProcessChanges]]. + * This function should be used to modify the exporter's sourceDbChanges, if necessary, using [[ChangedInstanceIds.addCustomChange]]. See [[ChangedInstanceIds.addCustomChange]] for more information. + * @param sourceDbChanges will only be defined if the transformer was called with [[IModelTransformOptions.argsForProcessChanges]]. + * @note If defined, sourceDbChanges will already be populated with the changesets passed to the transformer, if any when this function is called by the transformer. + * @note The transformer will have built up the remap table between the source and target iModels before calling this function. This means that functions like [[IModelTransformer.context.findTargetElementId]] will return meaningful results. + * @note Its expected that this function be overridden by a subclass of transformer if it needs to modify sourceDbChanges. + */ + protected async addCustomChanges( + _sourceDbChanges?: ChangedInstanceIds + ): Promise {} + /** * Helper function for processChangesets. Remaps the id of element deleted found in the 'change' to an element in the targetDb. * @param change the change to process, must be of changeType "Deleted" * @param mapOfDeletedElemIdToScopeEsas a map of elementIds to changedECInstances (which are ESAs). the elementId is not the id of the esa itself, but the elementid that the esa was stored on before the esa's deletion. * All ESAs in this map are part of the transformer's scope / ESA data and are tracked in case the ESA is deleted in the target. - * @param isRelationship is relationship or not * @param alreadyImportedElementInserts used to handle entity recreation and not delete already handled element inserts. * @param alreadyImportedModelInserts used to handle entity recreation and not delete already handled model inserts. * @returns void */ - private async processDeletedOp( - change: ChangedECInstance, - mapOfDeletedElemIdToScopeEsas: Map, - isRelationship: boolean, + private async processElementDeleteOp( + changedInstanceId: Id64String, alreadyImportedElementInserts: Set, - alreadyImportedModelInserts: Set + alreadyImportedModelInserts: Set, + mapOfDeletedElemIdToScopeEsas: Map, + federationGuid?: Id64String ) { // we need a connected iModel with changes to remap elements with deletions const notConnectedModel = this.sourceDb.iTwinId === undefined; const noChanges = - this.synchronizationVersion.index === this.sourceDb.changeset.index; + this.synchronizationVersion.index === this.sourceDb.changeset.index && + (this.exporter.sourceDbChanges === undefined || + !this.exporter.sourceDbChanges.hasChanges); if (notConnectedModel || noChanges) return; + let targetId = await this.getTargetIdFromSourceId( + changedInstanceId, + false, + mapOfDeletedElemIdToScopeEsas, + federationGuid + ); + if (targetId === undefined && this.sourceDb === this.provenanceSourceDb) { + targetId = this._queryProvenanceForElement(changedInstanceId); + } + // since we are processing one changeset at a time, we can see local source deletes + // of entities that were never synced and can be safely ignored + const deletionNotInTarget = !targetId; + if (deletionNotInTarget) return; + this.context.remapElement(changedInstanceId, targetId!); + // If an entity insert and an entity delete both point to the same entity in target iModel, that means that entity was recreated. + // In such case an entity update will be triggered and we no longer need to delete the entity. + if (alreadyImportedElementInserts.has(targetId!)) { + this.exporter.sourceDbChanges?.element.deleteIds.delete( + changedInstanceId + ); + } + if (alreadyImportedModelInserts.has(targetId!)) { + this.exporter.sourceDbChanges?.model.deleteIds.delete(changedInstanceId); + } + } + + /** + * Find the corresponding id in the targetDb given a id from the sourceDb + * @param id the id in the source that we want to find the target id for + * @param isRelationship Changes the way we look for the federationGuid , if true we look for the federationGuid on the element itself, if false we expect it to be passed in because it was part of the ChangedECInstance. + * Typically the source and targetIds of the relationship and not the relationshipId itself is passed to this function + * @param mapOfDeletedElemIdToScopeEsas a map of elementIds to changedECInstances (which are ESAs). the elementId is not the id of the esa itself, but the elementid that the esa was stored on before the esa's deletion. + * All ESAs in this map are part of the transformer's scope / ESA data and are tracked in case the ESA is deleted in the target. + * @param federationGuid + * @returns id of the corresponding entity in the targetDb or undefined if not found + */ + private async getTargetIdFromSourceId( + id: Id64String, + isRelationship: boolean, + mapOfDeletedElemIdToScopeEsas?: Map, + federationGuid?: Id64String + ): Promise { /** * if our ChangedECInstance is in the provenanceDb, then we can use the ids we find in the ChangedECInstance to query for ESAs. * This is because the ESAs are stored on an element Id thats present in the provenanceDb. */ const changeDataInProvenanceDb = this.sourceDb === this.provenanceDb; - const getTargetIdFromSourceId = async (id: Id64String) => { - let identifierValue: string | undefined; - let element; - if (isRelationship) { - element = this.sourceDb.elements.tryGetElement(id); - } - const fedGuid = isRelationship - ? element?.federationGuid - : change.FederationGuid; - if (changeDataInProvenanceDb) { - // TODO: clarify what happens if there are multiple (e.g. elements were merged) - for await (const row of this.sourceDb.createQueryReader( - "SELECT esa.Identifier FROM bis.ExternalSourceAspect esa WHERE Scope.Id=:scopeId AND Kind=:kind AND Element.Id=:relatedElementId LIMIT 1", - QueryBinder.from([ - this.targetScopeElementId, - ExternalSourceAspect.Kind.Element, - id, - ]) - )) { - identifierValue = row.Identifier; - } - identifierValue = - identifierValue ?? mapOfDeletedElemIdToScopeEsas.get(id)?.Identifier; - } - - // Check for targetId by an esa first - if (changeDataInProvenanceDb && identifierValue) { - const targetId = identifierValue; - return targetId; - } - - // Check for targetId using sourceId's fedguid if we didn't find an esa. - if (fedGuid) { - const targetId = this._queryElemIdByFedGuid(this.targetDb, fedGuid); - return targetId; - } - return undefined; - }; - - const changedInstanceId = change.ECInstanceId; + let identifierValue: string | undefined; + let element; if (isRelationship) { - const sourceIdOfRelationshipInSource = change.SourceECInstanceId; - const targetIdOfRelationshipInSource = change.TargetECInstanceId; - const classFullName = change.$meta?.classFullName; - - const sourceIdOfRelationshipInTarget = await getTargetIdFromSourceId( - sourceIdOfRelationshipInSource - ); - const targetIdOfRelationshipInTarget = await getTargetIdFromSourceId( - targetIdOfRelationshipInSource - ); - if (sourceIdOfRelationshipInTarget && targetIdOfRelationshipInTarget) { - this._deletedSourceRelationshipData!.set(changedInstanceId, { - classFullName: classFullName ?? "", - sourceIdInTarget: sourceIdOfRelationshipInTarget, - targetIdInTarget: targetIdOfRelationshipInTarget, - }); - } else if (this.sourceDb === this.provenanceSourceDb) { - const relProvenance = this._queryProvenanceForRelationship( - changedInstanceId, - { - classFullName: classFullName ?? "", - sourceId: sourceIdOfRelationshipInSource, - targetId: targetIdOfRelationshipInSource, - } - ); - if (relProvenance && relProvenance.relationshipId) - this._deletedSourceRelationshipData!.set(changedInstanceId, { - classFullName: classFullName ?? "", - relId: relProvenance.relationshipId, - provenanceAspectId: relProvenance.aspectId, - }); - } - } else { - let targetId = await getTargetIdFromSourceId(changedInstanceId); - if (targetId === undefined && this.sourceDb === this.provenanceSourceDb) { - targetId = this._queryProvenanceForElement(changedInstanceId); - } - // since we are processing one changeset at a time, we can see local source deletes - // of entities that were never synced and can be safely ignored - const deletionNotInTarget = !targetId; - if (deletionNotInTarget) return; - this.context.remapElement(changedInstanceId, targetId!); - // If an entity insert and an entity delete both point to the same entity in target iModel, that means that entity was recreated. - // In such case an entity update will be triggered and we no longer need to delete the entity. - if (alreadyImportedElementInserts.has(targetId!)) { - this.exporter.sourceDbChanges?.element.deleteIds.delete( - changedInstanceId - ); - } - if (alreadyImportedModelInserts.has(targetId!)) { - this.exporter.sourceDbChanges?.model.deleteIds.delete( - changedInstanceId - ); + element = this.sourceDb.elements.tryGetElement(id); + } + const fedGuid = isRelationship ? element?.federationGuid : federationGuid; + // Check for targetId using sourceId's fedguid + if (fedGuid) { + const targetId = this._queryElemIdByFedGuid(this.targetDb, fedGuid); + if (targetId !== undefined) return targetId; + } + // Check for targetId by esa + if (changeDataInProvenanceDb) { + // TODO: clarify what happens if there are multiple (e.g. elements were merged) + for await (const row of this.sourceDb.createQueryReader( + "SELECT esa.Identifier FROM bis.ExternalSourceAspect esa WHERE Scope.Id=:scopeId AND Kind=:kind AND Element.Id=:relatedElementId LIMIT 1", + QueryBinder.from([ + this.targetScopeElementId, + ExternalSourceAspect.Kind.Element, + id, + ]) + )) { + identifierValue = row.Identifier; } + identifierValue = + identifierValue ?? mapOfDeletedElemIdToScopeEsas?.get(id)?.Identifier; + if (identifierValue) return identifierValue; } + + return undefined; } private async _tryInitChangesetData(args?: ProcessChangesOptions) { diff --git a/packages/transformer/src/test/IModelTransformerUtils.ts b/packages/transformer/src/test/IModelTransformerUtils.ts index a1aab993..472b18f5 100644 --- a/packages/transformer/src/test/IModelTransformerUtils.ts +++ b/packages/transformer/src/test/IModelTransformerUtils.ts @@ -139,6 +139,55 @@ export class IModelTransformerTestUtils extends TestUtils.IModelTestUtils { return iModelDb; } + /** Returns path to a schema which contains a multiAspect TestSchema2:MyMultiAspect. + * The schema is created in the output directory. + * The multi aspect has a prop 'MyProp1'. + * Users should import this schema in order to insert multi aspects. + */ + public static getPathToSchemaWithMultiAspect(): string { + const testSchema1Path = IModelTransformerTestUtils.prepareOutputFile( + "IModelTransformer", + "TestSchema2.ecschema.xml" + ); + IModelJsFs.writeFileSync( + testSchema1Path, + ` + + + + bis:ElementMultiAspect + + + ` + ); + return testSchema1Path; + } + + /** Returns path to a schema which contains a UniqueAspect TestSchema1:MyUniqueAspect. + * The schema is created in the output directory. + * the only two ElementUniqueAspect's in bis are ignored by the transformer, so we can add our own to test their export + * The unique aspect has a prop 'MyProp1'. + * Users should import this schema in order to insert unique aspects. + */ + public static getPathToSchemaWithUniqueAspect(): string { + const testSchema1Path = IModelTransformerTestUtils.prepareOutputFile( + "IModelTransformer", + "TestSchema1.ecschema.xml" + ); + IModelJsFs.writeFileSync( + testSchema1Path, + ` + + + + bis:ElementUniqueAspect + + + ` + ); + return testSchema1Path; + } + public static populateTeamIModel( teamDb: IModelDb, teamName: string, diff --git a/packages/transformer/src/test/TestUtils/TimelineTestUtil.ts b/packages/transformer/src/test/TestUtils/TimelineTestUtil.ts index fd9cff0b..594184b3 100644 --- a/packages/transformer/src/test/TestUtils/TimelineTestUtil.ts +++ b/packages/transformer/src/test/TestUtils/TimelineTestUtil.ts @@ -34,6 +34,7 @@ import { } from "../IModelTransformerUtils"; import { IModelTestUtils } from "./IModelTestUtils"; import { omit } from "@itwin/core-bentley"; +import { ExportChangesOptions, IModelExporter } from "../../IModelExporter"; const saveAndPushChanges = async ( accessToken: string, @@ -253,7 +254,12 @@ export type TimelineStateChange = source: string, opts?: { since?: number; - initTransformer?: (transformer: IModelTransformer) => void; + init?: { + initTransformer?: (transformer: IModelTransformer) => void; + afterInitializeExporter?: ( + exporter: IModelExporter + ) => Promise; // Run this code after exporter.initialize is called + }; expectThrow?: boolean; assert?: { afterProcessChanges?: (transformer: IModelTransformer) => void; @@ -357,7 +363,12 @@ export async function runTimeline( src: string, opts: { since?: number; - initTransformer?: (transformer: IModelTransformer) => void; + init?: { + afterInitializeExporter?: ( + exporter: IModelExporter + ) => Promise; + initTransformer?: (transformer: IModelTransformer) => void; + }; expectThrow?: boolean; assert?: { afterProcessChanges?: (transformer: IModelTransformer) => void; @@ -498,7 +509,7 @@ export async function runTimeline( syncSource, { since: startIndex, - initTransformer, + init: initFxns, expectThrow, assert: assertFxns, }, @@ -511,16 +522,21 @@ export async function runTimeline( let targetStateBefore: TimelineIModelElemState | undefined; if (process.env.TRANSFORMER_BRANCH_TEST_DEBUG) targetStateBefore = getIModelState(target.db); - + let argsForProcessChanges: ExportChangesOptions = { csFileProps: [] }; + if (startIndex) { + argsForProcessChanges = { startChangeset: { index: startIndex } }; + } const syncer = new IModelTransformer(source.db, target.db, { ...transformerOpts, - argsForProcessChanges: { - startChangeset: startIndex - ? { index: startIndex } - : { index: undefined }, - }, + argsForProcessChanges, }); - initTransformer?.(syncer); + + if (initFxns?.afterInitializeExporter) { + await syncer.exporter.initialize(argsForProcessChanges); + await initFxns?.afterInitializeExporter?.(syncer.exporter); + } + + initFxns?.initTransformer?.(syncer); try { await syncer.process(); expect( diff --git a/packages/transformer/src/test/standalone/IModelTransformer.test.ts b/packages/transformer/src/test/standalone/IModelTransformer.test.ts index 75292349..d88c4ffb 100644 --- a/packages/transformer/src/test/standalone/IModelTransformer.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformer.test.ts @@ -3086,24 +3086,9 @@ describe("IModelTransformer", () => { rootSubject: { name: "deferred-element-with-aspects" }, }); - const testSchema1Path = IModelTransformerTestUtils.prepareOutputFile( - "IModelTransformer", - "TestSchema1.ecschema.xml" - ); - // the only two ElementUniqueAspect's in bis are ignored by the transformer, so we add our own to test their export - IModelJsFs.writeFileSync( - testSchema1Path, - ` - - - - bis:ElementUniqueAspect - - - ` - ); - - await sourceDb.importSchemas([testSchema1Path]); + const testSchemaPath = + IModelTransformerTestUtils.getPathToSchemaWithUniqueAspect(); + await sourceDb.importSchemas([testSchemaPath]); const myPhysicalModelId = PhysicalModel.insert( sourceDb, diff --git a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts index 814caa23..f59a0dd6 100644 --- a/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts +++ b/packages/transformer/src/test/standalone/IModelTransformerHub.test.ts @@ -20,9 +20,11 @@ import { ECSqlStatement, // eslint-disable-next-line @typescript-eslint/no-redeclare Element, + ElementAspect, ElementGroupsMembers, ElementOwnsChildElements, ElementOwnsExternalSourceAspects, + ElementOwnsMultiAspects, ElementRefersToElements, ExternalSourceAspect, GenericSchema, @@ -1302,7 +1304,11 @@ describe("IModelTransformerHub", () => { master: { sync: [ "branch1", - { initTransformer: setForceOldRelationshipProvenanceMethod }, + { + init: { + initTransformer: setForceOldRelationshipProvenanceMethod, + }, + }, ], }, }, // first master<-branch1 reverse sync picking up new relationship from branch imodel @@ -1346,7 +1352,11 @@ describe("IModelTransformerHub", () => { branch1: { sync: [ "master", - { initTransformer: setForceOldRelationshipProvenanceMethod }, + { + init: { + initTransformer: setForceOldRelationshipProvenanceMethod, + }, + }, ], }, }, // forward sync master->branch1 to pick up delete of relationship @@ -1804,6 +1814,379 @@ describe("IModelTransformerHub", () => { } }); + it("should propagate custom inserts and custom deletes", async () => { + let ecClassIdOfRel: Id64String | undefined; + const masterIModelName = "Master"; + const masterSeedFileName = path.join(outputDir, `${masterIModelName}.bim`); + if (IModelJsFs.existsSync(masterSeedFileName)) + IModelJsFs.removeSync(masterSeedFileName); + const masterSeedState = { 1: 1, 2: 1, 20: 1, 21: 1, 40: 1, 41: 2, 42: 3 }; + const masterSeedDb = SnapshotDb.createEmpty(masterSeedFileName, { + rootSubject: { name: masterIModelName }, + }); + // eslint-disable-next-line deprecation/deprecation + masterSeedDb.nativeDb.setITwinId(iTwinId); // workaround for "ContextId was not properly setup in the checkpoint" issue + const schemaPathForMultiAspect = + IModelTransformerTestUtils.getPathToSchemaWithMultiAspect(); + await masterSeedDb.importSchemas([schemaPathForMultiAspect]); + for await (const row of masterSeedDb.createQueryReader( + "SELECT ECInstanceId FROM ECdbMeta.ECClassDef WHERE Name LIKE 'ElementGroupsMembers'" + )) { + ecClassIdOfRel = row.ECInstanceId; + } + populateTimelineSeed(masterSeedDb, masterSeedState); + + const masterSeed: TimelineIModelState = { + // HACK: we know this will only be used for seeding via its path and performCheckpoint + db: masterSeedDb as any as BriefcaseDb, + id: "master-seed", + state: masterSeedState, + }; + + let relId: Id64String | undefined; + let sourceIdOfRel: Id64String | undefined; + let targetIdOfRel: Id64String | undefined; + let elementIdInSource: Id64String | undefined; + let aspectIdInSource: Id64String | undefined; + + // I won't delete the element in this case and only the aspect so that I can make sure specific aspect deletes work as expected. + let elementId2InSource: Id64String | undefined; + let aspectId2InSource: Id64String | undefined; + + let physicalModelIdInSource: Id64String | undefined; + let modelUnderRepositoryModel: Id64String | undefined; + const timeline: Timeline = [ + { master: { seed: masterSeed } }, // masterSeedState is above + { branch1: { branch: "master" } }, + { master: { 100: 100, 101: 101 } }, + { + master: { + manualUpdate(db) { + // insert relationship into master + sourceIdOfRel = IModelTestUtils.queryByUserLabel(db, "40"); + targetIdOfRel = IModelTestUtils.queryByUserLabel(db, "2"); + const rel = ElementGroupsMembers.create( + db, + sourceIdOfRel, + targetIdOfRel, + 0 + ); + relId = rel.insert(); + + elementIdInSource = IModelTestUtils.queryByUserLabel(db, "100"); + physicalModelIdInSource = PhysicalModel.insert( + db, + IModel.rootSubjectId, + "MyPhysicalModel" + ); + modelUnderRepositoryModel = DefinitionModel.insert( + db, + IModel.rootSubjectId, + "MyModelUnderRepositoryModel" + ); + // insert aspect + const multiAspectProps = { + classFullName: "TestSchema2:MyMultiAspect", + element: { + id: elementIdInSource, + relClassName: ElementOwnsMultiAspects.classFullName, + }, + myProp1: "prop_value", + }; + aspectIdInSource = db.elements.insertAspect(multiAspectProps); + + elementId2InSource = IModelTestUtils.queryByUserLabel(db, "101"); + multiAspectProps.element.id = elementId2InSource; + multiAspectProps.myProp1 = "prop_value2"; + aspectId2InSource = db.elements.insertAspect(multiAspectProps); + }, + }, + }, + { branch1: { sync: ["master"] } }, // master->branch1 forward sync to pick up relationship change + { + branch1: { + // delete relationship and element from branch so that we can attempt to add it back in as a custom 'Inserted' change + manualUpdate(db) { + const sourceIdInTarget = IModelTestUtils.queryByUserLabel(db, "40"); + const targetIdInTarget = IModelTestUtils.queryByUserLabel(db, "2"); + const rel = db.relationships.getInstance( + ElementGroupsMembers.classFullName, + { sourceId: sourceIdInTarget, targetId: targetIdInTarget } + ); + expect(rel).to.not.be.undefined; + rel.delete(); + + const idOfElement = IModelTestUtils.queryByUserLabel(db, "100"); + expect(idOfElement).to.not.be.undefined; + const aspectsOnElement = db.elements.getAspects( + idOfElement, + ElementAspect.classFullName + ); + expect(aspectsOnElement.length).to.equal(1); + db.elements.deleteElement(idOfElement); + const physicalPartitionIdInTarget = + IModelTestUtils.queryByCodeValue(db, "MyPhysicalModel"); + expect(physicalPartitionIdInTarget).to.not.equal(Id64.invalid); + db.models.deleteModel(physicalPartitionIdInTarget); + db.elements.deleteElement(physicalPartitionIdInTarget); + const modelUnderRepositoryModelId = + IModelTestUtils.queryByCodeValue( + db, + "MyModelUnderRepositoryModel" + ); + expect(modelUnderRepositoryModelId).to.not.equal(Id64.invalid); + db.models.deleteModel(modelUnderRepositoryModelId); + db.elements.deleteElement(modelUnderRepositoryModelId); + + db.elements.deleteAspect(aspectId2InSource!); + }, + }, + }, + { + assert({ branch1 }) { + // Extra assert to make sure relationship and element are deleted in branch1 + const sourceIdInTarget = IModelTestUtils.queryByUserLabel( + branch1.db, + "40" + ); + const targetIdInTarget = IModelTestUtils.queryByUserLabel( + branch1.db, + "2" + ); + const rel = + branch1.db.relationships.tryGetInstance( + ElementGroupsMembers.classFullName, + { sourceId: sourceIdInTarget, targetId: targetIdInTarget } + ); + expect(rel).to.be.undefined; + const element = IModelTestUtils.queryByUserLabel(branch1.db, "100"); + expect(element).to.equal(Id64.invalid); + // assume if element is gone, aspect is gone + const physicalPartitionIdInTarget = IModelTestUtils.queryByCodeValue( + branch1.db, + "MyPhysicalModel" + ); + expect(physicalPartitionIdInTarget).to.equal(Id64.invalid); + const modelUnderRepositoryModelInTarget = + IModelTestUtils.queryByCodeValue( + branch1.db, + "MyModelUnderRepositoryModel" + ); + expect(modelUnderRepositoryModelInTarget).to.equal(Id64.invalid); + + const element2 = IModelTestUtils.queryByUserLabel(branch1.db, "101"); + expect(element2).to.not.equal(Id64.invalid); + const aspectsOnElement2 = branch1.db.elements.getAspects( + element2, + ElementAspect.classFullName + ); + expect(aspectsOnElement2.length).to.equal(0); + }, + }, + { + branch1: { + sync: [ + "master", + { + init: { + afterInitializeExporter: async (exporter) => { + // Add custom changes to re-insert relationship and element + await exporter.sourceDbChanges?.addCustomRelationshipChange( + ecClassIdOfRel!, + "Inserted", + relId!, + sourceIdOfRel!, + targetIdOfRel! + ); + exporter.sourceDbChanges?.addCustomElementChange( + "Inserted", + elementIdInSource! + ); + exporter.sourceDbChanges?.addCustomAspectChange( + "Inserted", + aspectIdInSource! + ); + exporter.sourceDbChanges?.addCustomElementChange( + "Inserted", + physicalModelIdInSource! + ); + exporter.sourceDbChanges?.addCustomModelChange( + "Inserted", + physicalModelIdInSource! + ); + exporter.sourceDbChanges?.addCustomModelChange( + "Inserted", + modelUnderRepositoryModel! + ); + exporter.sourceDbChanges?.addCustomElementChange( + "Inserted", + modelUnderRepositoryModel! + ); + exporter.sourceDbChanges?.addCustomAspectChange( + "Inserted", + aspectId2InSource! + ); + }, + }, + }, + ], + }, + }, + { + assert({ branch1 }) { + // Validate custom changes worked and we can find the inserted elements in branch1 + const sourceIdInTarget = IModelTestUtils.queryByUserLabel( + branch1.db, + "40" + ); + const targetIdInTarget = IModelTestUtils.queryByUserLabel( + branch1.db, + "2" + ); + const rel = + branch1.db.relationships.getInstance( + ElementGroupsMembers.classFullName, + { sourceId: sourceIdInTarget, targetId: targetIdInTarget } + ); + expect(rel).to.not.be.undefined; + const elementInTarget = IModelTestUtils.queryByUserLabel( + branch1.db, + "100" + ); + expect(elementInTarget).to.not.equal(Id64.invalid); + const aspectsOnElement = branch1.db.elements.getAspects( + elementInTarget, + ElementAspect.classFullName + ); + expect(aspectsOnElement.length).to.equal(1); + + const element2InTarget = IModelTestUtils.queryByUserLabel( + branch1.db, + "101" + ); + const aspectsOnElement2 = branch1.db.elements.getAspects( + element2InTarget, + ElementAspect.classFullName + ); + expect(aspectsOnElement2.length).to.equal(1); // validates that the custom aspect insert worked. + + const physicalPartitionIdInTarget = IModelTestUtils.queryByCodeValue( + branch1.db, + "MyPhysicalModel" + ); + expect(physicalPartitionIdInTarget).to.not.equal(Id64.invalid); + expect(branch1.db.elements.getElement(physicalPartitionIdInTarget)).to + .not.be.undefined; + expect(branch1.db.models.getModel(physicalPartitionIdInTarget)).to.not + .be.undefined; + const modelUnderRepositoryModelInTarget = + IModelTestUtils.queryByCodeValue( + branch1.db, + "MyModelUnderRepositoryModel" + ); + expect(modelUnderRepositoryModelInTarget).to.not.equal(Id64.invalid); + expect( + branch1.db.elements.getElement(modelUnderRepositoryModelInTarget) + ).to.not.be.undefined; + expect(branch1.db.models.getModel(modelUnderRepositoryModelInTarget)) + .to.not.be.undefined; + }, + }, + { + branch1: { + sync: [ + "master", + { + init: { + afterInitializeExporter: async (exporter) => { + // Add custom changes to delete relationship and element + await exporter.sourceDbChanges?.addCustomRelationshipChange( + ecClassIdOfRel!, + "Deleted", + relId!, + sourceIdOfRel!, + targetIdOfRel! + ); + exporter.sourceDbChanges?.addCustomElementChange( + "Deleted", + elementIdInSource! + ); + exporter.sourceDbChanges?.addCustomElementChange( + "Deleted", + physicalModelIdInSource! + ); + exporter.sourceDbChanges?.addCustomModelChange( + "Deleted", + physicalModelIdInSource! + ); + exporter.sourceDbChanges?.addCustomModelChange( + "Deleted", + modelUnderRepositoryModel! + ); + exporter.sourceDbChanges?.addCustomElementChange( + "Deleted", + modelUnderRepositoryModel! + ); + + exporter.sourceDbChanges?.addCustomAspectChange( + "Deleted", + aspectId2InSource! + ); + }, + }, + }, + ], + }, + }, + { + assert({ branch1 }) { + // Assert that they were deleted. + const sourceIdInTarget = IModelTestUtils.queryByUserLabel( + branch1.db, + "40" + ); + const targetIdInTarget = IModelTestUtils.queryByUserLabel( + branch1.db, + "2" + ); + const rel = + branch1.db.relationships.tryGetInstance( + ElementGroupsMembers.classFullName, + { sourceId: sourceIdInTarget, targetId: targetIdInTarget } + ); + + const element = IModelTestUtils.queryByUserLabel(branch1.db, "100"); + expect(element).to.equal(Id64.invalid); + expect(rel).to.be.undefined; + const physicalPartitionIdInTarget = IModelTestUtils.queryByCodeValue( + branch1.db, + "MyPhysicalModel" + ); + expect(physicalPartitionIdInTarget).to.equal(Id64.invalid); + const modelUnderRepositoryModelInTarget = + IModelTestUtils.queryByCodeValue( + branch1.db, + "MyModelUnderRepositoryModel" + ); + expect(modelUnderRepositoryModelInTarget).to.equal(Id64.invalid); + + const element2 = IModelTestUtils.queryByUserLabel(branch1.db, "101"); + expect(element2).to.not.equal(Id64.invalid); + // const aspectsOnElement2 = branch1.db.elements.getAspects( + // element2, + // ElementAspect.classFullName + // ); + // expect(aspectsOnElement2.length).to.equal(0); // validates that the custom aspect delete worked. TODO: doesnt seem like it did? Doesnt even seem like any aspect deletes are handled at the moment? + }, + }, + ]; + const { tearDown } = await runTimeline(timeline, { + iTwinId, + accessToken, + }); + await tearDown(); + }); + it("ModelSelector processChanges", async () => { const sourceIModelName = "ModelSelectorSource"; const sourceIModelId = await HubWrappers.recreateIModel({ @@ -3797,7 +4180,10 @@ describe("IModelTransformerHub", () => { "branch", { expectThrow: false, - initTransformer: setBranchRelationshipDataBehaviorToUnsafeMigrate, + init: { + initTransformer: + setBranchRelationshipDataBehaviorToUnsafeMigrate, + }, }, ], }, @@ -3894,7 +4280,10 @@ describe("IModelTransformerHub", () => { sync: [ "branch", { - initTransformer: setBranchRelationshipDataBehaviorToUnsafeMigrate, + init: { + initTransformer: + setBranchRelationshipDataBehaviorToUnsafeMigrate, + }, }, ], }, @@ -3947,7 +4336,10 @@ describe("IModelTransformerHub", () => { sync: [ "master", { - initTransformer: setBranchRelationshipDataBehaviorToUnsafeMigrate, + init: { + initTransformer: + setBranchRelationshipDataBehaviorToUnsafeMigrate, + }, }, ], }, @@ -4033,9 +4425,11 @@ describe("IModelTransformerHub", () => { sync: [ "branch", { - initTransformer: (transformer) => - (transformer["_options"]["branchRelationshipDataBehavior"] = - "unsafe-migrate"), + init: { + initTransformer: (transformer) => + (transformer["_options"]["branchRelationshipDataBehavior"] = + "unsafe-migrate"), + }, }, ], // Sync again with no changes except for ones which may get made by unsafe-migrate. }, @@ -4174,7 +4568,10 @@ describe("IModelTransformerHub", () => { "master", { expectThrow: false, - initTransformer: setBranchRelationshipDataBehaviorToUnsafeMigrate, + init: { + initTransformer: + setBranchRelationshipDataBehaviorToUnsafeMigrate, + }, }, ], }, @@ -4185,7 +4582,10 @@ describe("IModelTransformerHub", () => { "branch", { expectThrow: false, - initTransformer: setBranchRelationshipDataBehaviorToUnsafeMigrate, + init: { + initTransformer: + setBranchRelationshipDataBehaviorToUnsafeMigrate, + }, }, ], },