Fix observation sync issues

plugins/arcgis/service/src/ArcGISConfig.ts

@@ -1,3 +1,5 @@
+import { MageEventId } from "@ngageoint/mage.service/lib/entities/events/entities.events"
+
 /**
  * Contains an arc feature service url and layers.
  */
@@ -9,8 +11,8 @@ export interface FeatureServiceConfig {
     url: string
 
     /**
-     * Serialized ArcGISIdentityManager
-     */
+    * Serialized ArcGISIdentityManager
+    */
     identityManager: string
 
     /**
@@ -35,9 +37,9 @@ export interface FeatureLayerConfig {
     geometryType?: string
 
     /**
-     * The event ids or names that sync to this arc feature layer.
+     * The event ids that sync to this arc feature layer.
      */
-    events?: (number|string)[]
+    eventIds?: MageEventId[]
 }
 
 

plugins/arcgis/service/src/EventDeletionHandler.ts

@@ -34,6 +34,10 @@ export class EventDeletionHandler {
         this._config = config;
     }
 
+    public updateConfig(newConfig: ArcGISPluginConfig): void {
+        this._config = newConfig;
+    }
+
     /**
      * 
      * @param activeEvents The current set of active events.
@@ -83,7 +87,7 @@ export class EventDeletionHandler {
     }
 
     /**
-     * Called when the query is finished.  It goes through the results and gathers all even Ids currently stored
+     * Called when the query is finished.  It goes through the results and gathers all event Ids currently stored
      * in the arc layer.  It then will remove any events from the arc layer that do not exist.
      * @param layerProcessor The feature layer processor.
      * @param result The returned results.

plugins/arcgis/service/src/EventLayerProcessorOrganizer.ts

@@ -19,7 +19,7 @@ export class EventLayerProcessorOrganizer {
         for (const event of events) {
             let syncProcessors = new Array<FeatureLayerProcessor>();
             for (const layerProcessor of layerProcessors) {
-                if (layerProcessor.layerInfo.hasEvent(event.name)) {
+                if (layerProcessor.layerInfo.hasEvent(event.id)) {
                     syncProcessors.push(layerProcessor);
                 }
             }

plugins/arcgis/service/src/FeatureLayerProcessor.ts

@@ -85,7 +85,7 @@ export class FeatureLayerProcessor {
 
         for (const arcObservation of observations.deletions) {
             if (this.layerInfo.geometryType == arcObservation.esriGeometryType) {
-                this.sender.sendDelete(Number(arcObservation.id));
+                this.sender.sendDelete(arcObservation.id);
             }
         }
     }

plugins/arcgis/service/src/FeatureQuerier.ts

@@ -1,7 +1,8 @@
 import { ArcGISPluginConfig } from "./ArcGISPluginConfig";
 import { LayerInfo } from "./LayerInfo";
 import { QueryObjectResult } from "./QueryObjectResult";
-import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request";
+import { ArcGISIdentityManager } from "@esri/arcgis-rest-request";
+import { queryFeatures } from '@esri/arcgis-rest-feature-service';
 
 /**
  * Performs various queries on observations for a specific arc feature layer.
@@ -52,22 +53,17 @@ export class FeatureQuerier {
      * @param geometry query the geometry, default is true
      */
     async queryObservation(observationId: string, response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) {
-        const queryUrl = new URL(this._url)
-        if (this._config.eventIdField == null) {
-            queryUrl.searchParams.set('where', `${this._config.observationIdField} LIKE '${observationId}${this._config.idSeparator}%'`);
-        } else {
-            queryUrl.searchParams.set('where', `${this._config.observationIdField} = ${observationId}`);
-        }
-        queryUrl.searchParams.set('outFields', this.outFields(fields))
-        queryUrl.searchParams.set('returnGeometry', geometry === false ? 'false' : 'true')
-        this._console.info('ArcGIS query: ' + queryUrl)
-
-        const queryResponse = await request(queryUrl.toString(), {
+        const where = !this._config.eventIdField 
+            ? `${this._config.observationIdField} LIKE '${observationId}${this._config.idSeparator}%'`
+            : `${this._config.observationIdField} = '${observationId}'`;
+        this._console.info('ArcGIS query observation: ' + this._url.toString() + where);
+        await queryFeatures({
+            url: this._url.toString(),
             authentication: this._identityManager,
-            params: { f: 'json' }
-        });
-
-        response(queryResponse as QueryObjectResult);
+            where,
+            returnGeometry: geometry,
+            outFields: fields?.length ? fields : '*'
+        }).then((queryResponse) => response(queryResponse as QueryObjectResult)).catch((error) => this._console.error('Error in FeatureQuerier.queryObservation :: ' + error));
     }
 
     /**
@@ -77,19 +73,14 @@ export class FeatureQuerier {
      * @param geometry query the geometry, default is true
      */
     async queryObservations(response: (result: QueryObjectResult) => void, fields?: string[], geometry?: boolean) {
-        const queryUrl = new URL(this._url)
-        queryUrl.searchParams.set('where', `${this._config.observationIdField} IS NOT NULL`);
-        queryUrl.searchParams.set('outFields', this.outFields(fields));
-        queryUrl.searchParams.set('returnGeometry', geometry === false ? 'false' : 'true');
-
-        this._console.info('ArcGIS query: ' + queryUrl)
-
-        const queryResponse = await request(queryUrl.toString(), {
+        this._console.info('ArcGIS query observation: ' + this._url.toString());
+        await queryFeatures({
+            url: this._url.toString(),
             authentication: this._identityManager,
-            params: { f: 'json' }
-        });
-
-        response(queryResponse as QueryObjectResult);
+            where: `${this._config.observationIdField} IS NOT NULL`,
+            returnGeometry: geometry,
+            outFields: fields?.length ? fields : '*'
+        }).then((queryResponse) => response(queryResponse as QueryObjectResult)).catch((error) => this._console.error('Error in FeatureQuerier.queryObservations :: ' + error));
     }
 
     /**
@@ -98,37 +89,14 @@ export class FeatureQuerier {
      * @param field field to query
      */
     async queryDistinct(response: (result: QueryObjectResult) => void, field: string) {
-        const queryUrl = new URL(this._url);
-        queryUrl.searchParams.set('where', `${field} IS NOT NULL`);
-        queryUrl.searchParams.set('returnDistinctValues', 'true');
-        queryUrl.searchParams.set('outFields', this.outFields([field]));
-        queryUrl.searchParams.set('returnGeometry', 'false');      
-        this._console.info('ArcGIS query: ' + queryUrl)
-
-        try {
-            const queryResponse = await request(queryUrl.toString(), {
-                authentication: this._identityManager,
-                params: { f: 'json' }
-
-            });  
-
-            response(queryResponse as QueryObjectResult);
-        } catch (err) {
-            console.error("could not query", err)
-        }
-    }
-
-    /**
-     * Build the out fields query parameter
-     * @param fields query fields
-     * @returns out fields
-     */
-    private outFields(fields?: string[]): string {
-        if (fields != null && fields.length > 0) {
-            return fields.join(',');
-        } else {
-            return '*';
-        }
+        this._console.info('ArcGIS query observation: ' + this._url.toString());
+        await queryFeatures({
+            url: this._url.toString(),
+            authentication: this._identityManager,
+            where: `${field} IS NOT NULL`,
+            returnGeometry: false,
+            outFields: field ? [field] : '*',
+            returnDistinctValues: true
+        }).then((queryResponse) => response(queryResponse as QueryObjectResult)).catch((error) => this._console.error('Error in FeatureQuerier.queryDistinct :: ' + error));
     }
-
 }

plugins/arcgis/service/src/FeatureServiceAdmin.ts

@@ -1,6 +1,6 @@
 import { ArcGISPluginConfig } from "./ArcGISPluginConfig"
 import { FeatureServiceConfig, FeatureLayerConfig } from "./ArcGISConfig"
-import { MageEvent, MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events'
+import { MageEvent, MageEventId, MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events'
 import { Layer, Field } from "./AddLayersRequest"
 import { Form, FormField, FormFieldType, FormId } from '@ngageoint/mage.service/lib/entities/events/entities.events.forms'
 import { ObservationsTransformer } from "./ObservationsTransformer"
@@ -120,29 +120,29 @@ export class FeatureServiceAdmin {
 	}
 
 	/**
-	 * Get the layer events
+	 * Get the Mage layer events
 	 * @param layer feature layer
 	 * @param eventRepo event repository
-	 * @returns layer events
+	 * @returns Mage layer events
 	 */
 	private async layerEvents(layer: FeatureLayerConfig, eventRepo: MageEventRepository): Promise<MageEvent[]> {
-		const layerEvents: Set<number|string> = new Set()
-		if (layer.events != null) {
-			for (const layerEvent of layer.events) {
-				layerEvents.add(layerEvent)
+		const layerEventIds: Set<MageEventId> = new Set()
+		 if (layer.eventIds != null) {
+			for (const layerEventId of layer.eventIds) {
+				layerEventIds.add(layerEventId)
 			}
 		}
 
 		let mageEvents
-		if (layerEvents.size > 0) {
+		if (layerEventIds.size > 0) {
 			mageEvents = await eventRepo.findAll()
 		} else {
 			mageEvents = await eventRepo.findActiveEvents()
 		}
 
 		const events: MageEvent[] = []
 		for (const mageEvent of mageEvents) {
-			if (layerEvents.size == 0 || layerEvents.has(mageEvent.name) || layerEvents.has(mageEvent.id)) {
+			if (layerEventIds.size == 0 || layerEventIds.has(mageEvent.id)) {
 				const event = await eventRepo.findById(mageEvent.id)
 				if (event != null) {
 					events.push(event)

plugins/arcgis/service/src/LayerInfo.ts

@@ -1,3 +1,4 @@
+import { MageEventId } from "@ngageoint/mage.service/lib/entities/events/entities.events";
 import { LayerInfoResult, LayerField } from "./LayerInfoResult";
 
 /**
@@ -28,7 +29,7 @@ export class LayerInfo {
     /**
      * The events that are synching to this layer.
      */
-    events: Set<string> = new Set<string>()
+    events: Set<MageEventId> = new Set<MageEventId>()
 
     /**
      * Constructor.
@@ -37,12 +38,10 @@ export class LayerInfo {
      * @param layerInfo The layer info.
      * @param token The access token.
      */
-    constructor(url: string, events: string[], layerInfo: LayerInfoResult) {
+    constructor(url: string, events: MageEventId[], layerInfo: LayerInfoResult) {
         this.url = url
-        if (events != undefined && events != null && events.length == 0) {
-            this.events.add('nothing to sync')
-        }
-        if (events != undefined || events != null) {
+
+        if (events && events.length > 0) {
             for (const event of events) {
                 this.events.add(event);
             }
@@ -69,11 +68,11 @@ export class LayerInfo {
 
     /**
      * Determine if the layer is enabled for the event.
-     * @param event The event.
+     * @param eventId The event.
      * @return true if enabled
      */
-    hasEvent(event: string) {
-        return this.events.size == 0 || this.events.has(event)
+    hasEvent(eventId: MageEventId) {
+        return this.events.size == 0 || this.events.has(eventId)
     }
 
 }

plugins/arcgis/service/src/ObservationBinner.ts

@@ -89,7 +89,8 @@ export class ObservationBinner {
         const bins = new ObservationBins();
 
         for (const arcObservation of observations.observations) {
-            if (arcObservation.lastModified != arcObservation.createdAt) {
+            // TODO: Would probably want a better way to determine which observations need to be updated in arcgis
+            if (observations.firstRun || arcObservation.lastModified != arcObservation.createdAt) {
                 bins.updates.add(arcObservation);
             } else if (!this._addedObs.has(arcObservation.id)) {
                 bins.adds.add(arcObservation);