Cgmes import update. Import EQ and SSH separately. ShuntCompensators

cgmes/cgmes-completion/src/test/java/com/powsybl/cgmes/completion/CgmesCompletionTest.java

@@ -10,16 +10,13 @@
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
 import com.powsybl.cgmes.conversion.CgmesImport;
-import com.powsybl.cgmes.conversion.CgmesModelExtension;
-import com.powsybl.cgmes.model.CgmesModel;
 import com.powsybl.commons.config.InMemoryPlatformConfig;
 import com.powsybl.commons.datasource.ReadOnlyDataSource;
 import com.powsybl.commons.datasource.ResourceDataSource;
 import com.powsybl.commons.datasource.ResourceSet;
 import com.powsybl.computation.ComputationManager;
 import com.powsybl.computation.local.LocalComputationManager;
-import com.powsybl.iidm.network.Importers;
-import com.powsybl.iidm.network.Network;
+import com.powsybl.iidm.network.*;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -74,10 +71,11 @@ void miniGridNodeBreakerMissingVoltageLevel() throws IOException {
         }
         assertNotNull(network);
 
-        // Check that a specific terminal has a voltage level, navigating the CGMES model
-        CgmesModel cgmes = network.getExtension(CgmesModelExtension.class).getCgmesModel();
+        // Check that a specific terminal has a voltage level, by using the alias to get the busBarSection
         String terminalId = "T_BBS";
-        String voltageLevelId = cgmes.voltageLevel(cgmes.terminal(terminalId), cgmes.isNodeBreaker());
+        Connectable<?> connectable = network.getConnectable(terminalId);
+        BusbarSection busBarSection = (BusbarSection) connectable;
+        String voltageLevelId = busBarSection.getTerminal().getVoltageLevel().getId();
         if (voltageLevelId == null || voltageLevelId.isEmpty()) {
             fail("Missing voltage level for terminal " + terminalId);
         }

cgmes/cgmes-conformity/src/main/resources/conformity-modified/cas-1.1.3-data-4.0.3/MicroGrid/BaseCase/BC_BE_v2_hidden_tc/MicroGridTestConfiguration_BC_BE_SSH_V2.xml

@@ -178,15 +178,15 @@
     <cim:TapChanger.controlEnabled>true</cim:TapChanger.controlEnabled>
   </cim:PhaseTapChangerAsymmetrical>
   <cim:PhaseTapChangerSymmetrical rdf:about="#_6ebbef67-3061-4236-a6fd-6ccc4595f6c3-x">
-    <cim:TapChanger.step>3</cim:TapChanger.step>
+    <cim:TapChanger.step>9</cim:TapChanger.step>
     <cim:TapChanger.controlEnabled>false</cim:TapChanger.controlEnabled>
   </cim:PhaseTapChangerSymmetrical>
   <cim:RatioTapChanger rdf:about="#_955d9cd0-4a10-4031-b008-60c0dc340a07">
     <cim:TapChanger.step>10</cim:TapChanger.step>
     <cim:TapChanger.controlEnabled>false</cim:TapChanger.controlEnabled>
   </cim:RatioTapChanger>
   <cim:RatioTapChanger rdf:about="#_955d9cd0-4a10-4031-b008-60c0dc340a07-x">
-    <cim:TapChanger.step>5</cim:TapChanger.step>
+    <cim:TapChanger.step>12</cim:TapChanger.step>
     <cim:TapChanger.controlEnabled>false</cim:TapChanger.controlEnabled>
   </cim:RatioTapChanger>
   <cim:RatioTapChanger rdf:about="#_83cc66dd-8d93-4a2c-8103-f1f5a9cf7e2e">

cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java

@@ -176,6 +176,21 @@ private Network importData1(ReadOnlyDataSource ds, NetworkFactory networkFactory
         return new Conversion(cgmes, config(p), activatedPreProcessors(p), activatedPostProcessors(p), networkFactory).convert(conversionReportNode);
     }
 
+    @Override
+    public void update(Network network, ReadOnlyDataSource ds, Properties p, ReportNode reportNode) {
+        TripleStoreOptions tripleStoreOptions = new TripleStoreOptions();
+        tripleStoreOptions.setQueryCatalog("-update");
+        ReadOnlyDataSource alternativeDataSourceForBoundary = null;
+        CgmesModel cgmes = CgmesModelFactory.create(
+                ds,
+                alternativeDataSourceForBoundary,
+                TripleStoreFactory.DEFAULT_IMPLEMENTATION,
+                reportNode,
+                tripleStoreOptions);
+        Conversion conversion = new Conversion(cgmes, config(p));
+        conversion.update(network, reportNode);
+    }
+
     static class FilteredReadOnlyDataSource implements ReadOnlyDataSource {
         private final ReadOnlyDataSource ds;
         private final Predicate<String> filter;

cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesReports.java

@@ -17,6 +17,8 @@
  */
 public final class CgmesReports {
 
+    private static final String EQUIPMENT_ID = "equipmentId";
+
     private CgmesReports() {
     }
 
@@ -51,6 +53,14 @@ public static ReportNode convertingElementTypeReport(ReportNode reportNode, Stri
                 .add();
     }
 
+    public static ReportNode updatingElementTypeReport(ReportNode reportNode, String elementType) {
+        return reportNode.newReportNode()
+                .withMessageTemplate("updatingElementType", "Updating ${elementType}.")
+                .withUntypedValue("elementType", elementType)
+                .withSeverity(TypedValue.INFO_SEVERITY)
+                .add();
+    }
+
     public static ReportNode fixingDanglingLinesIssuesReport(ReportNode reportNode) {
         return reportNode.newReportNode()
                 .withMessageTemplate("fixingDanglingLinesIssues", "Fixing issues with dangling lines.")
@@ -84,17 +94,26 @@ public static void importedCgmesNetworkReport(ReportNode reportNode, String netw
     public static void badVoltageTargetValueRegulatingControlReport(ReportNode reportNode, String eqId, double targetValue) {
         reportNode.newReportNode()
                 .withMessageTemplate("badVoltageTargetValueRegulatingControl", "Equipment ${equipmentId} has a regulating control with bad target value for voltage: ${targetValue}.")
-                .withUntypedValue("equipmentId", eqId)
+                .withUntypedValue(EQUIPMENT_ID, eqId)
                 .withTypedValue("targetValue", targetValue, TypedValue.VOLTAGE)
                 .withSeverity(TypedValue.WARN_SEVERITY)
                 .add();
     }
 
+    public static void badTargetValueRegulatingControlReport(ReportNode reportNode, String eqId, double targetValue) {
+        reportNode.newReportNode()
+                .withMessageTemplate("badTargetValueRegulatingControl", "Equipment ${equipmentId} has a regulating control with bad target value: ${targetValue}.")
+                .withUntypedValue(EQUIPMENT_ID, eqId)
+                .withUntypedValue("targetValue", targetValue)
+                .withSeverity(TypedValue.WARN_SEVERITY)
+                .add();
+    }
+
     public static void badTargetDeadbandRegulatingControlReport(ReportNode reportNode, String eqId, double targetDeadband) {
         reportNode.newReportNode()
                 .withMessageTemplate("badTargetDeadbandRegulatingControl", "Equipment ${equipmentId} has a regulating control with bad target deadband: ${targetDeadband}.")
-                .withUntypedValue("equipmentId", eqId)
-                .withTypedValue("targetDeadband", targetDeadband, TypedValue.VOLTAGE)
+                .withUntypedValue(EQUIPMENT_ID, eqId)
+                .withUntypedValue("targetDeadband", targetDeadband)
                 .withSeverity(TypedValue.WARN_SEVERITY)
                 .add();
     }

cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/Context.java

@@ -64,6 +64,10 @@ public Context(CgmesModel cgmes, Config config, Network network, ReportNode repo
         reactiveCapabilityCurveData = new HashMap<>();
         powerTransformerRatioTapChangers = new HashMap<>();
         powerTransformerPhaseTapChangers = new HashMap<>();
+        cgmesTerminals = new HashMap<>();
+        regulatingControls = new HashMap<>();
+        operationalLimits = new HashMap<>();
+        generatingUnits = new HashMap<>();
     }
 
     public CgmesModel cgmes() {
@@ -100,6 +104,11 @@ public void convertedTerminal(String terminalId, Terminal t, int n, PowerFlow f)
         }
     }
 
+    public void convertedTerminalWithOnlyEq(String terminalId, Terminal t, int n) {
+        // Record the mapping between CGMES and IIDM terminals
+        terminalMapping().add(terminalId, t, n);
+    }
+
     private boolean setPQAllowed(Terminal t) {
         return t.getConnectable().getType() != IdentifiableType.BUSBAR_SECTION;
     }
@@ -205,6 +214,51 @@ public PropertyBags phaseTapChangerTable(String tableId) {
         return phaseTapChangerTables.get(tableId);
     }
 
+    public void loadCgmesTerminals() {
+        cgmes.terminals().forEach(p -> {
+            String id = p.getId(CgmesNames.TERMINAL);
+            cgmesTerminals.put(id, p);
+        });
+
+    }
+
+    public PropertyBag cgmesTerminal(String id) {
+        return cgmesTerminals.get(id);
+    }
+
+    public void loadRegulatingControls() {
+        cgmes.regulatingControls().forEach(p -> {
+            String id = p.getId(CgmesNames.REGULATING_CONTROL);
+            regulatingControls.put(id, p);
+        });
+    }
+
+    public PropertyBag regulatingControl(String id) {
+        return regulatingControls.get(id);
+    }
+
+    public void loadOperationalLimits() {
+        cgmes.operationalLimits().forEach(op -> {
+            String id = op.getId(CgmesNames.OPERATIONAL_LIMIT);
+            operationalLimits.put(id, op);
+        });
+    }
+
+    public PropertyBag operationalLimit(String id) {
+        return operationalLimits.get(id);
+    }
+
+    public void loadGeneratingUnits() {
+        cgmes.generatingUnits().forEach(generatingUnit -> {
+            String id = generatingUnit.getId(CgmesNames.GENERATING_UNIT);
+            generatingUnits.put(id, generatingUnit);
+        });
+    }
+
+    public PropertyBag generatingUnit(String id) {
+        return generatingUnits.get(id);
+    }
+
     // Handling issues found during conversion
 
     public ReportNode getReportNode() {
@@ -318,6 +372,10 @@ private static void logIssue(ConversionIssueCategory category, String what, Supp
     private final Map<String, PropertyBags> reactiveCapabilityCurveData;
     private final Map<String, PropertyBag> powerTransformerRatioTapChangers;
     private final Map<String, PropertyBag> powerTransformerPhaseTapChangers;
+    private final Map<String, PropertyBag> cgmesTerminals;
+    private final Map<String, PropertyBag> regulatingControls;
+    private final Map<String, PropertyBag> operationalLimits;
+    private final Map<String, PropertyBag> generatingUnits;
 
     private static final Logger LOG = LoggerFactory.getLogger(Context.class);
 }

cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/Conversion.java

@@ -37,6 +37,7 @@
 import java.util.stream.Stream;
 
 import static com.powsybl.cgmes.conversion.Conversion.Config.StateProfile.SSH;
+import static com.powsybl.cgmes.conversion.Update.*;
 import static java.util.stream.Collectors.groupingBy;
 
 /**
@@ -158,7 +159,14 @@ public Network convert(ReportNode reportNode) {
 
         // Create base network with metadata information
         Network network = createNetwork();
+        network.setMinimumAcceptableValidationLevel(ValidationLevel.EQUIPMENT);
+
+        // FIXME(Luma) we are not reusing conversion objects,
+        //  so it is safe to store the context as an attribute for later use in update
+        //  to do things right the context should have been created in the constructor,
+        //  together with the empty network
         Context context = createContext(network, reportNode);
+
         assignNetworkProperties(context);
         addMetadataModels(network, context);
         addCimCharacteristics(network);
@@ -253,7 +261,7 @@ public Network convert(ReportNode reportNode) {
         // Set voltages and angles
         context.pushReportNode(CgmesReports.settingVoltagesAndAnglesReport(reportNode));
         voltageAngles(nodes, context);
-        completeVoltagesAndAngles(network);
+        completeVoltagesAndAnglesLegacy(network);
         context.popReportNode();
 
         // Save/store data for debug or external validation
@@ -276,9 +284,64 @@ public Network convert(ReportNode reportNode) {
         }
 
         CgmesReports.importedCgmesNetworkReport(reportNode, network.getId());
+
+        // FIXME(Luma) in the first step, the Conversion object has used only info from EQ,
+        //  we call the update method on the same conversion object,
+        //  that has the context created during the convert (first) step
+        //  and all the data already loaded in the triplestore,
+        //  we only need to switch to a different set of queries
+        updateWithAllInputs(network, context, reportNode);
+
         return network;
     }
 
+    private void updateWithAllInputs(Network network, Context convertContext, ReportNode reportNode) {
+        // FIXME(Luma) Before switching to update we must invalidate all caches of the cgmes model
+        //  and change the query catalog to "update" mode
+        if (!sshIncludedInCgmesModel(this.cgmes)) {
+            return;
+        }
+        this.cgmes.invalidateCaches();
+        this.cgmes.setQueryCatalog("-update");
+        Context updateContext = createUpdateContext(network, reportNode);
+
+        // add processes to create new equipment using update data (ssh and sv data)
+
+        update(network, updateContext, reportNode);
+    }
+
+    private static boolean sshIncludedInCgmesModel(CgmesModel cgmes) {
+        return cgmes.version().contains("CIM14") || cgmes.fullModels().stream().anyMatch(fullModel -> fullModel.getId("profileList").contains("SteadyStateHypothesis"));
+    }
+
+    public void update(Network network, ReportNode reportNode) {
+        Objects.requireNonNull(network);
+        Objects.requireNonNull(reportNode);
+        Context updateContext = createUpdateContext(network, reportNode);
+        update(network, updateContext, reportNode);
+    }
+
+    private void update(Network network, Context updateContext, ReportNode reportNode) {
+        // FIXME(Luma) Inspect the contents of the loaded data
+        if (LOG.isDebugEnabled()) {
+            PropertyBags nts = cgmes.numObjectsByType();
+            LOG.debug("CGMES objects read for the update:");
+            nts.forEach(nt -> LOG.debug(String.format("  %5d %s", nt.asInt("numObjects"), nt.getLocal("Type"))));
+            nts.forEach(nt -> LOG.debug(cgmes.allObjectsOfType(nt.getLocal("Type")).tabulateLocals()));
+        }
+
+        updateLoads(network, cgmes, updateContext);
+        updateGenerators(network, cgmes, updateContext);
+        updateTransformers(network, updateContext);
+        updateStaticVarCompensators(network, cgmes, updateContext);
+        updateShuntCompensators(network, cgmes, updateContext);
+
+        completeVoltagesAndAngles(network);
+
+        network.runValidationChecks(false, reportNode);
+        network.setMinimumAcceptableValidationLevel(ValidationLevel.STEADY_STATE_HYPOTHESIS);
+    }
+
     /**
      * Retrieve the Collection of OperationalLimitGroups for identifiable that have flow limits
      * (branch, dangling line, 3w-transformer).
@@ -389,18 +452,21 @@ private Source isBoundaryBaseVoltage(String graph) {
         return graph.contains("EQ") && graph.contains("BD") ? Source.BOUNDARY : Source.IGM;
     }
 
-    private static void completeVoltagesAndAngles(Network network) {
-
-        // Voltage and angle in starBus as properties
-        network.getThreeWindingsTransformers()
-            .forEach(ThreeWindingsTransformerConversion::calculateVoltageAndAngleInStarBus);
+    private static void completeVoltagesAndAnglesLegacy(Network network) {
 
         // Voltage and angle in boundary buses
         network.getDanglingLineStream(DanglingLineFilter.UNPAIRED)
             .forEach(AbstractConductingEquipmentConversion::calculateVoltageAndAngleInBoundaryBus);
         network.getTieLines().forEach(tieLine -> AbstractConductingEquipmentConversion.calculateVoltageAndAngleInBoundaryBus(tieLine.getDanglingLine1(), tieLine.getDanglingLine2()));
     }
 
+    private static void completeVoltagesAndAngles(Network network) {
+
+        // Voltage and angle in starBus as properties
+        network.getThreeWindingsTransformers()
+                .forEach(ThreeWindingsTransformerConversion::calculateVoltageAndAngleInStarBus);
+    }
+
     private static void createControlArea(CgmesControlAreas cgmesControlAreas, PropertyBag ca) {
         String controlAreaId = ca.getId("ControlArea");
         cgmesControlAreas.newCgmesControlArea()
@@ -480,6 +546,17 @@ private Context createContext(Network network, ReportNode reportNode) {
         return context;
     }
 
+    private Context createUpdateContext(Network network, ReportNode reportNode) {
+        Context context = new Context(cgmes, config, network, reportNode);
+        context.loadCgmesTerminals();
+        context.loadRatioTapChangers();
+        context.loadPhaseTapChangers();
+        context.loadRegulatingControls();
+        context.loadOperationalLimits();
+        context.loadGeneratingUnits();
+        return context;
+    }
+
     private void assignNetworkProperties(Context context) {
         context.network().setProperty(NETWORK_PS_CGMES_MODEL_DETAIL,
                 context.nodeBreaker()
@@ -849,6 +926,13 @@ public enum StateProfile {
             SV
         }
 
+        public enum DefaultValue {
+            EQ,
+            DEFAULT,
+            EMPTY,
+            PREVIOUS
+        }
+
         public List<String> substationIdsExcludedFromMapping() {
             return Collections.emptyList();
         }
@@ -1043,6 +1127,14 @@ public boolean disconnectNetworkSideOfDanglingLinesIfBoundaryIsDisconnected() {
             return disconnectNetworkSideOfDanglingLinesIfBoundaryIsDisconnected;
         }
 
+        public boolean updateTerminalConnectionInNodeBreakerVoltageLevel() {
+            return UPDATE_TERMINAL_CONNECTION_IN_NODE_BREAKER_VOLTAGE_LEVEL;
+        }
+
+        public List<DefaultValue> updateDefaultValuesPriority() {
+            return updateDefaultValuesPriority;
+        }
+
         public boolean getCreateFictitiousVoltageLevelsForEveryNode() {
             return createFictitiousVoltageLevelsForEveryNode;
         }
@@ -1081,6 +1173,8 @@ public Config setCreateFictitiousVoltageLevelsForEveryNode(boolean b) {
 
         private double missingPermanentLimitPercentage = 100;
         private boolean createFictitiousVoltageLevelsForEveryNode = true;
+        private static final boolean UPDATE_TERMINAL_CONNECTION_IN_NODE_BREAKER_VOLTAGE_LEVEL = false;
+        private final List<DefaultValue> updateDefaultValuesPriority = List.of(DefaultValue.EQ, DefaultValue.DEFAULT, DefaultValue.EMPTY);
     }
 
     private final CgmesModel cgmes;