From 4e79c262225d4f011e3167553b01579127e13b4a Mon Sep 17 00:00:00 2001 From: Luma Date: Mon, 27 Jun 2022 09:51:59 +0200 Subject: [PATCH] CGMES import parameter for IIDM id source: mRID or rdfID (#2180) Signed-off-by: Luma --- .../powsybl/cgmes/conversion/CgmesImport.java | 21 +++++- .../test/SourceForIidmIdentifiersTest.java | 64 +++++++++++++++++++ .../cgmes/model/CgmesModelFactory.java | 29 ++++++--- .../powsybl/triplestore/api/TripleStore.java | 14 +++- .../triplestore/api/TripleStoreFactory.java | 33 +++++++++- .../api/TripleStoreFactoryService.java | 10 +++ .../triplestore/api/TripleStoreOptions.java | 31 +++++++++ .../rdf4j/TripleStoreFactoryServiceRDF4J.java | 8 ++- .../impl/rdf4j/TripleStoreRDF4J.java | 13 +++- 9 files changed, 205 insertions(+), 18 deletions(-) create mode 100644 cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/SourceForIidmIdentifiersTest.java create mode 100644 triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreOptions.java diff --git a/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java b/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java index bc138fb9d90..71f5a320bfd 100644 --- a/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java +++ b/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java @@ -25,6 +25,7 @@ import com.powsybl.iidm.parameters.ParameterDefaultValueConfig; import com.powsybl.iidm.parameters.ParameterType; import com.powsybl.triplestore.api.TripleStoreFactory; +import com.powsybl.triplestore.api.TripleStoreOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,7 +105,14 @@ public String getFormat() { @Override public Network importData(ReadOnlyDataSource ds, NetworkFactory networkFactory, Properties p) { - CgmesModel cgmes = CgmesModelFactory.create(ds, boundary(p), tripleStore(p)); + TripleStoreOptions options = new TripleStoreOptions(); + String sourceForIidmIds = Parameter.readString(getFormat(), p, SOURCE_FOR_IIDM_ID_PARAMETER, defaultValueConfig); + if (sourceForIidmIds.equalsIgnoreCase(SOURCE_FOR_IIDM_ID_MRID)) { + options.setRemoveInitialUnderscoreForIdentifiers(true); + } else if (sourceForIidmIds.equalsIgnoreCase(SOURCE_FOR_IIDM_ID_RDFID)) { + options.setRemoveInitialUnderscoreForIdentifiers(false); + } + CgmesModel cgmes = CgmesModelFactory.create(ds, boundary(p), tripleStore(p), options); return new Conversion(cgmes, config(ds, p), activatedPostProcessors(p), networkFactory).convert(); } @@ -270,9 +278,13 @@ private void copyStream(ReadOnlyDataSource from, DataSource to, String fromName, public static final String POST_PROCESSORS = "iidm.import.cgmes.post-processors"; public static final String POWSYBL_TRIPLESTORE = "iidm.import.cgmes.powsybl-triplestore"; public static final String PROFILE_FOR_INITIAL_VALUES_SHUNT_SECTIONS_TAP_POSITIONS = "iidm.import.cgmes.profile-for-initial-values-shunt-sections-tap-positions"; + public static final String SOURCE_FOR_IIDM_ID = "iidm.import.cgmes.source-for-iidm-id"; public static final String STORE_CGMES_MODEL_AS_NETWORK_EXTENSION = "iidm.import.cgmes.store-cgmes-model-as-network-extension"; public static final String STORE_CGMES_CONVERSION_CONTEXT_AS_NETWORK_EXTENSION = "iidm.import.cgmes.store-cgmes-conversion-context-as-network-extension"; + public static final String SOURCE_FOR_IIDM_ID_MRID = "mRID"; + public static final String SOURCE_FOR_IIDM_ID_RDFID = "rdfID"; + private static final Parameter ALLOW_UNSUPPORTED_TAP_CHANGERS_PARAMETER = new Parameter( ALLOW_UNSUPPORTED_TAP_CHANGERS, ParameterType.BOOLEAN, @@ -350,6 +362,12 @@ private void copyStream(ReadOnlyDataSource from, DataSource to, String fromName, "Store the initial CGMES model as a network extension", Boolean.TRUE) .addAdditionalNames("storeCgmesModelAsNetworkExtension"); + private static final Parameter SOURCE_FOR_IIDM_ID_PARAMETER = new Parameter( + SOURCE_FOR_IIDM_ID, + ParameterType.STRING, + "Source for IIDM identifiers", + SOURCE_FOR_IIDM_ID_MRID, + List.of(SOURCE_FOR_IIDM_ID_MRID, SOURCE_FOR_IIDM_ID_RDFID)); private static final List STATIC_PARAMETERS = ImmutableList.of( ALLOW_UNSUPPORTED_TAP_CHANGERS_PARAMETER, @@ -364,6 +382,7 @@ private void copyStream(ReadOnlyDataSource from, DataSource to, String fromName, POST_PROCESSORS_PARAMETER, POWSYBL_TRIPLESTORE_PARAMETER, PROFILE_FOR_INITIAL_VALUES_SHUNT_SECTIONS_TAP_POSITIONS_PARAMETER, + SOURCE_FOR_IIDM_ID_PARAMETER, STORE_CGMES_CONVERSION_CONTEXT_AS_NETWORK_EXTENSION_PARAMETER, STORE_CGMES_MODEL_AS_NETWORK_EXTENSION_PARAMETER); diff --git a/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/SourceForIidmIdentifiersTest.java b/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/SourceForIidmIdentifiersTest.java new file mode 100644 index 00000000000..c064bf896bc --- /dev/null +++ b/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/SourceForIidmIdentifiersTest.java @@ -0,0 +1,64 @@ +/** + * Copyright (c) 2022, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package com.powsybl.cgmes.conversion.test; + +import com.powsybl.cgmes.conformity.CgmesConformity1Catalog; +import com.powsybl.cgmes.conversion.CgmesImport; +import com.powsybl.iidm.import_.Importers; +import com.powsybl.iidm.network.Network; +import com.powsybl.triplestore.api.TripleStore; +import com.powsybl.triplestore.api.TripleStoreFactory; +import com.powsybl.triplestore.api.TripleStoreOptions; +import org.junit.Test; + +import java.util.Properties; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Luma Zamarreño + */ +public class SourceForIidmIdentifiersTest { + + @Test + public void microGridMasterResourceIdsExplicit() { + Properties importParams = new Properties(); + importParams.put(CgmesImport.SOURCE_FOR_IIDM_ID, CgmesImport.SOURCE_FOR_IIDM_ID_MRID); + Network network = Importers.importData("CGMES", CgmesConformity1Catalog.microGridBaseCaseBE().dataSource(), importParams); + network.getIdentifiables().forEach(idf -> assertFalse(idf.getId().startsWith("_"))); + } + + @Test + public void microGridMasterResourceIdsDefault() { + Network network = Importers.importData("CGMES", CgmesConformity1Catalog.microGridBaseCaseBE().dataSource(), null); + network.getIdentifiables().forEach(idf -> assertFalse(idf.getId().startsWith("_"))); + } + + @Test + public void microGridRDFIds() { + Properties importParams = new Properties(); + importParams.put(CgmesImport.SOURCE_FOR_IIDM_ID, CgmesImport.SOURCE_FOR_IIDM_ID_RDFID); + Network network = Importers.importData("CGMES", CgmesConformity1Catalog.microGridBaseCaseBE().dataSource(), importParams); + network.getIdentifiables().forEach(idf -> assertTrue(idf.getId().startsWith("_") || idf.getId().startsWith("urn:uuid:"))); + } + + @Test + public void tipleStoreOptions() { + TripleStoreOptions options0 = new TripleStoreOptions(); + assertTrue(options0.isRemoveInitialUnderscoreForIdentifiers()); + TripleStoreOptions options1 = new TripleStoreOptions(true); + assertTrue(options1.isRemoveInitialUnderscoreForIdentifiers()); + TripleStoreOptions options2 = new TripleStoreOptions(false); + assertFalse(options2.isRemoveInitialUnderscoreForIdentifiers()); + + TripleStore ts = TripleStoreFactory.create(options2); + assertFalse(ts.getOptions().isRemoveInitialUnderscoreForIdentifiers()); + } + +} diff --git a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesModelFactory.java b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesModelFactory.java index 9e46292e4bb..049685ca022 100644 --- a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesModelFactory.java +++ b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesModelFactory.java @@ -7,15 +7,12 @@ package com.powsybl.cgmes.model; -import java.util.Objects; - import com.powsybl.cgmes.model.triplestore.CgmesModelTripleStore; import com.powsybl.commons.PowsyblException; import com.powsybl.commons.datasource.ReadOnlyDataSource; -import com.powsybl.triplestore.api.PropertyBag; -import com.powsybl.triplestore.api.PropertyBags; -import com.powsybl.triplestore.api.TripleStore; -import com.powsybl.triplestore.api.TripleStoreFactory; +import com.powsybl.triplestore.api.*; + +import java.util.Objects; /** * @author Luma Zamarreño @@ -44,14 +41,28 @@ public static CgmesModel create( Objects.requireNonNull(mainDataSource); Objects.requireNonNull(implementation); - CgmesModel cgmes = createImplementation(implementation, mainDataSource); + CgmesModel cgmes = createImplementation(implementation, new TripleStoreOptions(), mainDataSource); + cgmes.read(mainDataSource, alternativeDataSourceForBoundary); + return cgmes; + } + + public static CgmesModel create( + ReadOnlyDataSource mainDataSource, + ReadOnlyDataSource alternativeDataSourceForBoundary, + String implementation, + TripleStoreOptions tripleStoreOptions) { + Objects.requireNonNull(mainDataSource); + Objects.requireNonNull(implementation); + Objects.requireNonNull(tripleStoreOptions); + + CgmesModel cgmes = createImplementation(implementation, tripleStoreOptions, mainDataSource); cgmes.read(mainDataSource, alternativeDataSourceForBoundary); return cgmes; } - private static CgmesModel createImplementation(String implementation, ReadOnlyDataSource ds) { + private static CgmesModel createImplementation(String implementation, TripleStoreOptions tripleStoreOptions, ReadOnlyDataSource ds) { // Only triple store implementations are available - TripleStore tripleStore = TripleStoreFactory.create(implementation); + TripleStore tripleStore = TripleStoreFactory.create(implementation, tripleStoreOptions); String cimNamespace = new CgmesOnDataSource(ds).cimNamespace(); return new CgmesModelTripleStore(cimNamespace, tripleStore); } diff --git a/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStore.java b/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStore.java index 65ad456ca3c..748644cd703 100644 --- a/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStore.java +++ b/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStore.java @@ -7,14 +7,14 @@ package com.powsybl.triplestore.api; +import com.powsybl.commons.datasource.DataSource; + import java.io.InputStream; import java.io.PrintStream; import java.util.List; import java.util.Set; import java.util.function.Consumer; -import com.powsybl.commons.datasource.DataSource; - /** * A Triplestore database. * A Triplestore database is a database for the storage and retrieval of triples. @@ -25,6 +25,14 @@ */ public interface TripleStore { + /** + * Obtain the options that have been used to configure this Triplestore + * @return options Triplestore configuration options + */ + default TripleStoreOptions getOptions() { + return null; + } + /** * Read statements from an input stream and store them in the Triplestore under the given context name. * @@ -131,7 +139,7 @@ default void write(DataSource ds, String contextName) { /** * Perform a SPARQL update on the Triplestore. * - * @param query the text of the query, written in SPARQL Update language + * @param queryText the text of the query, written in SPARQL Update language */ void update(String queryText); diff --git a/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreFactory.java b/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreFactory.java index 587c0ff3514..e0f57d718f1 100644 --- a/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreFactory.java +++ b/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreFactory.java @@ -7,14 +7,14 @@ package com.powsybl.triplestore.api; +import com.powsybl.commons.PowsyblException; +import com.powsybl.commons.util.ServiceLoaderCache; + import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import com.powsybl.commons.PowsyblException; -import com.powsybl.commons.util.ServiceLoaderCache; - /** * Factory for the creation of Triplestore databases. * It relies on named factory services to create instances. @@ -37,6 +37,16 @@ public static TripleStore create() { return create(DEFAULT_IMPLEMENTATION); } + /** + * Create a Triplestore database using the default implementation and given options. + * + * @param options Triplestore configuration options + * @return a Triplestore based on the default implementation + */ + public static TripleStore create(TripleStoreOptions options) { + return create(DEFAULT_IMPLEMENTATION, options); + } + /** * Crate a Triplestore that is a copy of the given Triplestore. * Copied Triplestore will be based on the same implementation of the source Triplestore. @@ -71,6 +81,23 @@ public static TripleStore create(String impl) { throw new PowsyblException("No implementation available for triple store " + impl); } + /** + * Create a Triplestore database using the given implementation and options. + * + * @param impl the name of the Triplestore implementation that must be used + * @param options for Triplestore configuration + * @return a Triplestore based on the given implementation + */ + public static TripleStore create(String impl, TripleStoreOptions options) { + Objects.requireNonNull(impl); + for (TripleStoreFactoryService ts : LOADER.getServices()) { + if (ts.getImplementationName().equals(impl)) { + return ts.create(options); + } + } + throw new PowsyblException("No implementation available for triple store " + impl); + } + /** * List all Triplestore implementations available. * diff --git a/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreFactoryService.java b/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreFactoryService.java index ef699a0360b..1a1df5c4ddf 100644 --- a/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreFactoryService.java +++ b/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreFactoryService.java @@ -21,6 +21,16 @@ public interface TripleStoreFactoryService { */ TripleStore create(); + /** + * Create an instance of a Triplestore with given options. + * + * @param options that configure the Triplestore behaviour + * @return Triplestore instance + */ + default TripleStore create(TripleStoreOptions options) { + return create(); + } + /** * Create an instance of a Triplestore that is a copy of a given Triplestore. * diff --git a/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreOptions.java b/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreOptions.java new file mode 100644 index 00000000000..3b0529119f3 --- /dev/null +++ b/triple-store/triple-store-api/src/main/java/com/powsybl/triplestore/api/TripleStoreOptions.java @@ -0,0 +1,31 @@ +/** + * Copyright (c) 2022, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.triplestore.api; + +/** + * @author Luma Zamarreño + */ +public class TripleStoreOptions { + + private boolean removeInitialUnderscoreForIdentifiers = true; + + public TripleStoreOptions() { + } + + public TripleStoreOptions(boolean removeInitialUnderscoreForIdentifiers) { + this.removeInitialUnderscoreForIdentifiers = removeInitialUnderscoreForIdentifiers; + } + + public TripleStoreOptions setRemoveInitialUnderscoreForIdentifiers(boolean removeInitialUnderscoreForIdentifiers) { + this.removeInitialUnderscoreForIdentifiers = removeInitialUnderscoreForIdentifiers; + return this; + } + + public boolean isRemoveInitialUnderscoreForIdentifiers() { + return removeInitialUnderscoreForIdentifiers; + } +} diff --git a/triple-store/triple-store-impl-rdf4j/src/main/java/com/powsybl/triplestore/impl/rdf4j/TripleStoreFactoryServiceRDF4J.java b/triple-store/triple-store-impl-rdf4j/src/main/java/com/powsybl/triplestore/impl/rdf4j/TripleStoreFactoryServiceRDF4J.java index 949c42e9311..9424292f476 100644 --- a/triple-store/triple-store-impl-rdf4j/src/main/java/com/powsybl/triplestore/impl/rdf4j/TripleStoreFactoryServiceRDF4J.java +++ b/triple-store/triple-store-impl-rdf4j/src/main/java/com/powsybl/triplestore/impl/rdf4j/TripleStoreFactoryServiceRDF4J.java @@ -10,6 +10,7 @@ import com.google.auto.service.AutoService; import com.powsybl.triplestore.api.TripleStore; import com.powsybl.triplestore.api.TripleStoreFactoryService; +import com.powsybl.triplestore.api.TripleStoreOptions; /** * @author Luma Zamarreño @@ -22,9 +23,14 @@ public TripleStore create() { return new TripleStoreRDF4J(); } + @Override + public TripleStore create(TripleStoreOptions options) { + return new TripleStoreRDF4J(options); + } + @Override public TripleStore copy(TripleStore source) { - TripleStore ts = new TripleStoreRDF4J(); + TripleStore ts = new TripleStoreRDF4J(source.getOptions()); ts.add(source); return ts; } diff --git a/triple-store/triple-store-impl-rdf4j/src/main/java/com/powsybl/triplestore/impl/rdf4j/TripleStoreRDF4J.java b/triple-store/triple-store-impl-rdf4j/src/main/java/com/powsybl/triplestore/impl/rdf4j/TripleStoreRDF4J.java index 1fcdae85e9c..4b97b73022a 100644 --- a/triple-store/triple-store-impl-rdf4j/src/main/java/com/powsybl/triplestore/impl/rdf4j/TripleStoreRDF4J.java +++ b/triple-store/triple-store-impl-rdf4j/src/main/java/com/powsybl/triplestore/impl/rdf4j/TripleStoreRDF4J.java @@ -46,10 +46,20 @@ public class TripleStoreRDF4J extends AbstractPowsyblTripleStore { static final String NAME = "rdf4j"; public TripleStoreRDF4J() { + this(new TripleStoreOptions()); + } + + public TripleStoreRDF4J(TripleStoreOptions options) { + this.options = options; repo = new SailRepository(new MemoryStore()); repo.init(); } + @Override + public TripleStoreOptions getOptions() { + return options; + } + @Override public String getImplementationName() { return NAME; @@ -183,7 +193,7 @@ public PropertyBags query(String query) { List names = r.getBindingNames(); while (r.hasNext()) { BindingSet s = r.next(); - PropertyBag result = new PropertyBag(names); + PropertyBag result = new PropertyBag(names, options.isRemoveInitialUnderscoreForIdentifiers()); names.forEach(name -> { if (s.hasBinding(name)) { @@ -417,6 +427,7 @@ public List getNamespaces() { private final Repository repo; private boolean writeBySubject = true; + private final TripleStoreOptions options; private static final boolean EXPLAIN_QUERIES = false;