diff --git a/minerva-converter/src/main/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLExport.java b/minerva-converter/src/main/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLExport.java index 3f58ec37..71bb8dc3 100644 --- a/minerva-converter/src/main/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLExport.java +++ b/minerva-converter/src/main/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLExport.java @@ -12,8 +12,8 @@ import org.apache.jena.rdf.model.Resource; import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.binding.Binding; +import org.apache.jena.sparql.engine.binding.BindingBuilder; import org.apache.jena.sparql.engine.binding.BindingFactory; -import org.apache.jena.sparql.engine.binding.BindingMap; import org.apache.jena.vocabulary.RDF; import org.apache.log4j.Logger; import org.geneontology.minerva.curie.CurieHandler; @@ -288,11 +288,11 @@ private Map> evidencesForFacts(Set facts, Mode @SafeVarargs private final Binding createBinding(Pair... bindings) { - BindingMap map = BindingFactory.create(); + BindingBuilder builder = BindingFactory.builder(); for (Pair binding : bindings) { - map.add(binding.getLeft(), binding.getRight()); + builder.add(binding.getLeft(), binding.getRight()); } - return map; + return builder.build(); } private Set possibleExtensions(Set basicAnnotations, Model model) { diff --git a/minerva-converter/src/test/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLTest.java b/minerva-converter/src/test/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLTest.java index d0756d0d..b0d472d9 100644 --- a/minerva-converter/src/test/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLTest.java +++ b/minerva-converter/src/test/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLTest.java @@ -5,7 +5,7 @@ import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.reasoner.rulesys.Rule; -import org.apache.jena.system.JenaSystem; +import org.apache.jena.sys.JenaSystem; import org.geneontology.jena.OWLtoRules; import org.geneontology.minerva.curie.DefaultCurieHandler; import org.geneontology.rules.engine.RuleEngine; diff --git a/minerva-core/pom.xml b/minerva-core/pom.xml index 0725595a..4815a330 100644 --- a/minerva-core/pom.xml +++ b/minerva-core/pom.xml @@ -54,6 +54,10 @@ apache-jena-libs pom + + org.apache.jena + jena-commonsrdf + net.sf.trove4j trove4j @@ -73,6 +77,10 @@ org.apache.jena * + + org.apache.commons + commons-rdf-jena + @@ -92,6 +100,11 @@ commons-math3 3.6.1 + + commons-codec + commons-codec + 1.15 + org.obolibrary.robot robot-core diff --git a/minerva-core/src/main/java/org/geneontology/minerva/BlazegraphMolecularModelManager.java b/minerva-core/src/main/java/org/geneontology/minerva/BlazegraphMolecularModelManager.java index 7d062666..7c65c858 100644 --- a/minerva-core/src/main/java/org/geneontology/minerva/BlazegraphMolecularModelManager.java +++ b/minerva-core/src/main/java/org/geneontology/minerva/BlazegraphMolecularModelManager.java @@ -7,11 +7,18 @@ import info.aduna.iteration.Iterations; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.jena.graph.Graph; +import org.apache.jena.shex.ShapeMap; +import org.apache.jena.shex.ShexReport; +import org.apache.jena.shex.ShexSchema; +import org.apache.jena.shex.ShexValidator; +import org.apache.jena.sys.JenaSystem; import org.apache.log4j.Logger; import org.geneontology.minerva.MolecularModelManager.UnknownIdentifierException; import org.geneontology.minerva.curie.CurieHandler; import org.geneontology.minerva.util.AnnotationShorthand; import org.geneontology.minerva.util.BlazegraphMutationCounter; +import org.geneontology.minerva.util.JenaOwlTool; import org.geneontology.minerva.util.ReverseChangeGenerator; import org.openrdf.model.*; import org.openrdf.model.impl.URIImpl; @@ -22,6 +29,7 @@ import org.openrdf.repository.RepositoryException; import org.openrdf.repository.RepositoryResult; import org.openrdf.rio.*; +import org.openrdf.rio.helpers.BasicWriterSettings; import org.openrdf.rio.helpers.RDFHandlerBase; import org.openrdf.rio.helpers.StatementCollector; import org.semanticweb.owlapi.apibinding.OWLManager; @@ -67,6 +75,7 @@ public class BlazegraphMolecularModelManager extends CoreMolecularMode public BlazegraphMolecularModelManager(OWLOntology tbox, CurieHandler curieHandler, String modelIdPrefix, @Nonnull String pathToJournal, String pathToExportFolder, String pathToOntologyJournal, boolean downloadOntologyJournal) throws OWLOntologyCreationException, IOException { super(tbox, pathToOntologyJournal, downloadOntologyJournal); + JenaSystem.init(); if (curieHandler == null) { LOG.error("curie handler required for blazegraph model manager startup "); System.exit(-1); @@ -195,23 +204,14 @@ public void saveModel(ModelContainer m) final OWLOntologyManager manager = ont.getOWLOntologyManager(); List changes = preSaveFileHandler(ont); synchronized (ont) { - try { - this.writeModelToDatabase(ont, modelId); - // reset modified flag for abox after successful save - m.setAboxModified(false); - // dump stored model to export file - if (this.pathToExportFolder != null) { - File folder = new File(this.pathToExportFolder); - dumpStoredModel(modelId, folder); - } - } finally { - if (changes != null) { - List invertedChanges = ReverseChangeGenerator - .invertChanges(changes); - if (invertedChanges != null && !invertedChanges.isEmpty()) { - manager.applyChanges(invertedChanges); - } - } + manager.applyChanges(changes); + this.writeModelToDatabase(ont, modelId); + // reset modified flag for abox after successful save + m.setAboxModified(false); + // dump stored model to export file + if (this.pathToExportFolder != null) { + File folder = new File(this.pathToExportFolder); + dumpStoredModel(modelId, folder); } } } @@ -241,14 +241,10 @@ private void writeModelToDatabase(OWLOntology model, IRI modelId) throws Reposit } private List preSaveFileHandler(OWLOntology model) throws UnknownIdentifierException { - List allChanges = null; + List allChanges = new ArrayList<>(); for (PreFileSaveHandler handler : preFileSaveHandlers) { List changes = handler.handle(model); if (changes != null && !changes.isEmpty()) { - if (allChanges == null) { - allChanges = new ArrayList( - changes.size()); - } allChanges.addAll(changes); } } @@ -994,4 +990,29 @@ public int addTaxonToDatabaseWithSparql(IRI model_iri, IRI taxon_iri) throws Rep return changes; } + public static class ShexValidatingPreSaveFileHandler implements PreFileSaveHandler { + + private final ShexSchema schema; + private final ShapeMap shapeMap; + private final OWLAnnotationProperty conformanceProperty; + public ShexValidatingPreSaveFileHandler(ShexSchema schema, ShapeMap map, String propertyIRI) { + this.schema = schema; + this.shapeMap = map; + this.conformanceProperty = OWLManager.getOWLDataFactory().getOWLAnnotationProperty(IRI.create(propertyIRI)); + } + + @Override + public List handle(OWLOntology modelOnt) throws UnknownIdentifierException { + OWLDataFactory factory = OWLManager.getOWLDataFactory(); + Graph graph = JenaOwlTool.getJenaModel(modelOnt).getGraph(); + ShexReport report = ShexValidator.get().validate(graph, schema, shapeMap); + boolean conforms = report.conforms(); + System.out.println("CONFORMS? " + conforms); + System.out.println(factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(conforms))); + OWLOntologyChange remove = new RemoveOntologyAnnotation(modelOnt, factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(!conforms))); + OWLOntologyChange add = new AddOntologyAnnotation(modelOnt, factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(conforms))); + return Arrays.asList(remove, add); + } + } + } diff --git a/minerva-core/src/main/java/org/geneontology/minerva/util/AnnotationShorthand.java b/minerva-core/src/main/java/org/geneontology/minerva/util/AnnotationShorthand.java index 7667fa8a..e8e23c25 100644 --- a/minerva-core/src/main/java/org/geneontology/minerva/util/AnnotationShorthand.java +++ b/minerva-core/src/main/java/org/geneontology/minerva/util/AnnotationShorthand.java @@ -30,7 +30,8 @@ public enum AnnotationShorthand { deprecated(OWLRDFVocabulary.OWL_DEPRECATED.getIRI()), // model annotation to indicate deprecated models templatestate(IRI.create("http://geneontology.org/lego/templatestate"), "template"), // designate a model as a template modelstate(IRI.create("http://geneontology.org/lego/modelstate"), "state"), - wasDerivedFrom(IRI.create("http://www.w3.org/ns/prov#wasDerivedFrom"), "wasDerivedFrom"); + wasDerivedFrom(IRI.create("http://www.w3.org/ns/prov#wasDerivedFrom"), "wasDerivedFrom"), + conformsToGPAD(IRI.create("http://geneontology.org/lego/conforms_to_gpad"), "conforms-to-gpad"); private final IRI annotationProperty; diff --git a/minerva-core/src/main/java/org/geneontology/minerva/validation/ShexValidator.java b/minerva-core/src/main/java/org/geneontology/minerva/validation/ShexValidator.java index 404cf49c..5c2d4adc 100644 --- a/minerva-core/src/main/java/org/geneontology/minerva/validation/ShexValidator.java +++ b/minerva-core/src/main/java/org/geneontology/minerva/validation/ShexValidator.java @@ -14,13 +14,15 @@ import fr.inria.lille.shexjava.validation.Status; import fr.inria.lille.shexjava.validation.Typing; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.rdf.api.RDF; -import org.apache.commons.rdf.api.RDFTerm; -import org.apache.commons.rdf.jena.JenaGraph; -import org.apache.commons.rdf.jena.JenaRDF; +import org.apache.commons.rdf.api.*; import org.apache.commons.rdf.simple.SimpleRDF; +import org.apache.jena.commonsrdf.JenaCommonsRDF; +import org.apache.jena.commonsrdf.JenaRDF; +import org.apache.jena.commonsrdf.impl.JenaGraph; import org.apache.jena.query.*; import org.apache.jena.rdf.model.*; +import org.apache.jena.rdf.model.Literal; +import org.apache.jena.sys.JenaSystem; import org.apache.log4j.Logger; import org.geneontology.minerva.BlazegraphOntologyManager; import org.geneontology.minerva.curie.CurieHandler; @@ -34,7 +36,9 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.*; +import java.util.stream.Stream; /** * @author bgood @@ -67,6 +71,7 @@ public ShexValidator(File shex_schema_file, File shex_map_file, BlazegraphOntolo } public void init(File shex_schema_file, File shex_map_file, BlazegraphOntologyManager go_lego, CurieHandler curieHandler_) throws Exception { + JenaSystem.init(); schema = GenParser.parseSchema(shex_schema_file.toPath()); GoQueryMap = makeGoQueryMap(shex_map_file.getAbsolutePath()); //tbox_reasoner = tbox_reasoner_; @@ -115,9 +120,8 @@ public static Map makeGoQueryMap(String shapemap_file) throws IO public ShexValidationReport runShapeMapValidation(Model test_model) { boolean explain = true; ShexValidationReport r = new ShexValidationReport(); - JenaRDF jr = new JenaRDF(); //this shex implementation likes to use the commons JenaRDF interface, nothing exciting here - JenaGraph shexy_graph = jr.asGraph(test_model); + Graph shexy_graph = JenaCommonsRDF.fromJena(test_model.getGraph()); boolean all_good = true; try { Typing all_typed = runRefineWithTimeout(shexy_graph); @@ -251,7 +255,7 @@ public ShexValidationReport runShapeMapValidationWithRecursiveSingleNodeValidati ShexValidationReport r = new ShexValidationReport(); JenaRDF jr = new JenaRDF(); //this shex implementation likes to use the commons JenaRDF interface, nothing exciting here - JenaGraph shexy_graph = jr.asGraph(test_model); + Graph shexy_graph = JenaCommonsRDF.fromJena(test_model.getGraph()); //recursive only checks the focus node against the chosen shape. RecursiveValidationWithMemorization shex_model_validator = new RecursiveValidationWithMemorization(schema, shexy_graph); //for each shape in the query map (e.g. MF, BP, CC, etc.) @@ -391,7 +395,7 @@ private Map> getShapesToTestForEachResource(Model test_mod return node_s_shapes; } - private Typing runRefineWithTimeout(JenaGraph shexy_graph) { + private Typing runRefineWithTimeout(Graph shexy_graph) { final ExecutorService service = Executors.newSingleThreadExecutor(); try { final Future f = service.submit(() -> { diff --git a/minerva-core/src/test/resources/validation/should_pass/Test034-cell_differentiation_results_in_acquisition_of_features_of_seam cell.ttl b/minerva-core/src/test/resources/validation/should_pass/Test034-cell_differentiation_results_in_acquisition_of_features_of_seam_cell.ttl similarity index 100% rename from minerva-core/src/test/resources/validation/should_pass/Test034-cell_differentiation_results_in_acquisition_of_features_of_seam cell.ttl rename to minerva-core/src/test/resources/validation/should_pass/Test034-cell_differentiation_results_in_acquisition_of_features_of_seam_cell.ttl diff --git a/minerva-core/src/test/resources/validation/should_pass/Test035-cell_fate_commitment_results_in_commitment_to_body_wall_muscle cell_from_MS_lineage.ttl b/minerva-core/src/test/resources/validation/should_pass/Test035-cell_fate_commitment_results_in_commitment_to_body_wall_muscle_cell_from_MS_lineage.ttl similarity index 100% rename from minerva-core/src/test/resources/validation/should_pass/Test035-cell_fate_commitment_results_in_commitment_to_body_wall_muscle cell_from_MS_lineage.ttl rename to minerva-core/src/test/resources/validation/should_pass/Test035-cell_fate_commitment_results_in_commitment_to_body_wall_muscle_cell_from_MS_lineage.ttl diff --git a/minerva-server/src/main/java/org/geneontology/minerva/server/StartUpTool.java b/minerva-server/src/main/java/org/geneontology/minerva/server/StartUpTool.java index afa8a6a3..c3e4c388 100644 --- a/minerva-server/src/main/java/org/geneontology/minerva/server/StartUpTool.java +++ b/minerva-server/src/main/java/org/geneontology/minerva/server/StartUpTool.java @@ -1,5 +1,8 @@ package org.geneontology.minerva.server; +import org.apache.jena.shex.ShapeMap; +import org.apache.jena.shex.ShexSchema; +import org.apache.jena.shex.parser.ShExC; import org.apache.log4j.Logger; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; @@ -7,6 +10,7 @@ import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.geneontology.minerva.BlazegraphMolecularModelManager; import org.geneontology.minerva.MinervaOWLGraphWrapper; import org.geneontology.minerva.ModelReaderHelper; import org.geneontology.minerva.UndoAwareMolecularModelManager; @@ -93,6 +97,10 @@ public static class MinervaStartUpConfig { public MinervaShexValidator shex; public String pathToOntologyJournal; + public String pathToGPADSheX; + + public String pathToGPADShapeMap; + } public static void main(String[] args) throws Exception { @@ -172,6 +180,10 @@ else if (opts.nextEq("--golr-labels")) { conf.sparqlEndpointTimeout = Integer.parseInt(opts.nextOpt()); } else if (opts.nextEq("--ontojournal")) { conf.pathToOntologyJournal = opts.nextOpt(); + } else if (opts.nextEq("--gpad-shex")) { + conf.pathToGPADSheX = opts.nextOpt(); + } else if (opts.nextEq("--gpad-shape-map")) { + conf.pathToGPADShapeMap = opts.nextOpt(); } else { break; } @@ -340,6 +352,11 @@ public static Server startUp(final MinervaStartUpConfig conf) conf.curieHandler, conf.modelIdPrefix, conf.journalFile, conf.exportFolder, conf.pathToOntologyJournal, true); // set pre and post file handlers models.addPostLoadOntologyFilter(ModelReaderHelper.INSTANCE); + if (conf.pathToGPADSheX != null && conf.pathToGPADShapeMap != null) { + ShexSchema schema = ShExC.parse(conf.pathToGPADSheX); + ShapeMap shapeMap = ShExC.parseShapeMap(conf.pathToGPADShapeMap); + models.addPreFileSaveHandler(new BlazegraphMolecularModelManager.ShexValidatingPreSaveFileHandler(schema, shapeMap, "http://geneontology.org/lego/conforms_to_gpad")); + } // conf.shex.tbox_reasoner = models.getTbox_reasoner(); conf.shex.setGo_lego_repo(models.getGolego_repo()); conf.shex.curieHandler = conf.curieHandler; diff --git a/minerva-server/src/main/java/org/geneontology/minerva/server/handler/ModelSearchHandler.java b/minerva-server/src/main/java/org/geneontology/minerva/server/handler/ModelSearchHandler.java index 615e0da2..b578a726 100644 --- a/minerva-server/src/main/java/org/geneontology/minerva/server/handler/ModelSearchHandler.java +++ b/minerva-server/src/main/java/org/geneontology/minerva/server/handler/ModelSearchHandler.java @@ -10,6 +10,7 @@ import org.openrdf.repository.RepositoryException; import org.semanticweb.owlapi.model.IRI; +import javax.annotation.Nullable; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import java.io.IOException; @@ -89,6 +90,8 @@ static public class ModelMeta { private String date; private String title; private String state; + @SerializedName("conforms-to-gpad") + private Boolean conformsToGPAD; private Set contributors; private Set groups; private HashMap> query_match; @@ -96,11 +99,12 @@ static public class ModelMeta { @SerializedName("modified-p") private boolean modified; - public ModelMeta(String id, String date, String title, String state, Set contributors, Set groups, boolean modified) { + public ModelMeta(String id, String date, String title, String state, @Nullable Boolean conformsToGPAD, Set contributors, Set groups, boolean modified) { this.id = id; this.date = date; this.title = title; this.state = state; + this.conformsToGPAD = conformsToGPAD; this.contributors = contributors; this.groups = groups; this.modified = modified; @@ -150,6 +154,15 @@ public void setState(String state) { this.state = state; } + @Nullable + public Boolean getConformsToGPAD() { + return this.conformsToGPAD; + } + + public void setConformsToGPAD(Boolean conforms) { + this.conformsToGPAD = conforms; + } + public Set getContributors() { return contributors; } @@ -454,7 +467,7 @@ public ModelSearchResult search(Set taxa, String group_by_constraint = "GROUP BY ?id"; //default return block //TODO investigate need to add DISTINCT to GROUP_CONCAT here - String return_block = "?id (MIN(?date) AS ?mindate) (MIN(?title) AS ?mintitle) (MIN(?state) AS ?minstate) (GROUP_CONCAT(DISTINCT ?contributor;separator=\";\") AS ?contributors) (GROUP_CONCAT(DISTINCT ?group;separator=\";\") AS ?groups)"; + String return_block = "?id (MIN(?date) AS ?mindate) (MIN(?title) AS ?mintitle) (MIN(?state) AS ?minstate) (MIN(?conformsToGPAD) AS ?minConformsToGPAD) (GROUP_CONCAT(DISTINCT ?contributor;separator=\";\") AS ?contributors) (GROUP_CONCAT(DISTINCT ?group;separator=\";\") AS ?groups)"; if (count != null) { return_block = "(count(distinct ?id) as ?count)"; limit_constraint = ""; @@ -525,6 +538,12 @@ public ModelSearchResult search(Set taxa, if (state_binding != null) { state = state_binding.getValue().stringValue(); } + Binding conformanceBinding = bs.getBinding("minConformsToGPAD"); + Boolean conformance = null; + if (conformanceBinding != null) { + System.out.println("CONFORMANCE: " + conformanceBinding.getValue().stringValue()); + conformance = Boolean.valueOf(conformanceBinding.getValue().stringValue()); + } Binding group_binding = bs.getBinding("groups"); String groups_ = ""; if (group_binding != null) { @@ -539,7 +558,7 @@ public ModelSearchResult search(Set taxa, if (mm == null) { //look up model in in-memory cache to check edit state boolean is_modified = m3.isModelModified(model_iri); - mm = new ModelMeta(model_curie, date, title, state, contributors, groups, is_modified); + mm = new ModelMeta(model_curie, date, title, state, conformance, contributors, groups, is_modified); } //matching for (String ind : ind_return.keySet()) { diff --git a/minerva-server/src/main/resources/ModelSearchQueryTemplate.rq b/minerva-server/src/main/resources/ModelSearchQueryTemplate.rq index 6521f41e..aa48b64e 100644 --- a/minerva-server/src/main/resources/ModelSearchQueryTemplate.rq +++ b/minerva-server/src/main/resources/ModelSearchQueryTemplate.rq @@ -21,6 +21,7 @@ GRAPH ?id { ?contributor ; optional{?id ?group } . optional{?id lego:modelstate ?state } . +optional{?id lego:conforms_to_gpad ?conformsToGPAD} diff --git a/pom.xml b/pom.xml index 53634ef1..ed66035c 100644 --- a/pom.xml +++ b/pom.xml @@ -325,9 +325,14 @@ org.apache.jena apache-jena-libs - 3.12.0 + 4.9.0 pom + + org.apache.jena + jena-commonsrdf + 4.9.0 + net.sf.trove4j trove4j @@ -344,6 +349,7 @@ log4j-core ${log4j.version} + org.apache.logging.log4j log4j-1.2-api @@ -358,6 +364,30 @@ log4j log4j + + org.apache.jena + jena-arq + + + org.apache.jena + jena-base + + + org.apache.jena + jena-core + + + org.apache.jena + jena-iri + + + org.apache.jena + jena-shaded-guava + + + org.apache.jena + jena-tdb +