Add action to check conformance with GPAD ShEx on each change.

minerva-converter/src/main/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLExport.java

@@ -12,8 +12,8 @@
 import org.apache.jena.rdf.model.Resource;
 import org.apache.jena.sparql.core.Var;
 import org.apache.jena.sparql.engine.binding.Binding;
+import org.apache.jena.sparql.engine.binding.BindingBuilder;
 import org.apache.jena.sparql.engine.binding.BindingFactory;
-import org.apache.jena.sparql.engine.binding.BindingMap;
 import org.apache.jena.vocabulary.RDF;
 import org.apache.log4j.Logger;
 import org.geneontology.minerva.curie.CurieHandler;
@@ -288,11 +288,11 @@ private Map<Triple, Set<GPADEvidence>> evidencesForFacts(Set<Triple> facts, Mode
 
  @SafeVarargs
  private final Binding createBinding(Pair<Var, Node>... bindings) {
- BindingMap map = BindingFactory.create();
+ BindingBuilder builder = BindingFactory.builder();
  for (Pair<Var, Node> binding : bindings) {
- map.add(binding.getLeft(), binding.getRight());
+ builder.add(binding.getLeft(), binding.getRight());
  }
- return map;
+ return builder.build();
  }
 
  private Set<AnnotationExtension> possibleExtensions(Set<BasicGPADData> basicAnnotations, Model model) {

minerva-converter/src/test/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLTest.java

@@ -5,7 +5,7 @@
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
 import org.apache.jena.reasoner.rulesys.Rule;
-import org.apache.jena.system.JenaSystem;
+import org.apache.jena.sys.JenaSystem;
 import org.geneontology.jena.OWLtoRules;
 import org.geneontology.minerva.curie.DefaultCurieHandler;
 import org.geneontology.rules.engine.RuleEngine;

minerva-core/pom.xml

@@ -54,6 +54,10 @@
  <artifactId>apache-jena-libs</artifactId>
  <type>pom</type>
  </dependency>
+ <dependency>
+ <groupId>org.apache.jena</groupId>
+ <artifactId>jena-commonsrdf</artifactId>
+ </dependency>
  <dependency>
  <groupId>net.sf.trove4j</groupId>
  <artifactId>trove4j</artifactId>
@@ -73,6 +77,10 @@
  <groupId>org.apache.jena</groupId>
  <artifactId>*</artifactId>
  </exclusion>
+ <exclusion> <!-- declare the exclusion here -->
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-rdf-jena</artifactId>
+ </exclusion>
  </exclusions>
  </dependency>
  <dependency>
@@ -92,6 +100,11 @@
  <artifactId>commons-math3</artifactId>
  <version>3.6.1</version>
  </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.15</version>
+ </dependency>
  <dependency>
  <groupId>org.obolibrary.robot</groupId>
  <artifactId>robot-core</artifactId>

minerva-core/src/main/java/org/geneontology/minerva/BlazegraphMolecularModelManager.java

@@ -7,11 +7,18 @@
 import info.aduna.iteration.Iterations;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.jena.graph.Graph;
+import org.apache.jena.shex.ShapeMap;
+import org.apache.jena.shex.ShexReport;
+import org.apache.jena.shex.ShexSchema;
+import org.apache.jena.shex.ShexValidator;
+import org.apache.jena.sys.JenaSystem;
 import org.apache.log4j.Logger;
 import org.geneontology.minerva.MolecularModelManager.UnknownIdentifierException;
 import org.geneontology.minerva.curie.CurieHandler;
 import org.geneontology.minerva.util.AnnotationShorthand;
 import org.geneontology.minerva.util.BlazegraphMutationCounter;
+import org.geneontology.minerva.util.JenaOwlTool;
 import org.geneontology.minerva.util.ReverseChangeGenerator;
 import org.openrdf.model.*;
 import org.openrdf.model.impl.URIImpl;
@@ -22,6 +29,7 @@
 import org.openrdf.repository.RepositoryException;
 import org.openrdf.repository.RepositoryResult;
 import org.openrdf.rio.*;
+import org.openrdf.rio.helpers.BasicWriterSettings;
 import org.openrdf.rio.helpers.RDFHandlerBase;
 import org.openrdf.rio.helpers.StatementCollector;
 import org.semanticweb.owlapi.apibinding.OWLManager;
@@ -67,6 +75,7 @@ public class BlazegraphMolecularModelManager<METADATA> extends CoreMolecularMode
  public BlazegraphMolecularModelManager(OWLOntology tbox, CurieHandler curieHandler, String modelIdPrefix, @Nonnull String pathToJournal, String pathToExportFolder, String pathToOntologyJournal, boolean downloadOntologyJournal)
  throws OWLOntologyCreationException, IOException {
  super(tbox, pathToOntologyJournal, downloadOntologyJournal);
+ JenaSystem.init();
  if (curieHandler == null) {
  LOG.error("curie handler required for blazegraph model manager startup ");
  System.exit(-1);
@@ -195,23 +204,14 @@ public void saveModel(ModelContainer m)
  final OWLOntologyManager manager = ont.getOWLOntologyManager();
  List<OWLOntologyChange> changes = preSaveFileHandler(ont);
  synchronized (ont) {
- try {
- this.writeModelToDatabase(ont, modelId);
- // reset modified flag for abox after successful save
- m.setAboxModified(false);
- // dump stored model to export file
- if (this.pathToExportFolder != null) {
- File folder = new File(this.pathToExportFolder);
- dumpStoredModel(modelId, folder);
- }
- } finally {
- if (changes != null) {
- List<OWLOntologyChange> invertedChanges = ReverseChangeGenerator
- .invertChanges(changes);
- if (invertedChanges != null && !invertedChanges.isEmpty()) {
- manager.applyChanges(invertedChanges);
- }
- }
+ manager.applyChanges(changes);
+ this.writeModelToDatabase(ont, modelId);
+ // reset modified flag for abox after successful save
+ m.setAboxModified(false);
+ // dump stored model to export file
+ if (this.pathToExportFolder != null) {
+ File folder = new File(this.pathToExportFolder);
+ dumpStoredModel(modelId, folder);
  }
  }
  }
@@ -241,14 +241,10 @@ private void writeModelToDatabase(OWLOntology model, IRI modelId) throws Reposit
  }
 
  private List<OWLOntologyChange> preSaveFileHandler(OWLOntology model) throws UnknownIdentifierException {
- List<OWLOntologyChange> allChanges = null;
+ List<OWLOntologyChange> allChanges = new ArrayList<>();
  for (PreFileSaveHandler handler : preFileSaveHandlers) {
  List<OWLOntologyChange> changes = handler.handle(model);
  if (changes != null && !changes.isEmpty()) {
- if (allChanges == null) {
- allChanges = new ArrayList<OWLOntologyChange>(
- changes.size());
- }
  allChanges.addAll(changes);
  }
  }
@@ -994,4 +990,29 @@ public int addTaxonToDatabaseWithSparql(IRI model_iri, IRI taxon_iri) throws Rep
  return changes;
  }
 
+ public static class ShexValidatingPreSaveFileHandler implements PreFileSaveHandler {
+
+ private final ShexSchema schema;
+ private final ShapeMap shapeMap;
+ private final OWLAnnotationProperty conformanceProperty;
+ public ShexValidatingPreSaveFileHandler(ShexSchema schema, ShapeMap map, String propertyIRI) {
+ this.schema = schema;
+ this.shapeMap = map;
+ this.conformanceProperty = OWLManager.getOWLDataFactory().getOWLAnnotationProperty(IRI.create(propertyIRI));
+ }
+
+ @Override
+ public List<OWLOntologyChange> handle(OWLOntology modelOnt) throws UnknownIdentifierException {
+ OWLDataFactory factory = OWLManager.getOWLDataFactory();
+ Graph graph = JenaOwlTool.getJenaModel(modelOnt).getGraph();
+ ShexReport report = ShexValidator.get().validate(graph, schema, shapeMap);
+ boolean conforms = report.conforms();
+ System.out.println("CONFORMS? " + conforms);
+ System.out.println(factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(conforms)));
+ OWLOntologyChange remove = new RemoveOntologyAnnotation(modelOnt, factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(!conforms)));
+ OWLOntologyChange add = new AddOntologyAnnotation(modelOnt, factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(conforms)));
+ return Arrays.asList(remove, add);
+ }
+ }
+
 }

minerva-core/src/main/java/org/geneontology/minerva/util/AnnotationShorthand.java

@@ -30,7 +30,8 @@ public enum AnnotationShorthand {
  deprecated(OWLRDFVocabulary.OWL_DEPRECATED.getIRI()), // model annotation to indicate deprecated models
  templatestate(IRI.create("http://geneontology.org/lego/templatestate"), "template"), // designate a model as a template
  modelstate(IRI.create("http://geneontology.org/lego/modelstate"), "state"),
- wasDerivedFrom(IRI.create("http://www.w3.org/ns/prov#wasDerivedFrom"), "wasDerivedFrom");
+ wasDerivedFrom(IRI.create("http://www.w3.org/ns/prov#wasDerivedFrom"), "wasDerivedFrom"),
+ conformsToGPAD(IRI.create("http://geneontology.org/lego/conforms_to_gpad"), "conforms-to-gpad");
 
 
  private final IRI annotationProperty;

minerva-core/src/main/java/org/geneontology/minerva/validation/ShexValidator.java

@@ -14,13 +14,15 @@
 import fr.inria.lille.shexjava.validation.Status;
 import fr.inria.lille.shexjava.validation.Typing;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.rdf.api.RDF;
-import org.apache.commons.rdf.api.RDFTerm;
-import org.apache.commons.rdf.jena.JenaGraph;
-import org.apache.commons.rdf.jena.JenaRDF;
+import org.apache.commons.rdf.api.*;
 import org.apache.commons.rdf.simple.SimpleRDF;
+import org.apache.jena.commonsrdf.JenaCommonsRDF;
+import org.apache.jena.commonsrdf.JenaRDF;
+import org.apache.jena.commonsrdf.impl.JenaGraph;
 import org.apache.jena.query.*;
 import org.apache.jena.rdf.model.*;
+import org.apache.jena.rdf.model.Literal;
+import org.apache.jena.sys.JenaSystem;
 import org.apache.log4j.Logger;
 import org.geneontology.minerva.BlazegraphOntologyManager;
 import org.geneontology.minerva.curie.CurieHandler;
@@ -34,7 +36,9 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.*;
+import java.util.stream.Stream;
 
 /**
  * @author bgood
@@ -67,6 +71,7 @@ public ShexValidator(File shex_schema_file, File shex_map_file, BlazegraphOntolo
  }
 
  public void init(File shex_schema_file, File shex_map_file, BlazegraphOntologyManager go_lego, CurieHandler curieHandler_) throws Exception {
+ JenaSystem.init();
  schema = GenParser.parseSchema(shex_schema_file.toPath());
  GoQueryMap = makeGoQueryMap(shex_map_file.getAbsolutePath());
  //tbox_reasoner = tbox_reasoner_;
@@ -115,9 +120,8 @@ public static Map<String, String> makeGoQueryMap(String shapemap_file) throws IO
  public ShexValidationReport runShapeMapValidation(Model test_model) {
  boolean explain = true;
  ShexValidationReport r = new ShexValidationReport();
- JenaRDF jr = new JenaRDF();
  //this shex implementation likes to use the commons JenaRDF interface, nothing exciting here
- JenaGraph shexy_graph = jr.asGraph(test_model);
+ Graph shexy_graph = JenaCommonsRDF.fromJena(test_model.getGraph());
  boolean all_good = true;
  try {
  Typing all_typed = runRefineWithTimeout(shexy_graph);
@@ -251,7 +255,7 @@ public ShexValidationReport runShapeMapValidationWithRecursiveSingleNodeValidati
  ShexValidationReport r = new ShexValidationReport();
  JenaRDF jr = new JenaRDF();
  //this shex implementation likes to use the commons JenaRDF interface, nothing exciting here
- JenaGraph shexy_graph = jr.asGraph(test_model);
+ Graph shexy_graph = JenaCommonsRDF.fromJena(test_model.getGraph());
  //recursive only checks the focus node against the chosen shape.
  RecursiveValidationWithMemorization shex_model_validator = new RecursiveValidationWithMemorization(schema, shexy_graph);
  //for each shape in the query map (e.g. MF, BP, CC, etc.)
@@ -391,7 +395,7 @@ private Map<Resource, Set<String>> getShapesToTestForEachResource(Model test_mod
  return node_s_shapes;
  }
 
- private Typing runRefineWithTimeout(JenaGraph shexy_graph) {
+ private Typing runRefineWithTimeout(Graph shexy_graph) {
  final ExecutorService service = Executors.newSingleThreadExecutor();
  try {
  final Future<Typing> f = service.submit(() -> {

minerva-server/src/main/java/org/geneontology/minerva/server/StartUpTool.java

@@ -1,12 +1,16 @@
 package org.geneontology.minerva.server;
 
+import org.apache.jena.shex.ShapeMap;
+import org.apache.jena.shex.ShexSchema;
+import org.apache.jena.shex.parser.ShExC;
 import org.apache.log4j.Logger;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
+import org.geneontology.minerva.BlazegraphMolecularModelManager;
 import org.geneontology.minerva.MinervaOWLGraphWrapper;
 import org.geneontology.minerva.ModelReaderHelper;
 import org.geneontology.minerva.UndoAwareMolecularModelManager;
@@ -93,6 +97,10 @@ public static class MinervaStartUpConfig {
  public MinervaShexValidator shex;
  public String pathToOntologyJournal;
 
+ public String pathToGPADSheX;
+
+ public String pathToGPADShapeMap;
+
  }
 
  public static void main(String[] args) throws Exception {
@@ -172,6 +180,10 @@ else if (opts.nextEq("--golr-labels")) {
  conf.sparqlEndpointTimeout = Integer.parseInt(opts.nextOpt());
  } else if (opts.nextEq("--ontojournal")) {
  conf.pathToOntologyJournal = opts.nextOpt();
+ } else if (opts.nextEq("--gpad-shex")) {
+ conf.pathToGPADSheX = opts.nextOpt();
+ } else if (opts.nextEq("--gpad-shape-map")) {
+ conf.pathToGPADShapeMap = opts.nextOpt();
  } else {
  break;
  }
@@ -340,6 +352,11 @@ public static Server startUp(final MinervaStartUpConfig conf)
  conf.curieHandler, conf.modelIdPrefix, conf.journalFile, conf.exportFolder, conf.pathToOntologyJournal, true);
  // set pre and post file handlers
  models.addPostLoadOntologyFilter(ModelReaderHelper.INSTANCE);
+ if (conf.pathToGPADSheX != null && conf.pathToGPADShapeMap != null) {
+ ShexSchema schema = ShExC.parse(conf.pathToGPADSheX);
+ ShapeMap shapeMap = ShExC.parseShapeMap(conf.pathToGPADShapeMap);
+ models.addPreFileSaveHandler(new BlazegraphMolecularModelManager.ShexValidatingPreSaveFileHandler(schema, shapeMap, "http://geneontology.org/lego/conforms_to_gpad"));
+ }
  // conf.shex.tbox_reasoner = models.getTbox_reasoner();
  conf.shex.setGo_lego_repo(models.getGolego_repo());
  conf.shex.curieHandler = conf.curieHandler;

minerva-server/src/main/java/org/geneontology/minerva/server/handler/ModelSearchHandler.java

@@ -10,6 +10,7 @@
 import org.openrdf.repository.RepositoryException;
 import org.semanticweb.owlapi.model.IRI;
 
+import javax.annotation.Nullable;
 import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
 import java.io.IOException;
@@ -89,18 +90,21 @@ static public class ModelMeta {
  private String date;
  private String title;
  private String state;
+ @SerializedName("conforms-to-gpad")
+ private Boolean conformsToGPAD;
  private Set<String> contributors;
  private Set<String> groups;
  private HashMap<String, Set<String>> query_match;
 
  @SerializedName("modified-p")
  private boolean modified;
 
- public ModelMeta(String id, String date, String title, String state, Set<String> contributors, Set<String> groups, boolean modified) {
+ public ModelMeta(String id, String date, String title, String state, @Nullable Boolean conformsToGPAD, Set<String> contributors, Set<String> groups, boolean modified) {
  this.id = id;
  this.date = date;
  this.title = title;
  this.state = state;
+ this.conformsToGPAD = conformsToGPAD;
  this.contributors = contributors;
  this.groups = groups;
  this.modified = modified;
@@ -150,6 +154,15 @@ public void setState(String state) {
  this.state = state;
  }
 
+ @Nullable
+ public Boolean getConformsToGPAD() {
+ return this.conformsToGPAD;
+ }
+
+ public void setConformsToGPAD(Boolean conforms) {
+ this.conformsToGPAD = conforms;
+ }
+
  public Set<String> getContributors() {
  return contributors;
  }
@@ -454,7 +467,7 @@ public ModelSearchResult search(Set<String> taxa,
  String group_by_constraint = "GROUP BY ?id";
  //default return block
  //TODO investigate need to add DISTINCT to GROUP_CONCAT here
- String return_block = "?id (MIN(?date) AS ?mindate) (MIN(?title) AS ?mintitle) (MIN(?state) AS ?minstate) <ind_return_list> (GROUP_CONCAT(DISTINCT ?contributor;separator=\";\") AS ?contributors) (GROUP_CONCAT(DISTINCT ?group;separator=\";\") AS ?groups)";
+ String return_block = "?id (MIN(?date) AS ?mindate) (MIN(?title) AS ?mintitle) (MIN(?state) AS ?minstate) (MIN(?conformsToGPAD) AS ?minConformsToGPAD) <ind_return_list> (GROUP_CONCAT(DISTINCT ?contributor;separator=\";\") AS ?contributors) (GROUP_CONCAT(DISTINCT ?group;separator=\";\") AS ?groups)";
  if (count != null) {
  return_block = "(count(distinct ?id) as ?count)";
  limit_constraint = "";
@@ -525,6 +538,12 @@ public ModelSearchResult search(Set<String> taxa,
  if (state_binding != null) {
  state = state_binding.getValue().stringValue();
  }
+ Binding conformanceBinding = bs.getBinding("minConformsToGPAD");
+ Boolean conformance = null;
+ if (conformanceBinding != null) {
+ System.out.println("CONFORMANCE: " + conformanceBinding.getValue().stringValue());
+ conformance = Boolean.valueOf(conformanceBinding.getValue().stringValue());
+ }
  Binding group_binding = bs.getBinding("groups");
  String groups_ = "";
  if (group_binding != null) {
@@ -539,7 +558,7 @@ public ModelSearchResult search(Set<String> taxa,
  if (mm == null) {
  //look up model in in-memory cache to check edit state
  boolean is_modified = m3.isModelModified(model_iri);
- mm = new ModelMeta(model_curie, date, title, state, contributors, groups, is_modified);
+ mm = new ModelMeta(model_curie, date, title, state, conformance, contributors, groups, is_modified);
  }
  //matching
  for (String ind : ind_return.keySet()) {

minerva-server/src/main/resources/ModelSearchQueryTemplate.rq

@@ -21,6 +21,7 @@ GRAPH ?id {
 <http://purl.org/dc/elements/1.1/contributor> ?contributor ;
 optional{?id <http://purl.org/pav/providedBy> ?group } .
 optional{?id lego:modelstate ?state } .
+optional{?id lego:conforms_to_gpad ?conformsToGPAD}
 <types>
 <state_constraint>
 <contributor_constraint>