Metacat

package edu.ucsb.nceas.metacat.annotation;

import java.io.InputStream;

import java.io.StringWriter;

import java.sql.PreparedStatement;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

import java.util.Vector;

import org.apache.log4j.Logger;

import org.apache.wicket.protocol.http.mock.MockHttpServletRequest;

import org.dataone.service.types.v1.Identifier;

import org.dataone.service.types.v1.Session;

import org.dataone.service.types.v1.Subject;

import org.ecoinformatics.datamanager.parser.Attribute;

import org.ecoinformatics.datamanager.parser.DataPackage;

import org.ecoinformatics.datamanager.parser.Entity;

import org.ecoinformatics.datamanager.parser.generic.DataPackageParserInterface;

import org.ecoinformatics.datamanager.parser.generic.Eml200DataPackageParser;

import com.hp.hpl.jena.ontology.AllValuesFromRestriction;

import com.hp.hpl.jena.ontology.Individual;

import com.hp.hpl.jena.ontology.ObjectProperty;

import com.hp.hpl.jena.ontology.OntClass;

import com.hp.hpl.jena.ontology.OntModel;

import com.hp.hpl.jena.ontology.Ontology;

import com.hp.hpl.jena.rdf.model.ModelFactory;

import com.hp.hpl.jena.rdf.model.Property;

import com.hp.hpl.jena.rdf.model.Resource;

import com.hp.hpl.jena.util.iterator.ExtendedIterator;

import edu.ucsb.nceas.metacat.DBUtil;

import edu.ucsb.nceas.metacat.DocumentImpl;

import edu.ucsb.nceas.metacat.IdentifierManager;

import edu.ucsb.nceas.metacat.McdbDocNotFoundException;

import edu.ucsb.nceas.metacat.database.DBConnection;

import edu.ucsb.nceas.metacat.database.DBConnectionPool;

import edu.ucsb.nceas.metacat.dataone.MNodeService;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.replication.ReplicationService;

import edu.ucsb.nceas.metacat.util.DocumentUtil;

import edu.ucsb.nceas.utilities.SortedProperties;

public class DatapackageSummarizer {

	private static Logger logMetacat = Logger.getLogger(DatapackageSummarizer.class);

	public static String rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";

	public static String rdfs = "http://www.w3.org/2000/01/rdf-schema#";

	public static String owl = "http://www.w3.org/2002/07/owl#";

	public static String oboe = "http://ecoinformatics.org/oboe/oboe.1.0/oboe.owl#";

	public static String oboe_core = "http://ecoinformatics.org/oboe/oboe.1.0/oboe-core.owl#";

	public static String oboe_characteristics = "http://ecoinformatics.org/oboe/oboe.1.0/oboe-characteristics.owl#";

	public static String oboe_sbc = "http://ecoinformatics.org/oboe-ext/sbclter.1.0/oboe-sbclter.owl#";

	public static String oa = "http://www.w3.org/ns/oa#";

	public static String oa_source = "http://www.w3.org/ns/oa.rdf";

	public static String dcterms = "http://purl.org/dc/terms/";

	public static String dcterms_source = "http://dublincore.org/2012/06/14/dcterms.rdf";

	public static String foaf = "http://xmlns.com/foaf/0.1/";

	public static String foaf_source = "http://xmlns.com/foaf/spec/index.rdf";

    public static String prov = "http://www.w3.org/ns/prov#";

    public static String prov_source = "http://www.w3.org/ns/prov.owl";

    public static String cito =  "http://purl.org/spar/cito/";

	public static String OBOE_SBC = "OBOE-SBC";

    // package visibility for testing only

    boolean randomize = false;

    /**

     * Generate annotation for given metadata identifier

     * @param metadataPid

     */

    public String generateAnnotation(Identifier metadataPid) throws Exception {

    	DataPackage dataPackage = this.getDataPackage(metadataPid);

		OntModel m = ModelFactory.createOntologyModel();

		Ontology ont = m.createOntology("http://annotation/" + metadataPid.getValue());

		// TODO: import the ontologies we use

		ont.addImport(m.createResource(oboe));

		m.addSubModel(ModelFactory.createOntologyModel().read(oboe));

		ont.addImport(m.createResource(oboe_sbc));

		m.addSubModel(ModelFactory.createOntologyModel().read(oboe_sbc));

		ont.addImport(m.createResource(oa));

		m.addSubModel(ModelFactory.createOntologyModel().read(oa_source));

		ont.addImport(m.createResource(dcterms));

		m.addSubModel(ModelFactory.createOntologyModel().read(dcterms_source));

		ont.addImport(m.createResource(foaf));

		m.addSubModel(ModelFactory.createOntologyModel().read(foaf_source));

		ont.addImport(m.createResource(prov));

		//m.addSubModel(ModelFactory.createOntologyModel().read(prov_source));

		ont.addImport(m.createResource(cito));

		// properties

		ObjectProperty hasBodyProperty = m.getObjectProperty(oa + "hasBody");

		ObjectProperty hasTargetProperty = m.getObjectProperty(oa + "hasTarget");

		ObjectProperty hasSourceProperty = m.getObjectProperty(oa + "hasSource");

		ObjectProperty hasSelectorProperty = m.getObjectProperty(oa + "hasSelector");

		ObjectProperty annotatedByProperty = m.getObjectProperty(oa + "annotatedBy");

		Property identifierProperty = m.getProperty(dcterms + "identifier");

		Property conformsToProperty = m.getProperty(dcterms + "conformsTo");

		Property wasAttributedTo = m.getProperty(prov + "wasAttributedTo");

		Property nameProperty = m.getProperty(foaf + "name");

		Property rdfValue = m.getProperty(rdf + "value");

		ObjectProperty ofCharacteristic = m.getObjectProperty(oboe_core + "ofCharacteristic");

		ObjectProperty usesStandard = m.getObjectProperty(oboe_core + "usesStandard");

		ObjectProperty ofEntity = m.getObjectProperty(oboe_core + "ofEntity");

		ObjectProperty hasMeasurement = m.getObjectProperty(oboe_core + "hasMeasurement");

		// classes

		OntClass entityClass =  m.getOntClass(oboe_core + "Entity");

		OntClass observationClass =  m.getOntClass(oboe_core + "Observation");

		OntClass measurementClass =  m.getOntClass(oboe_core + "Measurement");

		OntClass characteristicClass = m.getOntClass(oboe_core + "Characteristic");

		OntClass standardClass =  m.getOntClass(oboe_core + "Standard");

		Resource annotationClass =  m.getOntClass(oa + "Annotation");

		Resource specificResourceClass =  m.getOntClass(oa + "SpecificResource");

		Resource fragmentSelectorClass =  m.getOntClass(oa + "FragmentSelector");

		Resource provEntityClass =  m.getResource(prov + "Entity");

		Resource personClass =  m.getResource(prov + "Person");

		// these apply to every attribute annotation

		Individual meta1 = m.createIndividual(ont.getURI() + "#meta", provEntityClass);

		meta1.addProperty(identifierProperty, metadataPid.getValue());

		// decide who should be credited with the package

		Individual p1 = null;

		// look up creators from the EML metadata

		List<String> creators = dataPackage.getCreators();

		//creators = Arrays.asList("Matthew Jones");

		if (creators != null && creators.size() > 0) {	

			// use an orcid if we can find one from their system

			String orcidUri = OrcidService.lookupOrcid(null, null, creators.toArray(new String[0]));

			if (orcidUri != null) {

				p1 = m.createIndividual(orcidUri, personClass);

				p1.addProperty(identifierProperty, orcidUri);

			} else {

				p1 = m.createIndividual(ont.getURI() + "#person", personClass);

			}

			// include the name we have in the metadata

			p1.addProperty(nameProperty, creators.get(0));

		}

		// attribute the package to this creator if we have one

		if (p1 != null) {

			meta1.addProperty(wasAttributedTo, p1);

		}

		// loop through the tables and attributes

		int entityCount = 1;

		Entity[] entities = dataPackage.getEntityList();

		if (entities != null) {

			for (Entity entity: entities) {

				String entityName = entity.getName();

				Individual o1 = m.createIndividual(ont.getURI() + "#observation" + entityCount, observationClass);

				Resource entityConcept = lookupEntity(entityClass, entity);

				if (entityConcept != null) {

					AllValuesFromRestriction avfr = m.createAllValuesFromRestriction(null, ofEntity, entityConcept);

					o1.addOntClass(avfr);

				}

				logMetacat.debug("Entity name: " + entityName);

				Attribute[] attributes = entity.getAttributeList().getAttributes();

				int attributeCount = 1;

				if (attributes != null) {

					for (Attribute attribute: attributes) {

						// for naming the individuals uniquely

						String cnt = entityCount + "_" + attributeCount;

						String attributeName = attribute.getName();

						String attributeLabel = attribute.getLabel();

						String attributeDefinition = attribute.getDefinition();

						String attributeType = attribute.getAttributeType();

						String attributeScale = attribute.getMeasurementScale();

						String attributeUnitType = attribute.getUnitType();

						String attributeUnit = attribute.getUnit();

						String attributeDomain = attribute.getDomain().getClass().getSimpleName();

						logMetacat.debug("Attribute name: " + attributeName);

						logMetacat.debug("Attribute label: " + attributeLabel);

						logMetacat.debug("Attribute definition: " + attributeDefinition);

						logMetacat.debug("Attribute type: " + attributeType);

						logMetacat.debug("Attribute scale: " + attributeScale);

						logMetacat.debug("Attribute unit type: " + attributeUnitType);

						logMetacat.debug("Attribute unit: " + attributeUnit);

						logMetacat.debug("Attribute domain: " + attributeDomain);

						// look up the characteristic or standard subclasses

						Resource standard = this.lookupStandard(standardClass, attribute);

						Resource characteristic = this.lookupCharacteristic(characteristicClass, attribute);

						if (standard != null || characteristic != null) {

							// instances

							Individual m1 = m.createIndividual(ont.getURI() + "#measurement" + cnt, measurementClass);

							Individual a1 = m.createIndividual(ont.getURI() + "#annotation" + cnt, annotationClass);

							Individual t1 = m.createIndividual(ont.getURI() + "#target" + cnt, specificResourceClass);

							String xpointer = "xpointer(/eml/dataSet/" + entityCount + "/attributeList/" + attributeCount + ")";

							Individual s1 = m.createIndividual(ont.getURI() + "#" + xpointer, fragmentSelectorClass);

							s1.addLiteral(rdfValue, xpointer);

							s1.addProperty(conformsToProperty, "http://www.w3.org/TR/xptr/");

							// statements about the annotation

							a1.addProperty(hasBodyProperty, m1);

							a1.addProperty(hasTargetProperty, t1);

							t1.addProperty(hasSourceProperty, meta1);

							t1.addProperty(hasSelectorProperty, s1);

							//a1.addProperty(annotatedByProperty, p1);

							// describe the measurement in terms of restrictions

							if (standard != null) {

								AllValuesFromRestriction avfr = m.createAllValuesFromRestriction(null, usesStandard, standard);

								m1.addOntClass(avfr);

							}

							if (characteristic != null) {

								AllValuesFromRestriction avfr = m.createAllValuesFromRestriction(null, ofCharacteristic, characteristic);

								m1.addOntClass(avfr);

							}

							// attach to the observation

							// TODO: evaluate whether the measurement can apply to the given observed entity

							o1.addProperty(hasMeasurement, m1);

						}

						attributeCount++;

					}

				}

				entityCount++;

			}

		}

		StringWriter sw = new StringWriter();

		// only write the base model

		//m.write(sw, "RDF/XML-ABBREV");

		m.write(sw, null);

		return sw.toString();

	}

	private Resource lookupStandard(OntClass standardClass, Attribute attribute) {

		// what's our unit?

		String unit = attribute.getUnit().toLowerCase();

		List<String> tokens = Arrays.asList(unit.split(" "));

		boolean found = false;

		ExtendedIterator iter = standardClass.listSubClasses(false);

		if (randomize) {

			List subclasses = iter.toList();

			int size = subclasses.size();

			Long index = new Long(Math.round(Math.floor((Math.random() * (size-1)))));

			OntClass subclass = (OntClass) subclasses.get( index.intValue() );

			return subclass;

		}

		while (iter.hasNext()) {

			OntClass subclass = (OntClass) iter.next();

			String subclassName = subclass.getLocalName().toLowerCase();

			logMetacat.debug("subclass: " + subclassName);

			if (tokens.contains(subclassName)) {

				found = true;

			}

			if (subclass.hasLabel(unit, null)) {

				found = true;

			}

			if (found) {

				return subclass;

			}

		}

		// try to look it up if we got this far

		return BioPortalService.lookupAnnotationClass(standardClass, unit, OBOE_SBC);

	}

	private Resource lookupCharacteristic(OntClass characteristicClass, Attribute attribute) {

		// what's our label?

		String label = attribute.getLabel().toLowerCase();

		List<String> tokens = Arrays.asList(label.split(" "));

		boolean found = false;

		// find something that matches

		ExtendedIterator iter = characteristicClass.listSubClasses();

		if (randomize) {

			List subclasses = iter.toList();

			int size = subclasses.size();

			Long index = new Long(Math.round(Math.floor((Math.random() * (size-1)))));

			OntClass subclass = (OntClass) subclasses.get( index.intValue() );

			return subclass;

		}

		while (iter.hasNext()) {

			OntClass subclass = (OntClass) iter.next();

			String subclassName = subclass.getLocalName().toLowerCase();

			logMetacat.debug("subclass: " + subclassName);

			if (tokens.contains(subclassName)) {

				found = true;

			}

			if (subclass.hasLabel(label, null)) {

				found = true;

			}

			if (found) {

				return subclass;

			}

		}

		// try to look it up if we got this far

		return BioPortalService.lookupAnnotationClass(characteristicClass, attribute.getDefinition(), OBOE_SBC);

	}

	private Resource lookupEntity(OntClass entityClass, Entity entity) {

		// what's our description like?

		String name = entity.getName();

		String definition = entity.getDefinition();

		// try to look it up if we got this far

		return BioPortalService.lookupAnnotationClass(entityClass, definition, OBOE_SBC);

	}

	private DataPackage getDataPackage(Identifier pid) throws Exception {

		// for using the MN API as the MN itself

		MockHttpServletRequest request = new MockHttpServletRequest(null, null, null);

		Session session = new Session();

        Subject subject = MNodeService.getInstance(request).getCapabilities().getSubject(0);

        session.setSubject(subject);

		InputStream emlStream = MNodeService.getInstance(request).get(session, pid);

		// parse the metadata

		DataPackageParserInterface parser = new Eml200DataPackageParser();

		parser.parse(emlStream);

		DataPackage dataPackage = parser.getDataPackage();

		return dataPackage;

	}

	private void summarize(List<Identifier> identifiers) throws SQLException {

		DBConnection dbconn = null;

		try {

			dbconn = DBConnectionPool.getDBConnection("DatapackageSummarizer.summarize");

			PreparedStatement dropStatement = dbconn.prepareStatement("DROP TABLE IF EXISTS entity_summary");

			dropStatement.execute();

			PreparedStatement createStatement = dbconn.prepareStatement(

					"CREATE TABLE entity_summary (" +

					"guid text, " +

					"title text, " +

					"entity text," +

					"attributeName text," +

					"attributeLabel text," +

					"attributeDefinition text," +

					"attributeType text," +

					"attributeScale text," +

					"attributeUnitType text," +

					"attributeUnit text," +

					"attributeDomain text" +

					")");

			createStatement.execute();

			PreparedStatement insertStatement = dbconn.prepareStatement(

					"INSERT INTO entity_summary " +

					"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");

			for (Identifier pid: identifiers) {

				logMetacat.debug("Parsing pid: " + pid.getValue());

				try {

					// get the package

					DataPackage dataPackage = this.getDataPackage(pid);

					String title = dataPackage.getTitle();

					logMetacat.debug("Title: " + title);

					Entity[] entities = dataPackage.getEntityList();

					if (entities != null) {

						for (Entity entity: entities) {

							String entityName = entity.getName();

							logMetacat.debug("Entity name: " + entityName);

							Attribute[] attributes = entity.getAttributeList().getAttributes();

							for (Attribute attribute: attributes) {

								String attributeName = attribute.getName();

								String attributeLabel = attribute.getLabel();

								String attributeDefinition = attribute.getDefinition();

								String attributeType = attribute.getAttributeType();

								String attributeScale = attribute.getMeasurementScale();

								String attributeUnitType = attribute.getUnitType();

								String attributeUnit = attribute.getUnit();

								String attributeDomain = attribute.getDomain().getClass().getSimpleName();

								logMetacat.debug("Attribute name: " + attributeName);

								logMetacat.debug("Attribute label: " + attributeLabel);

								logMetacat.debug("Attribute definition: " + attributeDefinition);

								logMetacat.debug("Attribute type: " + attributeType);

								logMetacat.debug("Attribute scale: " + attributeScale);

								logMetacat.debug("Attribute unit type: " + attributeUnitType);

								logMetacat.debug("Attribute unit: " + attributeUnit);

								logMetacat.debug("Attribute domain: " + attributeDomain);

								// set the values for this attribute

								insertStatement.setString(1, pid.getValue());

								insertStatement.setString(2, title);

								insertStatement.setString(3, entityName);

								insertStatement.setString(4, attributeName);

								insertStatement.setString(5, attributeLabel);

								insertStatement.setString(6, attributeDefinition);

								insertStatement.setString(7, attributeType);

								insertStatement.setString(8, attributeScale);

								insertStatement.setString(9, attributeUnitType);

								insertStatement.setString(10, attributeUnit);

								insertStatement.setString(11, attributeDomain);

								insertStatement.execute();

							}		

						}

					}

				} catch (Exception e) {

					logMetacat.warn("error parsing metadata for: " + pid.getValue(), e);

				}

			}

		} catch (SQLException sqle) {

			// just throw it

			throw sqle;

		} finally {

			if (dbconn != null) {

				DBConnectionPool.returnDBConnection(dbconn, 0);

				dbconn.close();

			}

		}

	}

	public static void main(String[] args) throws Exception {

		// set up the properties based on the test/deployed configuration of the workspace

			SortedProperties testProperties = new SortedProperties("test/test.properties");

			testProperties.load();

			String metacatContextDir = testProperties.getProperty("metacat.contextDir");

			PropertyService.getInstance(metacatContextDir + "/WEB-INF");

			testGenerate();

//			testSummary();

			System.exit(0);

	}

	public static void testGenerate() throws Exception {

		Identifier metadataPid = new Identifier();

		metadataPid.setValue("tao.1.4");

		DatapackageSummarizer ds = new DatapackageSummarizer();

		String rdfString = ds.generateAnnotation(metadataPid);

		logMetacat.info("RDF annotation: \n" + rdfString);

	}

	public static void testSummary() throws Exception {

		// summarize the packages

		DatapackageSummarizer ds = new DatapackageSummarizer();

		List<Identifier> identifiers = new ArrayList<Identifier>();

		Map<Integer, String> serverCodes = ReplicationService.getServerCodes();

		// select the metadata ids we want to summarize

		boolean includeReplicas = true;

		Iterator<Integer> codeIter = Arrays.asList(new Integer[] {1}).iterator();

		if (includeReplicas ) {

			codeIter = serverCodes.keySet().iterator();

		}

		Vector<String> idList = new Vector<String>();

		while (codeIter.hasNext()) {

			int serverLocation = codeIter.next();

			Vector<String> idList0 = DBUtil.getAllDocidsByType(DocumentImpl.EML2_0_0NAMESPACE, false, serverLocation);

			Vector<String> idList1 = DBUtil.getAllDocidsByType(DocumentImpl.EML2_0_1NAMESPACE, false, serverLocation);

			Vector<String> idList2 = DBUtil.getAllDocidsByType(DocumentImpl.EML2_1_0NAMESPACE, false, serverLocation);

			Vector<String> idList3 = DBUtil.getAllDocidsByType(DocumentImpl.EML2_1_1NAMESPACE, false, serverLocation);

			idList.addAll(idList0);

			idList.addAll(idList1);

			idList.addAll(idList2);

			idList.addAll(idList3);

		}

		// go through all the identifiers now

		for (String localId : idList) {

			try {

				String guid = IdentifierManager.getInstance().getGUID(

						DocumentUtil.getDocIdFromAccessionNumber(localId), 

						DocumentUtil.getRevisionFromAccessionNumber(localId));

				Identifier pid = new Identifier();

				pid.setValue(guid);

				identifiers.add(pid);

			} catch (McdbDocNotFoundException nfe) {

				// just skip it

				continue;

			}

		}

		ds.summarize(identifiers);

		System.exit(0);

	}

}