Metacat

	private String annotation_id = "http://doi.org/annotation.1.1";

	private static final String ANNOTATION_SYSTEM_META_FILE_PATH = "src/test/resources/annotation-system-meta-example.xml";

	private static final String ANNOTATION_FILE_PATH = "src/test/resources/annotation-example.rdf";;

//    @Test

//    @Test

//    @Test

     * Test building index for annotation.

     */

    @Test

    public void testAnnotation() throws Exception {

       SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAFILEPATH);

       InputStream emlInputStream = new FileInputStream(new File(EMLFILEPATH)); 

       Identifier pid = new Identifier();

       pid.setValue(id);

       solrIndex.update(pid, systemMetadata, emlInputStream);

       String result = doQuery(solrIndex.getSolrServer());

       List<String> ids = solrIndex.getSolrIds();

       boolean foundId = false;

       for(String identifiers :ids) {

           if (id.equals(identifiers)) {

               foundId = true;

           }

       }

       assertTrue(foundId);

       assertTrue(result.contains("version1"));

       // augment with the dynamic field

       SystemMetadata annotationSystemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, ANNOTATION_SYSTEM_META_FILE_PATH);

       InputStream annotationInputStream = new FileInputStream(new File(ANNOTATION_FILE_PATH)); 

       Identifier annotationPid = new Identifier();

       annotationPid.setValue(annotation_id);

       solrIndex.update(annotationPid, annotationSystemMetadata, annotationInputStream);

       String annotationResult = doQuery(solrIndex.getSolrServer());

       assertTrue(annotationResult.contains("measurement_sm"));

    }

    /**

/**

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.index.annotation;

import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.IOException;

import java.io.InputStream;

import java.net.MalformedURLException;

import java.util.ArrayList;

import java.util.Date;

import java.util.HashMap;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

import java.util.Set;

import javax.xml.parsers.ParserConfigurationException;

import javax.xml.transform.Result;

import javax.xml.transform.Source;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.TransformerFactoryConfigurationError;

import javax.xml.transform.dom.DOMSource;

import javax.xml.transform.stream.StreamResult;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrServer;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.SolrDocument;

import org.apache.solr.common.params.SolrParams;

import org.apache.solr.schema.IndexSchema;

import org.apache.solr.servlet.SolrRequestParsers;

import org.dataone.cn.indexer.convert.SolrDateConverter;

import org.dataone.cn.indexer.parser.AbstractDocumentSubprocessor;

import org.dataone.cn.indexer.parser.IDocumentSubprocessor;

import org.dataone.cn.indexer.parser.ISolrField;

import org.dataone.cn.indexer.solrhttp.SolrDoc;

import org.dataone.cn.indexer.solrhttp.SolrElementField;

import org.dataone.service.exceptions.NotFound;

import org.dataone.service.exceptions.UnsupportedType;

import org.dataone.service.util.DateTimeMarshaller;

import org.w3c.dom.Document;

import org.xml.sax.SAXException;

import com.hp.hpl.jena.ontology.OntModel;

import com.hp.hpl.jena.query.Dataset;

import com.hp.hpl.jena.query.Query;

import com.hp.hpl.jena.query.QueryExecution;

import com.hp.hpl.jena.query.QueryExecutionFactory;

import com.hp.hpl.jena.query.QueryFactory;

import com.hp.hpl.jena.query.QuerySolution;

import com.hp.hpl.jena.query.ResultSet;

import com.hp.hpl.jena.rdf.model.ModelFactory;

import com.hp.hpl.jena.tdb.TDBFactory;

import edu.ucsb.nceas.metacat.common.SolrServerFactory;

import edu.ucsb.nceas.metacat.common.query.SolrQueryServiceController;

/**

 * A solr index parser for an RDF/XML file.

 * The solr doc of the RDF/XML object only has the system metadata information.

 * The solr docs of the science metadata doc and data file have the annotation information.

 */

public class RdfXmlSubprocessor extends AbstractDocumentSubprocessor implements IDocumentSubprocessor {

    private static final String QUERY ="q=id:";

    private static Log log = LogFactory.getLog(RdfXmlSubprocessor.class);

    private static SolrServer solrServer =  null;

    static {

        try {

            solrServer = SolrServerFactory.createSolrServer();

        } catch (Exception e) {

            log.error("RdfXmlSubprocessor - can't generate the SolrServer since - "+e.getMessage());

        }

    }

    @Override

    public Map<String, SolrDoc> processDocument(String identifier, Map<String, SolrDoc> docs, Document doc) throws Exception {

        SolrDoc resourceMapDoc = docs.get(identifier);

        List<SolrDoc> processedDocs = process(resourceMapDoc, doc);

        Map<String, SolrDoc> processedDocsMap = new HashMap<String, SolrDoc>();

        for (SolrDoc processedDoc : processedDocs) {

            processedDocsMap.put(processedDoc.getIdentifier(), processedDoc);

        }

        return processedDocsMap;

    }

    private InputStream toInputStream(Document doc) throws TransformerConfigurationException, TransformerException, TransformerFactoryConfigurationError {

    	ByteArrayOutputStream outputStream = new ByteArrayOutputStream();

    	Source xmlSource = new DOMSource(doc);

    	Result outputTarget = new StreamResult(outputStream);

    	TransformerFactory.newInstance().newTransformer().transform(xmlSource, outputTarget);

    	InputStream is = new ByteArrayInputStream(outputStream.toByteArray());

    	return is;

    }

    private List<SolrDoc> process(SolrDoc indexDocument, Document rdfXmlDocument) throws Exception {

    	// get the triplestore dataset

		Dataset dataset = TripleStoreService.getInstance().getDataset();

    	// read the annotation

		InputStream source = toInputStream(rdfXmlDocument);

    	String name = indexDocument.getIdentifier();

    	boolean loaded = dataset.containsNamedModel(name);

		if (!loaded) {

			OntModel ontModel = ModelFactory.createOntologyModel();

			ontModel.read(source, name);

			dataset.addNamedModel(name, ontModel);

		}

		//dataset.getDefaultModel().add(ontModel);

		// process each field query

        Map<String, SolrDoc> documentsToIndex = new HashMap<String, SolrDoc>();

		for (ISolrField field: this.getFieldList()) {

			String q = null;

			if (field instanceof SparqlField) {

				q = ((SparqlField) field).getQuery();

				q = q.replaceAll("\\$GRAPH_NAME", name);

				Query query = QueryFactory.create(q);

				QueryExecution qexec = QueryExecutionFactory.create(query, dataset);

				ResultSet results = qexec.execSelect();

				while (results.hasNext()) {

					SolrDoc solrDoc = null;

					QuerySolution solution = results.next();

					System.out.println(solution.toString());

					// find the index document we are trying to augment with the annotation

					if (solution.contains("pid")) {

						String id = solution.getLiteral("pid").getString();

						solrDoc = documentsToIndex.get(id);

						if (solrDoc == null) {

							solrDoc = new SolrDoc();

							documentsToIndex.put(id, solrDoc);

						}

					}

					// add the field to the index document

					if (solution.contains(field.getName())) {

						String value = solution.get(field.getName()).toString();

						SolrElementField f = new SolrElementField(field.getName(), value);

						if (!solrDoc.hasFieldWithValue(f.getName(), f.getValue())) {

							solrDoc.addField(f);

						}

					}

				}

			}

		}

		// clean up the triple store

		TDBFactory.release(dataset);

		// merge the existing index with the new[er] values

        Map<String, SolrDoc> existingDocuments = getSolrDocs(documentsToIndex.keySet());

        Map<String, SolrDoc> mergedDocuments = mergeDocs(documentsToIndex, existingDocuments);

        mergedDocuments.put(indexDocument.getIdentifier(), indexDocument);

        return new ArrayList<SolrDoc>(mergedDocuments.values());

    }

    private Map<String, SolrDoc> getSolrDocs(Set<String> ids) throws Exception {

        Map<String, SolrDoc> list = new HashMap<String, SolrDoc>();

        if (ids != null) {

            for (String id : ids) {

            	SolrDoc doc = getSolrDoc(id);

                if (doc != null) {

                    list.put(id, doc);

                }

            }

        }

        return list;

    }

    private Map<String, SolrDoc> mergeDocs(Map<String, SolrDoc> pending, Map<String, SolrDoc> existing) {

    	Map<String, SolrDoc> merged = new HashMap<String, SolrDoc>();

    	Iterator<String> pendingIter = pending.keySet().iterator();

    	while (pendingIter.hasNext()) {

    		String id = pendingIter.next();

    		SolrDoc pendingDoc = pending.get(id);

    		SolrDoc existingDoc = existing.get(id);

    		SolrDoc mergedDoc = new SolrDoc();

    		if (existingDoc != null) {

    			// merge the existing fields

    			for (SolrElementField field: existingDoc.getFieldList()) {

    				mergedDoc.addField(field);

    			}

    		}

    		// add the pending

    		for (SolrElementField field: pendingDoc.getFieldList()) {

				mergedDoc.addField(field);

			}

    		// include in results

			merged.put(id, mergedDoc);

    	}

    	return merged;

    }

	/*

	 * Get the SolrDoc for the specified id

	 */

	public static SolrDoc getSolrDoc(String id) throws SolrServerException, MalformedURLException, UnsupportedType, NotFound, ParserConfigurationException, IOException, SAXException {

		SolrDoc doc = new SolrDoc();

		if (solrServer != null) {

			String query = QUERY + "\"" + id + "\"";

			SolrParams solrParams = SolrRequestParsers.parseQueryString(query);

			QueryResponse qr = solrServer.query(solrParams);

			SolrDocument orig = qr.getResults().get(0);

			IndexSchema indexSchema = SolrQueryServiceController.getInstance().getSchema();

			for (String fieldName : orig.getFieldNames()) {

				// don't transfer the copyTo fields, otherwise there are errors

				if (indexSchema.isCopyFieldTarget(indexSchema.getField(fieldName))) {

					continue;

				}

				for (Object value : orig.getFieldValues(fieldName)) {

					String stringValue = value.toString();

					// special handling for dates in ISO 8601

					if (value instanceof Date) {

						stringValue = DateTimeMarshaller.serializeDateToUTC((Date) value);

						SolrDateConverter converter = new SolrDateConverter();

						stringValue = converter.convert(stringValue);

					}

					SolrElementField field = new SolrElementField(fieldName, stringValue);

					log.debug("Adding field: " + fieldName);

					doc.addField(field);

				}

			}

		}

		return doc;

	}

}

package edu.ucsb.nceas.metacat.index.annotation;

import java.util.List;

import javax.xml.xpath.XPath;

import org.dataone.cn.indexer.parser.ISolrField;

import org.dataone.cn.indexer.solrhttp.SolrElementField;

import org.w3c.dom.Document;

public class SparqlField implements ISolrField {

	private String name;

	private String query;

	public SparqlField(String name, String query) {

		this.name = name;

		this.query = query;

	}

	public String getName() {

		return name;

	}

	public void setName(String name) {

		this.name = name;

	}

	public String getQuery() {

		return query;

	}

	public void setQuery(String query) {

		this.query = query;

	}

	@Override

	public List<SolrElementField> getFields(Document arg0, String arg1)

			throws Exception {

		// TODO Auto-generated method stub

		return null;

	}

	@Override

	public void initExpression(XPath arg0) {

		// TODO Auto-generated method stub

	}

}

package edu.ucsb.nceas.metacat.index.annotation;

import java.io.File;

import com.hp.hpl.jena.query.Dataset;

import com.hp.hpl.jena.tdb.TDBFactory;

public class TripleStoreService {

	private static TripleStoreService instance;

	private TripleStoreService() {}

	public static TripleStoreService getInstance() {

		if (instance == null) {

			instance = new TripleStoreService();

		}

		return instance;

	}

	public Dataset getDataset() {

		String directory = "./tdb";

    	// for testing, delete the triplestore each time

    	File dir = new File(directory);

//    	if (dir.exists()) {

//    		dir.delete();

//    	}

		Dataset dataset = TDBFactory.createDataset(directory);

		return dataset;

	}

}

			value="/d100:systemMetadata/formatId[text()='http://www.openarchives.org/ore/terms']" />

    <import resource="application-context-annotation.xml" />

       <ref bean="rdfXmlSubprocessor" />

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

	xmlns:p="http://www.springframework.org/schema/p"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">

	<bean id="rdfXmlSubprocessor" class="edu.ucsb.nceas.metacat.index.annotation.RdfXmlSubprocessor">

		<property name="matchDocument"

			value="/d100:systemMetadata/formatId[text()='http://www.w3.org/TR/rdf-syntax-grammar']" />

		<property name="fieldList">

			<list>

				<ref bean="annotation.measurement" />

			</list>

		</property>	

	</bean>

	<bean id="annotation.measurement" class="edu.ucsb.nceas.metacat.index.annotation.SparqlField">

		<constructor-arg name="name" value="measurement_sm" />

		<constructor-arg name="query">

			<value>

				<![CDATA[

				PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 

				PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 

				PREFIX owl: <http://www.w3.org/2002/07/owl#> 

				PREFIX oboe-core: <http://ecoinformatics.org/oboe/oboe.1.0/oboe-core.owl#> 

				PREFIX ao: <http://purl.org/ao/>

				PREFIX dcterms: <http://purl.org/dc/terms/>

				PREFIX pav: <http://purl.org/pav/>

				PREFIX foaf: <http://xmlns.com/foaf/0.1/>

				SELECT ?measurement_sm ?pid

				FROM <$GRAPH_NAME>

				WHERE { 

						?measurement_sm rdf:type oboe-core:Measurement .

						?measurement_sm rdf:type ?restriction .

						?restriction owl:onProperty oboe-core:ofCharacteristic .

						?restriction owl:allValuesFrom ?characteristic .

						?characteristic rdfs:subClassOf+ ?allChar .

						?allChar rdfs:subClassOf oboe-core:Characteristic .

						OPTIONAL { 

							?measurement_sm rdf:type ?restriction2 .

							?restriction2 owl:onProperty oboe-core:usesStandard .

							?restriction2 owl:allValuesFrom ?standard .

						}

						?annotation ao:context ?measurement_sm .

						?annotation ao:annotatesResource ?metadata .

						?metadata dcterms:identifier ?pid . 

						?annotation pav:createdBy ?person .

						?person foaf:name ?name .

						?annotation pav:createdOn ?date .

				 	} 

				 ]]>

			</value>

		</constructor-arg>

		<!--property name="multivalue" value="false" /-->

	</bean>

</beans>

			<exclusions>

				<exclusion>

					<groupId>org.apache.httpcomponents</groupId>

					<artifactId>httpclient</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.apache.httpcomponents</groupId>

					<artifactId>httpmime</artifactId>

				</exclusion>

			</exclusions>

				<exclusion>

					<groupId>com.hp.hpl.jena</groupId>

					<artifactId>jena</artifactId>

				</exclusion>

				<exclusion>

					<groupId>org.apache.httpcomponents</groupId>

					<artifactId>httpclient</artifactId>

				</exclusion>

				<exclusion>

					<groupId>xml-apis</groupId>

					<artifactId>xml-apis</artifactId>

				</exclusion>

        <dependency>

			<groupId>org.apache.jena</groupId>

			<artifactId>jena-tdb</artifactId>

			<version>1.0.0</version>

		</dependency>

	   	<!--  

          -->

          <!--  

        -->