Metacat

/**

 *  '$RCSfile$'

 *    Purpose: A Class that implements administrative methods 

 *  Copyright: 2010 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Michael Daigle

 * 

 *   '$Author: berkley $'

 *     '$Date: 2010-06-08 12:34:30 -0700 (Tue, 08 Jun 2010) $'

 * '$Revision: 5374 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.util;

import java.io.ByteArrayInputStream;

import java.io.InputStream;

import java.io.OutputStream;

import java.math.BigInteger;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Date;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Vector;

import javax.activation.DataHandler;

import javax.activation.DataSource;

import javax.mail.internet.MimeBodyPart;

import javax.mail.internet.MimeMultipart;

import org.apache.commons.io.IOUtils;

import org.dataone.client.D1Client;

import org.dataone.client.MNode;

import org.dataone.client.ObjectFormatCache;

import org.dataone.client.auth.CertificateManager;

import org.dataone.ore.ResourceMapFactory;

import org.dataone.service.exceptions.NotFound;

import org.dataone.service.types.v1.AccessPolicy;

import org.dataone.service.types.v1.AccessRule;

import org.dataone.service.types.v1.Checksum;

import org.dataone.service.types.v1.Identifier;

import org.dataone.service.types.v1.NodeReference;

import org.dataone.service.types.v1.ObjectFormat;

import org.dataone.service.types.v1.Permission;

import org.dataone.service.types.v1.Session;

import org.dataone.service.types.v1.Subject;

import org.dataone.service.types.v1.SystemMetadata;

import org.dataone.service.types.v1.util.ChecksumUtil;

import org.dataone.service.util.Constants;

import org.dspace.foresite.ResourceMap;

import org.ecoinformatics.datamanager.DataManager;

import org.ecoinformatics.datamanager.database.DatabaseConnectionPoolInterface;

import org.ecoinformatics.datamanager.parser.DataPackage;

import edu.ucsb.nceas.metacat.MetaCatServlet;

import edu.ucsb.nceas.metacat.dataquery.MetacatDatabaseConnectionPoolFactory;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.restservice.InputStreamDataSource;

/**

 * @author berkley

 * A class to populate a metacat instance based on documents returned from a query

 */

public class MetacatPopulator

{

    private String sourceUrl = null;

    private String destinationUrl = null;

    private String query = null;

    private String username = null;

    private String password = null;

    private Session session = null;

    private String subjectDN = null;

    /**

     * create a new MetacatPopulator with given source and destination urls.  

     * These should be

     * of the form "http://<url>/<metacat_instance>"

     * If username and/or password is null, the query will be run as public

     * @param sourceUrl

     * @param destUrl

     * @param query

     * @param username

     * @param password

     */

    public MetacatPopulator(String sourceUrl, String destUrl, String query, String username, String password)

    {

        this.sourceUrl = sourceUrl;

        this.query = query;

        this.username = username;

        this.password = password;

        this.destinationUrl = destUrl;

        // TODO: use specific certificate?

        this.session = null; //new Session();

        this.subjectDN = CertificateManager.getInstance().getSubjectDN(CertificateManager.getInstance().loadCertificate());

    }

    /**

     * populate from the source

     */

    public void populate()

      throws Exception

    {

        //String sourceSessionid = loginSource();

        //do a query

        String params = "returndoctype=eml://ecoinformatics.org/eml-2.1.0&" +

                        "returndoctype=eml://ecoinformatics.org/eml-2.0.1&" +

                        "returndoctype=eml://ecoinformatics.org/eml-2.0.0&";

        params += "action=query&";

        params += "qformat=xml&";

        params += "anyfield=" + query;

        printHeader("Searching source");

        System.out.println("searching '" + sourceUrl + "' for '" + query + "'");

        InputStream is = getResponse(sourceUrl, "/metacat", params, "POST");

        String response = streamToString(is);

        //System.out.println("response: " + response);

        Vector<Document> docs = parseResponse(response);

        printHeader("Parsing source results");

        System.out.println("creating MN with url: " + destinationUrl + "/");

        MNode mn = D1Client.getMN(destinationUrl + "/");

        printHeader("Processing " + docs.size() + " results.");

        for (int i=0; i<docs.size(); i++) {

        	// for generating the ORE map

            Map<Identifier, List<Identifier>> idMap = new HashMap<Identifier, List<Identifier>>();

            List<Identifier> dataIds = new ArrayList<Identifier>();

            //for each document in the query

            Document doc = docs.get(i);

            String docid = doc.docid;

            //get the doc from source

            printHeader("Getting document " + doc.docid + " from source " + sourceUrl);

            params = "action=read&qformat=xml&docid=" + docid;

            is = getResponse(sourceUrl, "/metacat", params, "POST");

            String doctext = streamToString(is);

            System.out.println("doctext: " + doctext);

            is = stringToStream(doctext);

            //parse the document

            DatabaseConnectionPoolInterface connectionPool = MetacatDatabaseConnectionPoolFactory.getDatabaseConnectionPoolInterface();

        	DataManager dataManager = DataManager.getInstance(connectionPool, connectionPool.getDBAdapterName());

        	DataPackage dataPackage = dataManager.parseMetadata(is);

            if (dataPackage == null) {

                continue;

            }

            //go through the DistributionMetadata and download any described data

            is = stringToStream(doctext);

            doc.doctext = doctext;

            printHeader("creating document on destination " + destinationUrl);            

            SystemMetadata sysmeta = generateSystemMetadata(doc);

            // iterate through the data objects

            if (dataPackage.getEntityList() != null) {

	            for (int j=0; j < dataPackage.getEntityList().length; j++) {

	                String dataDocUrl = dataPackage.getEntityList()[j].getURL();

	                String dataDocMimeType = dataPackage.getEntityList()[j].getDataFormat();

	                if (dataDocMimeType == null) {

		                dataDocMimeType = 

		                	ObjectFormatCache.getInstance().getFormat("application/octet-stream").getFormatId().getValue();

	                }

	                String dataDocLocalId = "";

	                if (dataDocUrl.trim().startsWith("ecogrid://knb/")) { //we only handle ecogrid urls right now

	                    dataDocLocalId = dataDocUrl.substring(dataDocUrl.indexOf("ecogrid://knb/") + 

	                            "ecogrid://knb/".length(), dataDocUrl.length());

	                    //get the file

	                    params = "action=read&qformat=xml&docid=" + dataDocLocalId;

	                    InputStream dataDocIs = getResponse(sourceUrl, "/metacat", params, "POST");

	                    String dataDocText = streamToString(dataDocIs);

	                    //set the id

	                    Identifier did = new Identifier();

	                    did.setValue(dataDocLocalId);

	                    // add the data identifier for ORE map 

	                    dataIds.add(did);

	                    //create sysmeta for the data doc                    

	                    SystemMetadata dataDocSysMeta = generateSystemMetadata(doc);

	                    //overwrite the bogus values from the last call 

	                    dataDocSysMeta.setIdentifier(did);

	                    ObjectFormat format = null;

	                    try {

	                    	format = ObjectFormatCache.getInstance().getFormat(dataDocMimeType);

							dataDocSysMeta.setFormatId(format.getFormatId());

	                    } catch (NotFound e) {

							System.out.println(e.getMessage());

						}

	                    dataDocIs = stringToStream(dataDocText);

	                    Checksum checksum = ChecksumUtil.checksum(dataDocIs, "MD5");

	                    dataDocSysMeta.setChecksum(checksum);

	                    String sizeStr = 

	                    	Long.toString(dataDocText.getBytes(MetaCatServlet.DEFAULT_ENCODING).length);

	                    dataDocSysMeta.setSize(new BigInteger(sizeStr));

	                    boolean error = false;

	                    //create the data doc on d1

	                    try {

	                        mn.create(session, dataDocSysMeta.getIdentifier(), IOUtils.toInputStream(dataDocText), dataDocSysMeta);

	                    }

	                    catch(Exception e) {

	                        error = true;

	                        System.out.println("ERROR: Could not create data document with id " + 

	                                dataDocSysMeta.getIdentifier().getValue() + " : " + e.getMessage());

	                    }

	                    finally {

	                        if (error) {

	                            printHeader("Insertion of document " + dataDocSysMeta.getIdentifier().getValue() + 

	                                    "FAILED.");

	                        }

	                        else {

	                            printHeader("Done inserting document " + dataDocSysMeta.getIdentifier().getValue() +

	                                " which is described by " + sysmeta.getIdentifier().getValue());

	                        }

	                    }

	                }

	                else {

	                    System.out.println("WARNING: Could not process describes url " +

	                            dataDocUrl + " for document " + doc.docid + 

	                    ".  Only ecogrid://knb/ urls are currently supported.");

	                }

	            }

            }

            try {

              Identifier id = 

            	  mn.create(session, sysmeta.getIdentifier(), IOUtils.toInputStream(doc.doctext), sysmeta);

              System.out.println("Success inserting document " + id.getValue());

              // no need for an ORE map if there's no data

              if (!dataIds.isEmpty()) {

	              // generate the ORE map for this datapackage

	              Identifier resourceMapId = new Identifier();

	              resourceMapId.setValue("resourceMap_" + sysmeta.getIdentifier().getValue());

	              idMap.put(sysmeta.getIdentifier(), dataIds);

	              ResourceMap rm = ResourceMapFactory.getInstance().createResourceMap(resourceMapId, idMap);

	              String resourceMapXML = ResourceMapFactory.getInstance().serializeResourceMap(rm);

	              Document rmDoc = new Document(resourceMapId.getValue(), "http://www.openarchives.org/ore/terms", "", "");

	              rmDoc.doctext = resourceMapXML;

	              SystemMetadata resourceMapSysMeta = generateSystemMetadata(rmDoc);

	              mn.create(session, resourceMapId, IOUtils.toInputStream(resourceMapXML), resourceMapSysMeta);

	              // clean up the permissions (FORCE public read)

	              for (Identifier dataId: dataIds) {

	            	  mn.setAccessPolicy(session, dataId, sysmeta.getAccessPolicy());

	            	  System.out.println("Set public access policy for: " + dataId.getValue());

	              }

            }

            }

            catch(Exception e) {

                e.printStackTrace();

                System.out.println("Could not create document with id " + 

                        sysmeta.getIdentifier().getValue() + " : " + e.getMessage());

            }

            finally {

                printHeader("Done processing document " + sysmeta.getIdentifier().getValue());

            }

        }

        //logout();

    }

    /**

     * @param doc

     * @return

     */

    private SystemMetadata generateSystemMetadata(Document doc)

      throws Exception {

        SystemMetadata sm = new SystemMetadata();

        sm.setSerialVersion(BigInteger.valueOf(1));

        //set the id

        Identifier id = new Identifier();

        id.setValue(doc.docid.trim());

        sm.setIdentifier(id);

        //set the object format

        ObjectFormat format = ObjectFormatCache.getInstance().getFormat(doc.doctype);

        if (format == null) {

            if (doc.doctype.trim().equals("BIN")) {

                format = ObjectFormatCache.getInstance().getFormat("application/octet-stream");

            }

            else {

                format = ObjectFormatCache.getInstance().getFormat("text/plain");

            }

        }

        sm.setFormatId(format.getFormatId());

        //create the checksum

        ByteArrayInputStream bais = new ByteArrayInputStream(doc.doctext.getBytes(MetaCatServlet.DEFAULT_ENCODING));

        Checksum checksum = ChecksumUtil.checksum(bais, "MD5");

        sm.setChecksum(checksum);

        //set the size

        String sizeStr = Long.toString(doc.doctext.getBytes(MetaCatServlet.DEFAULT_ENCODING).length);

        sm.setSize(new BigInteger(sizeStr));

        //submitter, rights holder

        Subject p = new Subject();

        p.setValue(subjectDN);

        sm.setSubmitter(p);

        sm.setRightsHolder(p);

        try {

            Date dateCreated = parseMetacatDate(doc.createDate);

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = parseMetacatDate(doc.updateDate);

            sm.setDateSysMetadataModified(dateUpdated);

        }

        catch(Exception e) {

            System.out.println("couldn't parse a date: " + e.getMessage());

            Date dateCreated = new Date();

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = new Date();

            sm.setDateSysMetadataModified(dateUpdated);

        }

        NodeReference nr = new NodeReference();

        nr.setValue(PropertyService.getProperty("dataone.memberNodeId"));

        sm.setOriginMemberNode(nr);

        sm.setAuthoritativeMemberNode(nr);

        // create access policy

        AccessPolicy accessPolicy = new AccessPolicy();

        AccessRule accessRule = new AccessRule();

		accessRule.addPermission(Permission.READ);

        Subject subject = new Subject();

        subject.setValue(Constants.SUBJECT_PUBLIC);

		accessRule.addSubject(subject);

		accessPolicy.addAllow(accessRule);

		sm.setAccessPolicy(accessPolicy);

        return sm;

    }

    private void printHeader(String s) {

        System.out.println("****** " + s + " *******");

    }

    /**

     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into

     * a proper date object

     * @param date

     * @return

     */

    private Date parseMetacatDate(String date)

    {

        String year = date.substring(0, 4);

        String month = date.substring(5, 7);

        String day = date.substring(8, 10);

        Calendar c = Calendar.getInstance();

        c.set(new Integer(year).intValue(), 

              new Integer(month).intValue(), 

              new Integer(day).intValue());

        return c.getTime();

    }

    /**

     * send a request to the resource

     */

    private InputStream sendRequest(String contextRootUrl, String resource, 

            String sessionid, String method, String urlParamaters, 

            String contentType, InputStream dataStream) 

        throws Exception 

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl + resource;

        if (urlParamaters != null) {

            if (restURL.indexOf("?") == -1)             

                restURL += "?";

            restURL += urlParamaters; 

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        if(sessionid != null)

        {

            if(restURL.indexOf("?") == -1)

            {

                restURL += "?sessionid=" + sessionid;

            }

            else

            {

                restURL += "&sessionid=" + sessionid;

            }

        }

        URL u = null;

        InputStream content = null;

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        if (contentType!=null) {

            connection.setRequestProperty("Content-Type",contentType);

        }

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        if (!method.equals("GET")) {

            if (dataStream != null) {

                OutputStream out = connection.getOutputStream();

                IOUtils.copy(dataStream, out);

            }

        }

        return connection.getInputStream();   

    }

    /**

     * create a mime multipart message from object and sysmeta

     */

    private MimeMultipart createMimeMultipart(InputStream object)

      throws Exception

    {

        final MimeMultipart mmp = new MimeMultipart();

        MimeBodyPart objectPart = new MimeBodyPart();

        objectPart.addHeaderLine("Content-Transfer-Encoding: base64");

        objectPart.setFileName("doctext");

        DataSource ds = new InputStreamDataSource("doctext", object);

        DataHandler dh = new DataHandler(ds);

        objectPart.setDataHandler(dh);

        mmp.addBodyPart(objectPart);

        return mmp;

    }

    /**

     * parse a metacat query response and return a vector of docids

     * @param response

     * @return

     */

    private Vector<Document> parseResponse(String response)

    {

        Vector<Document> v = new Vector<Document>();

        int dstart = response.indexOf("<document>");

        int dend = response.indexOf("</document>", dstart);

        while(dstart != -1)

        {

            String doc = response.substring(dstart + "<document>".length(), dend);

            //System.out.println("adding " + docid);

            Document d = new Document(getFieldFromDoc(doc, "docid"),

                    getFieldFromDoc(doc, "doctype"),

                    getFieldFromDoc(doc, "createdate"),

                    getFieldFromDoc(doc, "updatedate"));

            v.add(d);

            dstart = response.indexOf("<document>", dend);

            dend = response.indexOf("</document>", dstart);

        }

        return v;

    }

    private String getFieldFromDoc(String doc, String fieldname)

    {

        String field = "<" + fieldname + ">";

        String fieldend = "</" + fieldname + ">";

        int start = doc.indexOf(field);

        int end = doc.indexOf(fieldend);

        String s = doc.substring(start + field.length(), end);

        //System.out.println("field: " + fieldname + " : " + s);

        return s;

    }

    /**

     * login the source

     * @return

     * @throws Exception

     */

    private String loginSource()

      throws Exception

    {

        return login(sourceUrl);

    }

    /**

     * returns a sessionid

     * @return

     */

    private String login(String sourceUrl)

      throws Exception

    {

        InputStream is = getResponse(sourceUrl, "/metacat", 

                "action=login&username=" + username + "&password=" + password + "&qformat=xml", "POST");

        String response = streamToString(is);

        //System.out.println("response: " + response);

        if(response.indexOf("sessionId") == -1)

        {

            throw new Exception("Error logging into " + sourceUrl);

        }

        String sessionid = response.substring(

                response.indexOf("<sessionId>") + "<sessionId>".length(), 

                response.indexOf("</sessionId>"));

        System.out.println("sessionid: " + sessionid);

        return sessionid;

    }

    /**

     * logout both the source and destination

     * @throws Exception

     */

    private void logout()

        throws Exception

    {

        getResponse(sourceUrl, "/metacat", "action=logout&username=" + username, "POST");

    }

    /**

     * get an http response

     * @param contextRootUrl

     * @param resource

     * @param urlParameters

     * @param method

     * @return

     * @throws Exception

     */

    private InputStream getResponse(String contextRootUrl, String resource, 

            String urlParameters, String method)

      throws Exception

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl+resource;

        if (urlParameters != null) {

            if (restURL.indexOf("?") == -1)             

                restURL += "?";

            restURL += urlParameters; 

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        URL u = null;

        InputStream content = null;            

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        content = connection.getInputStream();

        return content;

    }

    private String streamToString(InputStream is)

        throws Exception

    {

        byte b[] = new byte[1024];

        int numread = is.read(b, 0, 1024);

        String response = new String();

        while(numread != -1)

        {

            response += new String(b, 0, numread);

            numread = is.read(b, 0, 1024);

        }

        return response;

    }

    private InputStream stringToStream(String s)

      throws Exception

    {

        ByteArrayInputStream bais = new ByteArrayInputStream(s.getBytes(MetaCatServlet.DEFAULT_ENCODING));

        return bais;

    }

    private class Document

    {

        public String docid;

        public String doctype;

        public String createDate;

        public String updateDate;

        public String doctext;

        public Document(String docid, String doctype, String createDate, String updateDate)

        {

            this.docid = docid.trim();

            this.doctype = doctype.trim();

            this.createDate = createDate.trim();

            this.updateDate = updateDate.trim();

        }

    }

}