Metacat

/**

 *  '$RCSfile$'

 *    Purpose: A Class that implements administrative methods 

 *  Copyright: 2010 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Michael Daigle

 * 

 *   '$Author: berkley $'

 *     '$Date: 2010-06-08 12:34:30 -0700 (Tue, 08 Jun 2010) $'

 * '$Revision: 5374 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.util;

import java.security.MessageDigest;

import java.util.*;

import java.io.*;

import java.net.*;

import javax.activation.DataHandler;

import javax.activation.DataSource;

import javax.mail.MessagingException;

import javax.mail.internet.MimeBodyPart;

import javax.mail.internet.MimeMultipart;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.io.IOUtils;

import edu.ucsb.nceas.metacat.MetacatHandler;

import edu.ucsb.nceas.metacat.MetacatResultSet;

import edu.ucsb.nceas.metacat.MetacatResultSet.Document;

import edu.ucsb.nceas.metacat.dataone.CrudService;

import edu.ucsb.nceas.metacat.restservice.InputStreamDataSource;

import org.dataone.service.exceptions.InvalidSystemMetadata;

import org.dataone.service.exceptions.ServiceFailure;

import org.dataone.service.types.AuthToken;

import org.dataone.service.types.Checksum;

import org.dataone.service.types.ChecksumAlgorithm;

import org.dataone.service.types.NodeReference;

import org.dataone.service.types.ObjectFormat;

import org.dataone.service.types.Principal;

import org.dataone.service.types.SystemMetadata;

import org.dataone.service.types.Identifier;

import org.dataone.client.D1Client;

import org.dataone.client.MNode;

import org.dataone.eml.DataoneEMLParser;

import org.dataone.eml.EMLDocument;

import org.dataone.eml.EMLDocument.DistributionMetadata;

//import sun.tools.jstat.Identifier;

import com.gc.iotools.stream.is.InputStreamFromOutputStream;

/**

 * @author berkley

 * A class to populate a metacat instance based on documents returned from a query

 */

public class MetacatPopulator

{

    private String sourceUrl = null;

    private String destinationUrl = null;

    private String query = null;

    private String username = null;

    private String password = null;

    /**

     * create a new MetacatPopulator with given source and destination urls.  

     * These should be

     * of the form "http://<url>/<metacat_instance>"

     * If username and/or password is null, the query will be run as public

     * @param sourceUrl

     * @param destUrl

     * @param query

     * @param username

     * @param password

     */

    public MetacatPopulator(String sourceUrl, String destUrl, String query, String username, String password)

    {

        this.sourceUrl = sourceUrl;

        this.query = query;

        this.username = username;

        this.password = password;

        this.destinationUrl = destUrl;

    }

    /**

     * populate from the source

     */

    public void populate()

      throws Exception

    {

        printHeader("Source login");

        String sourceSessionid = loginSource();

        //do a query

        String params = "returndoctype=eml://ecoinformatics.org/eml-2.1.0&" +

                        "returndoctype=eml://ecoinformatics.org/eml-2.0.1&" +

                        "returndoctype=eml://ecoinformatics.org/eml-2.0.0&";

        params += "action=query&";

        params += "qformat=xml&";

        params += "anyfield=" + query;

        printHeader("Searching source");

        System.out.println("searching '" + sourceUrl + "' for '" + query + "' with sessionid '" + sourceSessionid + "'");

        InputStream is = getResponse(sourceUrl, "/metacat",

                params, "POST");

        String response = streamToString(is);

        //System.out.println("response: " + response);

        Vector<Document> docs = parseResponse(response);

        printHeader("Parsing source results");

        D1Client d1 = new D1Client(destinationUrl + "/");

        MNode mn = d1.getMN(destinationUrl + "/");

        printHeader("Processing " + docs.size() + " results.");

        printHeader("logging in to the destination " + destinationUrl);

        AuthToken authtoken = mn.login(username, password);

        for(int i=0; i<docs.size(); i++)

        {

            //for each document in the query

            Document doc = docs.get(i);

            String docid = doc.docid;

            //get the doc from source

            printHeader("Getting document " + doc.docid + " from source " + sourceUrl);

            params = "action=read&qformat=xml&docid=" + docid;

            is = getResponse(sourceUrl, "/metacat", params, "POST");

            String doctext = streamToString(is);

            System.out.println("doctext: " + doctext);

            is = stringToStream(doctext);

            //parse the document

            DataoneEMLParser parser = DataoneEMLParser.getInstance();

            EMLDocument emld = parser.parseDocument(is);

            if(emld == null)

            {

                continue;

            }

            //go through the DistributionMetadata and download any described data

            is = stringToStream(doctext);

            doc.doctext = doctext;

            printHeader("creating document on destination " + destinationUrl);            

            SystemMetadata sysmeta = generateSystemMetadata(doc);

            for(int j=0; j<emld.distributionMetadata.size(); j++)

            {

                Identifier emlId = sysmeta.getIdentifier();

                DistributionMetadata dm = emld.distributionMetadata.elementAt(j);

                String dataDocUrl = dm.url;

                String dataDocMimeType = dm.mimeType;

                String dataDocLocalId = "";

                if(dataDocUrl.trim().startsWith("ecogrid://knb/"))

                { //we only handle ecogrid urls right now

                    dataDocLocalId = dataDocUrl.substring(dataDocUrl.indexOf("ecogrid://knb/") + 

                            "ecogrid://knb/".length(), dataDocUrl.length());

                    //get the file

                    params = "action=read&qformat=xml&docid=" + dataDocLocalId;

                    InputStream dataDocIs = getResponse(sourceUrl, "/metacat", params, "POST");

                    String dataDocText = streamToString(dataDocIs);

                    //set the id

                    Identifier did = new Identifier();

                    did.setValue(dataDocLocalId);

                    //add the desribeby to the eml's sysmeta

                    System.out.println("adding describe for doc " + 

                            sysmeta.getIdentifier().getValue() + " :" + did.getValue());

                    sysmeta.addDescribe(did);

                    //create sysmeta for the data doc                    

                    SystemMetadata dataDocSysMeta = generateSystemMetadata(doc);

                    //overwrite the bogus values from the last call 

                    dataDocSysMeta.setIdentifier(did);

                    dataDocSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));

                    Checksum checksum = new Checksum();

                    dataDocIs = stringToStream(dataDocText);

                    ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");

                    checksum.setAlgorithm(ca);

                    checksum.setValue(checksum(dataDocIs));

                    dataDocSysMeta.setChecksum(checksum);

                    dataDocSysMeta.setSize(dataDocText.getBytes().length);

                    dataDocSysMeta.addDescribedBy(sysmeta.getIdentifier());

                    boolean error = false;

                    //create the data doc on d1

                    try

                    {

                        mn.create(authtoken, dataDocSysMeta.getIdentifier(), IOUtils.toInputStream(dataDocText), dataDocSysMeta);

                        mn.setAccess(authtoken, dataDocSysMeta.getIdentifier(), "public", "read", "allow", "allowFirst");

                    }

                    catch(Exception e)

                    {

                        error = true;

                        System.out.println("ERROR: Could not create data document with id " + 

                                dataDocSysMeta.getIdentifier().getValue() + " : " + e.getMessage());

                    }

                    finally

                    {

                        if(error)

                        {

                            printHeader("Insertion of document " + dataDocSysMeta.getIdentifier().getValue() + 

                                    "FAILED.");

                        }

                        else

                        {

                            printHeader("Done inserting document " + dataDocSysMeta.getIdentifier().getValue() +

                                " which is described by " + sysmeta.getIdentifier().getValue());

                        }

                    }

                }

                else

                {

                    System.out.println("WARNING: Could not process describes url " +

                            dataDocUrl + " for document " + doc.docid + 

                    ".  Only ecogrid://knb/ urls are currently supported.");

                }

            }

            try

            {

              Identifier id = mn.create(authtoken, sysmeta.getIdentifier(), 

                    IOUtils.toInputStream(doc.doctext), sysmeta);

              System.out.println("Success inserting document " + id.getValue());

            }

            catch(Exception e)

            {

                e.printStackTrace();

                System.out.println("Could not create document with id " + 

                        sysmeta.getIdentifier().getValue() + " : " + e.getMessage());

            }

            finally

            {

                printHeader("Done inserting document " + sysmeta.getIdentifier().getValue());

            }

        }

        logout();

    }

    /**

     * create the documents listed by an eml document as described in the 

     * new system

     * @param doc

     * @param emld

     */

    private void createDescribedDocuments(Document doc, EMLDocument emld)

    {

    }

    /**

     * @param doc

     * @return

     */

    private SystemMetadata generateSystemMetadata(Document doc)

      throws Exception

    {

        SystemMetadata sm = new SystemMetadata();

        //set the id

        Identifier id = new Identifier();

        id.setValue(doc.docid.trim());

        sm.setIdentifier(id);

        //set the object format

        ObjectFormat format = ObjectFormat.convert(doc.doctype);

        if(format == null)

        {

            if(doc.doctype.trim().equals("BIN"))

            {

                format = ObjectFormat.OCTET_STREAM;

            }

            else

            {

                format = ObjectFormat.TEXT_PLAIN;

            }

        }

        sm.setObjectFormat(format);

        //create the checksum

        ByteArrayInputStream bais = new ByteArrayInputStream(doc.doctext.getBytes());

        String checksumS = checksum(bais);

        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");

        Checksum checksum = new Checksum();

        checksum.setValue(checksumS);

        checksum.setAlgorithm(ca);

        sm.setChecksum(checksum);

        //set the size

        sm.setSize(doc.doctext.getBytes().length);

        //submitter

        Principal p = new Principal();

        p.setValue("unknown");

        sm.setSubmitter(p);

        sm.setRightsHolder(p);

        try

        {

            Date dateCreated = parseMetacatDate(doc.createDate);

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = parseMetacatDate(doc.updateDate);

            sm.setDateSysMetadataModified(dateUpdated);

        }

        catch(Exception e)

        {

            System.out.println("couldn't parse a date: " + e.getMessage());

            Date dateCreated = new Date();

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = new Date();

            sm.setDateSysMetadataModified(dateUpdated);

        }

        NodeReference nr = new NodeReference();

        nr.setValue("KNB");

        sm.setOriginMemberNode(nr);

        sm.setAuthoritativeMemberNode(nr);

        return sm;

    }

    private void printHeader(String s)

    {

        System.out.println("****** " + s + " *******");

    }

    /**

     * produce an md5 checksum for item

     */

    private String checksum(InputStream is)

      throws Exception

    {        

        byte[] buffer = new byte[1024];

        MessageDigest complete = MessageDigest.getInstance("MD5");

        int numRead;

        do 

        {

          numRead = is.read(buffer);

          if (numRead > 0) 

          {

            complete.update(buffer, 0, numRead);

          }

        } while (numRead != -1);

        return getHex(complete.digest());

    }

    /**

     * convert a byte array to a hex string

     */

    private static String getHex( byte [] raw ) 

    {

        final String HEXES = "0123456789ABCDEF";

        if ( raw == null ) {

          return null;

        }

        final StringBuilder hex = new StringBuilder( 2 * raw.length );

        for ( final byte b : raw ) {

          hex.append(HEXES.charAt((b & 0xF0) >> 4))

             .append(HEXES.charAt((b & 0x0F)));

        }

        return hex.toString();

    }

    /**

     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into

     * a proper date object

     * @param date

     * @return

     */

    private Date parseMetacatDate(String date)

    {

        String year = date.substring(0, 4);

        String month = date.substring(5, 7);

        String day = date.substring(8, 10);

        Calendar c = Calendar.getInstance();

        c.set(new Integer(year).intValue(), 

              new Integer(month).intValue(), 

              new Integer(day).intValue());

        return c.getTime();

    }

    /**

     * send a request to the resource

     */

    private InputStream sendRequest(String contextRootUrl, String resource, 

            String sessionid, String method, String urlParamaters, 

            String contentType, InputStream dataStream) 

        throws Exception 

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl + resource;

        if (urlParamaters != null) {

            if (restURL.indexOf("?") == -1)             

                restURL += "?";

            restURL += urlParamaters; 

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        if(sessionid != null)

        {

            if(restURL.indexOf("?") == -1)

            {

                restURL += "?sessionid=" + sessionid;

            }

            else

            {

                restURL += "&sessionid=" + sessionid;

            }

        }

        URL u = null;

        InputStream content = null;

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        if (contentType!=null) {

            connection.setRequestProperty("Content-Type",contentType);

        }

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        if (!method.equals("GET")) {

            if (dataStream != null) {

                OutputStream out = connection.getOutputStream();

                IOUtils.copy(dataStream, out);

            }

        }

        return connection.getInputStream();   

    }

    /**

     * create a mime multipart message from object and sysmeta

     */

    private MimeMultipart createMimeMultipart(InputStream object)

      throws Exception

    {

        final MimeMultipart mmp = new MimeMultipart();

        MimeBodyPart objectPart = new MimeBodyPart();

        objectPart.addHeaderLine("Content-Transfer-Encoding: base64");

        objectPart.setFileName("doctext");

        DataSource ds = new InputStreamDataSource("doctext", object);

        DataHandler dh = new DataHandler(ds);

        objectPart.setDataHandler(dh);

        mmp.addBodyPart(objectPart);

        return mmp;

    }

    /**

     * parse a metacat query response and return a vector of docids

     * @param response

     * @return

     */

    private Vector<Document> parseResponse(String response)

    {

        Vector<Document> v = new Vector<Document>();

        int dstart = response.indexOf("<document>");

        int dend = response.indexOf("</document>", dstart);

        while(dstart != -1)

        {

            String doc = response.substring(dstart + "<document>".length(), dend);

            //System.out.println("adding " + docid);

            Document d = new Document(getFieldFromDoc(doc, "docid"),

                    getFieldFromDoc(doc, "doctype"),

                    getFieldFromDoc(doc, "createdate"),

                    getFieldFromDoc(doc, "updatedate"));

            v.add(d);

            dstart = response.indexOf("<document>", dend);

            dend = response.indexOf("</document>", dstart);

        }

        return v;

    }

    private String getFieldFromDoc(String doc, String fieldname)

    {

        String field = "<" + fieldname + ">";

        String fieldend = "</" + fieldname + ">";

        int start = doc.indexOf(field);

        int end = doc.indexOf(fieldend);

        String s = doc.substring(start + field.length(), end);

        //System.out.println("field: " + fieldname + " : " + s);

        return s;

    }

    /**

     * login the source

     * @return

     * @throws Exception

     */

    private String loginSource()

      throws Exception

    {

        return login(sourceUrl);

    }

    /**

     * login the destination

     * @return

     * @throws Exception

     */

    private String loginDest()

        throws Exception

    {

        return login(destinationUrl);

    }

    /**

     * returns a sessionid

     * @return

     */

    private String login(String sourceUrl)

      throws Exception

    {

        InputStream is = getResponse(sourceUrl, "/metacat", 

                "action=login&username=" + username + "&password=" + password + "&qformat=xml", 

        "POST");

        String response = streamToString(is);

        //System.out.println("response: " + response);

        if(response.indexOf("sessionId") == -1)

        {

            throw new Exception("Error logging into " + sourceUrl);

        }

        String sessionid = response.substring(

                response.indexOf("<sessionId>") + "<sessionId>".length(), 

                response.indexOf("</sessionId>"));

        System.out.println("sessionid: " + sessionid);

        return sessionid;

    }

    /**

     * logout both the source and destination

     * @throws Exception

     */

    private void logout()

        throws Exception

    {

        getResponse(sourceUrl, "/metacat", "action=logout&username=" + username, "POST");

        getResponse(destinationUrl, "/metacat", "action=logout&username=" + username, "POST");

    }

    /**

     * get an http response

     * @param contextRootUrl

     * @param resource

     * @param urlParameters

     * @param method

     * @return

     * @throws Exception

     */

    private InputStream getResponse(String contextRootUrl, String resource, 

            String urlParameters, String method)

      throws Exception

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl+resource;

        if (urlParameters != null) {

            if (restURL.indexOf("?") == -1)             

                restURL += "?";

            restURL += urlParameters; 

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        URL u = null;

        InputStream content = null;            

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        content = connection.getInputStream();

        return content;

    }

    private String streamToString(InputStream is)

        throws Exception

    {

        byte b[] = new byte[1024];

        int numread = is.read(b, 0, 1024);

        String response = new String();

        while(numread != -1)

        {

            response += new String(b, 0, numread);

            numread = is.read(b, 0, 1024);

        }

        return response;

    }

    private InputStream stringToStream(String s)

      throws Exception

    {

        ByteArrayInputStream bais = new ByteArrayInputStream(s.getBytes());

        return bais;

    }

    private class Document

    {

        public String docid;

        public String doctype;

        public String createDate;

        public String updateDate;

        public String doctext;

        public Document(String docid, String doctype, String createDate, String updateDate)

        {

            this.docid = docid.trim();

            this.doctype = doctype.trim();

            this.createDate = createDate.trim();

            this.updateDate = updateDate.trim();

        }

    }

}