Metacat

/**

 *  '$RCSfile$'

 *    Purpose: A Class that implements administrative methods 

 *  Copyright: 2010 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Michael Daigle

 * 

 *   '$Author: berkley $'

 *     '$Date: 2010-06-08 12:34:30 -0700 (Tue, 08 Jun 2010) $'

 * '$Revision: 5374 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.util;

import java.security.MessageDigest;

import java.util.*;

import java.io.*;

import java.net.*;

import javax.activation.DataHandler;

import javax.activation.DataSource;

import javax.mail.MessagingException;

import javax.mail.internet.MimeBodyPart;

import javax.mail.internet.MimeMultipart;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.io.IOUtils;

import edu.ucsb.nceas.metacat.MetacatHandler;

import edu.ucsb.nceas.metacat.MetacatResultSet;

import edu.ucsb.nceas.metacat.MetacatResultSet.Document;

import edu.ucsb.nceas.metacat.dataone.CrudService;

import edu.ucsb.nceas.metacat.restservice.InputStreamDataSource;

import org.dataone.service.exceptions.InvalidSystemMetadata;

import org.dataone.service.exceptions.ServiceFailure;

import org.dataone.service.types.AuthToken;

import org.dataone.service.types.Checksum;

import org.dataone.service.types.ChecksumAlgorithm;

import org.dataone.service.types.NodeReference;

import org.dataone.service.types.ObjectFormat;

import org.dataone.service.types.Principal;

import org.dataone.service.types.SystemMetadata;

import org.dataone.service.types.Identifier;

import org.dataone.client.D1Client;

//import sun.tools.jstat.Identifier;

import com.gc.iotools.stream.is.InputStreamFromOutputStream;

/**

 * @author berkley

 * A class to populate a metacat instance based on documents returned from a query

 */

public class MetacatPopulator

{

    private String sourceUrl = null;

    private String destinationUrl = null;

    private String query = null;

    private String username = null;

    private String password = null;

    /**

     * create a new MetacatPopulator with given source and destination urls.  

     * These should be

     * of the form "http://<url>/<metacat_instance>"

     * If username and/or password is null, the query will be run as public

     * @param sourceUrl

     * @param destUrl

     * @param query

     * @param username

     * @param password

     */

    public MetacatPopulator(String sourceUrl, String destUrl, String query, String username, String password)

    {

        this.sourceUrl = sourceUrl;

        this.query = query;

        this.username = username;

        this.password = password;

        this.destinationUrl = destUrl;

    }

    /**

     * populate from the source

     */

    public void populate()

      throws Exception

    {

        printHeader("Source login");

        String sourceSessionid = loginSource();

        //do a query

        String params = "returndoctype=eml://ecoinformatics.org/eml-2.0.1&" +

                        "returndoctype=eml://ecoinformatics.org/eml-2.0.0&" +

                        "returndoctype=BIN&" +

                        "returndoctype=http://dataone.org/service/types/SystemMetadata/0.1&";

        params += "action=query&";

        params += "qformat=xml&";

        params += "anyfield=" + query;

        printHeader("Searching source");

        System.out.println("searching '" + sourceUrl + "' for '" + query + "' with sessionid '" + sourceSessionid + "'");

        InputStream is = getResponse(sourceUrl, "/metacat",

                params, "POST");

        String response = streamToString(is);

        //System.out.println("response: " + response);

        Vector<Document> docs = parseResponse(response);

        printHeader("Parsing source results");

        D1Client d1 = new D1Client(destinationUrl + "/");

        printHeader("Processing " + docs.size() + " results.");

        printHeader("logging in to the destination " + destinationUrl);

        AuthToken authtoken = d1.login(username, password);

        for(int i=0; i<docs.size(); i++)

        {

            //for each document in the query

            Document doc = docs.get(i);

            String docid = doc.docid;

            //get the doc from source

            printHeader("Getting document " + doc.docid + " from source " + sourceUrl);

            params = "action=read&qformat=xml&docid=" + docid;

            is = getResponse(sourceUrl, "/metacat", params, "POST");

            String doctext = streamToString(is);

            //System.out.println("Done retrieving document: " + doctext);

            is = stringToStream(doctext);

            doc.doctext = doctext;

            printHeader("creating document on destination " + destinationUrl);            

            SystemMetadata sysmeta = generateSystemMetadata(doc);

            try

            {

              Identifier id = d1.create(authtoken, sysmeta.getIdentifier(), 

                    IOUtils.toInputStream(doc.doctext), sysmeta);

              System.out.println("Success inserting document " + id.getValue());

            }

            catch(Exception e)

            {

                System.out.println("Could not create document with id " + 

                        sysmeta.getIdentifier().getValue() + " : " + e.getMessage());

            }

            finally

            {

                printHeader("Done inserting document " + sysmeta.getIdentifier().getValue());

            }

        }

        logout();

    }

    private void printHeader(String s)

    {

        System.out.println("****** " + s + " *******");

    }

    /**

     * produce an md5 checksum for item

     */

    private String checksum(InputStream is)

      throws Exception

    {        

        byte[] buffer = new byte[1024];

        MessageDigest complete = MessageDigest.getInstance("MD5");

        int numRead;

        do 

        {

          numRead = is.read(buffer);

          if (numRead > 0) 

          {

            complete.update(buffer, 0, numRead);

          }

        } while (numRead != -1);

        return getHex(complete.digest());

    }

    /**

     * convert a byte array to a hex string

     */

    private static String getHex( byte [] raw ) 

    {

        final String HEXES = "0123456789ABCDEF";

        if ( raw == null ) {

          return null;

        }

        final StringBuilder hex = new StringBuilder( 2 * raw.length );

        for ( final byte b : raw ) {

          hex.append(HEXES.charAt((b & 0xF0) >> 4))

             .append(HEXES.charAt((b & 0x0F)));

        }

        return hex.toString();

    }

    /**

     * @param doc

     * @return

     */

    private SystemMetadata generateSystemMetadata(Document doc)

      throws Exception

    {

        SystemMetadata sm = new SystemMetadata();

        //set the id

        Identifier id = new Identifier();

        id.setValue(doc.docid.trim());

        sm.setIdentifier(id);

        //set the object format

        ObjectFormat format = ObjectFormat.convert(doc.doctype);

        if(format == null)

        {

            if(doc.doctype.trim().equals("BIN"))

            {

                format = ObjectFormat.OCTET_STREAM;

            }

            else

            {

                format = ObjectFormat.convert("text/plain");

            }

        }

        sm.setObjectFormat(format);

        //create the checksum

        ByteArrayInputStream bais = new ByteArrayInputStream(doc.doctext.getBytes());

        String checksumS = checksum(bais);

        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");

        Checksum checksum = new Checksum();

        checksum.setValue(checksumS);

        checksum.setAlgorithm(ca);

        sm.setChecksum(checksum);

        //set the size

        sm.setSize(doc.doctext.getBytes().length);

        //submitter

        Principal p = new Principal();

        p.setValue("unknown");

        sm.setSubmitter(p);

        sm.setRightsHolder(p);

        try

        {

            Date dateCreated = parseMetacatDate(doc.createDate);

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = parseMetacatDate(doc.updateDate);

            sm.setDateSysMetadataModified(dateUpdated);

        }

        catch(Exception e)

        {

            System.out.println("couldn't parse a date: " + e.getMessage());

            Date dateCreated = new Date();

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = new Date();

            sm.setDateSysMetadataModified(dateUpdated);

        }

        NodeReference nr = new NodeReference();

        nr.setValue(sourceUrl);

        sm.setOriginMemberNode(nr);

        sm.setAuthoritativeMemberNode(nr);

        return sm;

    }

    /**

     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into

     * a proper date object

     * @param date

     * @return

     */

    private Date parseMetacatDate(String date)

    {

        String year = date.substring(0, 4);

        String month = date.substring(5, 7);

        String day = date.substring(8, 10);

        Calendar c = Calendar.getInstance();

        c.set(new Integer(year).intValue(), 

              new Integer(month).intValue(), 

              new Integer(day).intValue());

        return c.getTime();

    }

    /**

     * send a request to the resource

     */

    private InputStream sendRequest(String contextRootUrl, String resource, 

            String sessionid, String method, String urlParamaters, 

            String contentType, InputStream dataStream) 

        throws Exception 

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl + resource;

        if (urlParamaters != null) {

            if (restURL.indexOf("?") == -1)             

                restURL += "?";

            restURL += urlParamaters; 

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        if(sessionid != null)

        {

            if(restURL.indexOf("?") == -1)

            {

                restURL += "?sessionid=" + sessionid;

            }

            else

            {

                restURL += "&sessionid=" + sessionid;

            }

        }

        URL u = null;

        InputStream content = null;

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        if (contentType!=null) {

            connection.setRequestProperty("Content-Type",contentType);

        }

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        if (!method.equals("GET")) {

            if (dataStream != null) {

                OutputStream out = connection.getOutputStream();

                IOUtils.copy(dataStream, out);

            }

        }

        return connection.getInputStream();   

    }

    /**

     * create a mime multipart message from object and sysmeta

     */

    private MimeMultipart createMimeMultipart(InputStream object)

      throws Exception

    {

        final MimeMultipart mmp = new MimeMultipart();

        MimeBodyPart objectPart = new MimeBodyPart();

        objectPart.addHeaderLine("Content-Transfer-Encoding: base64");

        objectPart.setFileName("doctext");

        DataSource ds = new InputStreamDataSource("doctext", object);

        DataHandler dh = new DataHandler(ds);

        objectPart.setDataHandler(dh);

        mmp.addBodyPart(objectPart);

        return mmp;

    }

    /**

     * parse a metacat query response and return a vector of docids

     * @param response

     * @return

     */

    private Vector<Document> parseResponse(String response)

    {

        Vector<Document> v = new Vector<Document>();

        int dstart = response.indexOf("<document>");

        int dend = response.indexOf("</document>", dstart);

        while(dstart != -1)

        {

            String doc = response.substring(dstart + "<document>".length(), dend);

            //System.out.println("adding " + docid);

            Document d = new Document(getFieldFromDoc(doc, "docid"),

                    getFieldFromDoc(doc, "doctype"),

                    getFieldFromDoc(doc, "createdate"),

                    getFieldFromDoc(doc, "updatedate"));

            v.add(d);

            dstart = response.indexOf("<document>", dend);

            dend = response.indexOf("</document>", dstart);

        }

        return v;

    }

    private String getFieldFromDoc(String doc, String fieldname)

    {

        String field = "<" + fieldname + ">";

        String fieldend = "</" + fieldname + ">";

        int start = doc.indexOf(field);

        int end = doc.indexOf(fieldend);

        String s = doc.substring(start + field.length(), end);

        //System.out.println("field: " + fieldname + " : " + s);

        return s;

    }

    /**

     * login the source

     * @return

     * @throws Exception

     */

    private String loginSource()

      throws Exception

    {

        return login(sourceUrl);

    }

    /**

     * login the destination

     * @return

     * @throws Exception

     */

    private String loginDest()

        throws Exception

    {

        return login(destinationUrl);

    }

    /**

     * returns a sessionid

     * @return

     */

    private String login(String sourceUrl)

      throws Exception

    {

        InputStream is = getResponse(sourceUrl, "/metacat", 

                "action=login&username=" + username + "&password=" + password + "&qformat=xml", 

        "POST");

        String response = streamToString(is);

        //System.out.println("response: " + response);

        if(response.indexOf("sessionId") == -1)

        {

            throw new Exception("Error logging into " + sourceUrl);

        }

        String sessionid = response.substring(

                response.indexOf("<sessionId>") + "<sessionId>".length(), 

                response.indexOf("</sessionId>"));

        System.out.println("sessionid: " + sessionid);

        return sessionid;

    }

    /**

     * logout both the source and destination

     * @throws Exception

     */

    private void logout()

        throws Exception

    {

        getResponse(sourceUrl, "/metacat", "action=logout&username=" + username, "POST");

        getResponse(destinationUrl, "/metacat", "action=logout&username=" + username, "POST");

    }

    /**

     * get an http response

     * @param contextRootUrl

     * @param resource

     * @param urlParameters

     * @param method

     * @return

     * @throws Exception

     */

    private InputStream getResponse(String contextRootUrl, String resource, 

            String urlParameters, String method)

      throws Exception

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl+resource;

        if (urlParameters != null) {

            if (restURL.indexOf("?") == -1)             

                restURL += "?";

            restURL += urlParameters; 

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        URL u = null;

        InputStream content = null;            

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        content = connection.getInputStream();

        return content;

    }

    private String streamToString(InputStream is)

        throws Exception

    {

        byte b[] = new byte[1024];

        int numread = is.read(b, 0, 1024);

        String response = new String();

        while(numread != -1)

        {

            response += new String(b, 0, numread);

            numread = is.read(b, 0, 1024);

        }

        return response;

    }

    private InputStream stringToStream(String s)

      throws Exception

    {

        ByteArrayInputStream bais = new ByteArrayInputStream(s.getBytes());

        return bais;

    }

    private class Document

    {

        public String docid;

        public String doctype;

        public String createDate;

        public String updateDate;

        public String doctext;

        public Document(String docid, String doctype, String createDate, String updateDate)

        {

            this.docid = docid.trim();

            this.doctype = doctype.trim();

            this.createDate = createDate.trim();

            this.updateDate = updateDate.trim();

        }

    }

}