Metacat

/*  '$RCSfile$'

 *  Copyright: 2010 Regents of the University of California and the

 *              National Center for Ecological Analysis and Synthesis

 *  Purpose: To test the Access Controls in metacat by JUnit

 *

 *   '$Author: berkley $'

 *     '$Date: 2010-05-03 14:26:08 -0700 (Fri, 14 Aug 2009) $'

 * '$Revision: 5027 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.dataone;

import java.util.*;

import java.io.*;

import java.security.MessageDigest;

import edu.ucsb.nceas.MCTestCase;

import edu.ucsb.nceas.metacat.IdentifierManager;

import edu.ucsb.nceas.metacat.client.MetacatAuthException;

import edu.ucsb.nceas.metacat.client.MetacatException;

import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;

import edu.ucsb.nceas.metacat.util.MetacatPopulator;

import edu.ucsb.nceas.metacat.dataone.CrudService;

import junit.framework.Test;

import junit.framework.TestSuite;

import org.apache.log.LogEvent;

import org.dataone.service.exceptions.InvalidRequest;

import org.dataone.service.exceptions.InvalidToken;

import org.dataone.service.exceptions.NotAuthorized;

import org.dataone.service.exceptions.NotImplemented;

import org.dataone.service.exceptions.ServiceFailure;

import org.dataone.service.types.*;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.client.rest.MetacatRestClient;

import edu.ucsb.nceas.metacat.service.SessionService;

import edu.ucsb.nceas.metacat.util.SessionData;

/**

 * A JUnit test for testing the dataone MetacatPopulator class

 */

public class MetacatPopulatorTest extends MCTestCase 

{   

    /**

    * consstructor for the test

    */

    public MetacatPopulatorTest(String name)

    {

        super(name);

    }

    /**

     * Establish a testing framework by initializing appropriate objects

     */

    public void setUp() throws Exception 

    {

        super.setUp();

    }

    /**

     * Release any objects after tests are complete

     */

    public void tearDown() 

    {

    }

    /**

     * Create a suite of tests to be run together

     */

    public static Test suite() 

    {

        TestSuite suite = new TestSuite();

        suite.addTest(new MetacatPopulatorTest("initialize"));

        suite.addTest(new MetacatPopulatorTest("testPopulate"));

        return suite;

    }

    public void testPopulate()

    {

        try

        {

            MetacatPopulator mp = new MetacatPopulator(

                    "http://knb.ecoinformatics.org/knb", 

                    "http://knb-mn.ecoinformatics.org/knb", 

                    "knb-lter-and.2722", "uid=kepler,o=unaffiliated,dc=ecoinformatics,dc=org", "kepler");

            mp.populate();

        }

        catch(Exception e)

        {

            e.printStackTrace();

            fail("Error: " + e.getMessage());

        }

    }

    /**

     * init

     */

    public void initialize()

    {

        assertTrue(1==1);

    }

}

log4j.rootLogger=ERROR, stdout

log4j.appender.d1.layout.ConversionPattern=@build.context@ %d{yyyyMMdd-HH:mm:ss}: [%p]: %m [%c]%n

    private String streamToString(InputStream is)

    throws IOException

    {

        byte b[] = new byte[1024];

        int numread = is.read(b, 0, 1024);

        String response = new String();

        while(numread != -1)

        {

            response += new String(b, 0, numread);

            numread = is.read(b, 0, 1024);

        }

        return response;

    }

    private InputStream stringToStream(String s)

    throws IOException

    {

        ByteArrayInputStream bais = new ByteArrayInputStream(s.getBytes());

        return bais;

    }

            String s = streamToString(request.getInputStream());

            String searchString = "Content-Disposition: attachment; filename=systemmetadata";

            String endString = "------=_Part";

            int searchStringIndex = s.indexOf(searchString);

            sysmeta = new ByteArrayInputStream(

                    s.substring(searchStringIndex +

                            searchString.length(), 

                            s.indexOf(endString, searchStringIndex)).trim().getBytes());

            String txt = streamToString(sysmeta);

            System.out.println("sysmeta: " + txt);

            sysmeta = stringToStream(txt);

            searchString = "Content-Disposition: attachment; filename=object";

            searchStringIndex = s.indexOf(searchString);

            object = new ByteArrayInputStream(

                    s.substring(searchStringIndex +

                            searchString.length(), 

                            s.indexOf(endString, searchStringIndex)).trim().getBytes());

            txt = streamToString(object);

            System.out.println("object: " + txt);

            sysmeta = stringToStream(txt);

            //System.out.println("request stream: " + s);

            //Since mmp seems to have a bug where large object parts get truncated,

            //parse the stream here.  This has the disavantage of putting the

            //stream into memory.

            /*

            //mmp.writeTo(System.out);

                    try

                    {

                        String s = streamToString(object);

                        System.out.println("doc text: " + s);

                        System.out.println("size of doc text is: " + s.getBytes().length);

                        object = stringToStream(s);

                    }

                    catch(Exception e)

                    {}

                    System.out.println("Found object part, size is: " + part.getSize());

                    try

                    {

                      String s = streamToString(sysmeta);

                      System.out.println("system metadata part: " + s);

                      sysmeta = stringToStream(s);

                    }

                    catch(Exception e){}

                    System.out.println("system metadata part, size is " + part.getSize());

            }*/

        } /*catch (MessagingException e) {

        }*/ catch (IOException e) {

/**

 *  '$RCSfile$'

 *    Purpose: A Class that implements administrative methods 

 *  Copyright: 2010 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Michael Daigle

 * 

 *   '$Author: berkley $'

 *     '$Date: 2010-06-08 12:34:30 -0700 (Tue, 08 Jun 2010) $'

 * '$Revision: 5374 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.util;

import java.security.MessageDigest;

import java.util.*;

import java.io.*;

import java.net.*;

import javax.activation.DataHandler;

import javax.activation.DataSource;

import javax.mail.MessagingException;

import javax.mail.internet.MimeBodyPart;

import javax.mail.internet.MimeMultipart;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.io.IOUtils;

import edu.ucsb.nceas.metacat.MetacatHandler;

import edu.ucsb.nceas.metacat.MetacatResultSet;

import edu.ucsb.nceas.metacat.MetacatResultSet.Document;

import edu.ucsb.nceas.metacat.dataone.CrudService;

import edu.ucsb.nceas.metacat.restservice.InputStreamDataSource;

import org.dataone.service.exceptions.InvalidSystemMetadata;

import org.dataone.service.exceptions.ServiceFailure;

import org.dataone.service.types.AuthToken;

import org.dataone.service.types.Checksum;

import org.dataone.service.types.ChecksumAlgorithm;

import org.dataone.service.types.NodeReference;

import org.dataone.service.types.ObjectFormat;

import org.dataone.service.types.Principal;

import org.dataone.service.types.SystemMetadata;

import org.dataone.service.types.Identifier;

import org.dataone.client.D1Client;

//import sun.tools.jstat.Identifier;

import com.gc.iotools.stream.is.InputStreamFromOutputStream;

/**

 * @author berkley

 * A class to populate a metacat instance based on documents returned from a query

 */

public class MetacatPopulator

{

    private String sourceUrl = null;

    private String destinationUrl = null;

    private String query = null;

    private String username = null;

    private String password = null;

    /**

     * create a new MetacatPopulator with given source and destination urls.  

     * These should be

     * of the form "http://<url>/<metacat_instance>"

     * If username and/or password is null, the query will be run as public

     * @param sourceUrl

     * @param destUrl

     * @param query

     * @param username

     * @param password

     */

    public MetacatPopulator(String sourceUrl, String destUrl, String query, String username, String password)

    {

        this.sourceUrl = sourceUrl;

        this.query = query;

        this.username = username;

        this.password = password;

        this.destinationUrl = destUrl;

    }

    /**

     * populate from the source

     */

    public void populate()

      throws Exception

    {

        String sourceSessionid = loginSource();

        String destSessionid = loginDest();

        //do a query

        String params = "returndoctype=eml://ecoinformatics.org/eml-2.0.1&" +

                        "returndoctype=eml://ecoinformatics.org/eml-2.0.0&" +

                        "returndoctype=BIN&" +

                        "returndoctype=http://dataone.org/service/types/SystemMetadata/0.1&";

        params += "action=query&";

        params += "qformat=xml&";

        params += "anyfield=" + query;

        System.out.println("searching '" + sourceUrl + "' for '" + query + "' with sessionid '" + sourceSessionid + "'");

        InputStream is = getResponse(sourceUrl, "/metacat",

                params, "POST");

        String response = streamToString(is);

        //System.out.println("response: " + response);

        Vector<Document> docs = parseResponse(response);

        for(int i=0; i<docs.size(); i++)

        {

            //for each document in the query

            Document doc = docs.get(i);

            String docid = doc.docid;

            //get the doc from source

            System.out.println("retrieving doc " + docid);

            params = "action=read&qformat=xml&docid=" + docid;

            is = getResponse(sourceUrl, "/metacat", params, "POST");

            String doctext = streamToString(is);

            //System.out.println("Done retrieving document: " + doctext);

            is = stringToStream(doctext);

            doc.doctext = doctext;

            //params += "&doctext=" + doc;

            //is = getResponse(sourceUrl, "/metacat",

            //        params, "POST");

            //System.out.println("done with upload: " + streamToString(is));

            D1Client d1 = new D1Client(destinationUrl + "/");

            SystemMetadata sysmeta = generateSystemMetadata(doc);

            AuthToken authtoken = d1.login(username, password);

            Identifier fakeId = new Identifier();

            fakeId.setValue("XXX");

            Identifier id = d1.create(authtoken, /*sysmeta.getIdentifier()*/fakeId, 

                  IOUtils.toInputStream(doc.doctext), sysmeta);

            System.out.println("Document created with id " + id.getValue());

            //insert it into destination

            /*System.out.println("Creating multipart message");

            final MimeMultipart mmp = createMimeMultipart(is);

            final InputStreamFromOutputStream<String> multipartStream = 

                new InputStreamFromOutputStream<String>() {

                @Override

                public String produce(final OutputStream dataSink) throws Exception {

                    mmp.writeTo(dataSink);

                    IOUtils.closeQuietly(dataSink);

                    return "Completed";

                }

            };

            System.out.println("uploading document to " + destinationUrl + 

                    "with docid " + docid + " with sessionid " + destSessionid);

            InputStream uploadResponse = sendRequest(destinationUrl, "/object", 

                    destSessionid, "PUT", "action=inser&docid=" + docid, 

                    "multipart/form-data", multipartStream);

            String res = streamToString(uploadResponse);*/

            /*CrudService cs = new CrudService();

            cs.setContextUrl(destinationUrl);

            Identifier guid = new Identifier();

            guid.setValue(docid);

            cs.create(new AuthToken(destSessionid), guid, is, null);*/

            //System.out.println("uploading complete: " + res);

            //System.out.println("Done with document " + docid);

        }

        logout();

    }

    /**

     * produce an md5 checksum for item

     */

    private String checksum(InputStream is)

      throws Exception

    {        

        byte[] buffer = new byte[1024];

        MessageDigest complete = MessageDigest.getInstance("MD5");

        int numRead;

        do 

        {

          numRead = is.read(buffer);

          if (numRead > 0) 

          {

            complete.update(buffer, 0, numRead);

          }

        } while (numRead != -1);

        return getHex(complete.digest());

    }

    /**

     * convert a byte array to a hex string

     */

    private static String getHex( byte [] raw ) 

    {

        final String HEXES = "0123456789ABCDEF";

        if ( raw == null ) {

          return null;

        }

        final StringBuilder hex = new StringBuilder( 2 * raw.length );

        for ( final byte b : raw ) {

          hex.append(HEXES.charAt((b & 0xF0) >> 4))

             .append(HEXES.charAt((b & 0x0F)));

        }

        return hex.toString();

    }

    /**

     * @param doc

     * @return

     */

    private SystemMetadata generateSystemMetadata(Document doc)

      throws Exception

    {

        SystemMetadata sm = new SystemMetadata();

        //set the id

        Identifier id = new Identifier();

        id.setValue(doc.docid);

        sm.setIdentifier(id);

        //set the object format

        ObjectFormat format = ObjectFormat.convert(doc.doctype);

        if(format == null)

        {

            if(doc.doctype.trim().equals("BIN"))

            {

                format = ObjectFormat.APPLICATIONOCTETSTREAM;

            }

            else

            {

                format = ObjectFormat.convert("text/plain");

            }

        }

        sm.setObjectFormat(format);

        //create the checksum

        ByteArrayInputStream bais = new ByteArrayInputStream(doc.doctext.getBytes());

        String checksumS = checksum(bais);

        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");

        Checksum checksum = new Checksum();

        checksum.setValue(checksumS);

        checksum.setAlgorithm(ca);

        sm.setChecksum(checksum);

        //set the size

        sm.setSize(doc.doctext.getBytes().length);

        //submitter

        Principal p = new Principal();

        p.setValue("");

        sm.setSubmitter(p);

        sm.setRightsHolder(p);

        try

        {

            Date dateCreated = parseMetacatDate(doc.createDate);

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = parseMetacatDate(doc.updateDate);

            sm.setDateSysMetadataModified(dateUpdated);

        }

        catch(Exception e)

        {

            System.out.println("couldn't parse a date: " + e.getMessage());

            Date dateCreated = new Date();

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = new Date();

            sm.setDateSysMetadataModified(dateUpdated);

        }

        NodeReference nr = new NodeReference();

        nr.setValue(sourceUrl);

        sm.setOriginMemberNode(nr);

        sm.setAuthoritativeMemberNode(nr);

        return sm;

    }

    /**

     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into

     * a proper date object

     * @param date

     * @return

     */

    private Date parseMetacatDate(String date)

    {

        String year = date.substring(0, 4);

        String month = date.substring(5, 7);

        String day = date.substring(8, 10);

        Calendar c = Calendar.getInstance();

        c.set(new Integer(year).intValue(), 

              new Integer(month).intValue(), 

              new Integer(day).intValue());

        return c.getTime();

    }

    /**

     * send a request to the resource

     */

    private InputStream sendRequest(String contextRootUrl, String resource, 

            String sessionid, String method, String urlParamaters, 

            String contentType, InputStream dataStream) 

        throws Exception 

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl + resource;

        if (urlParamaters != null) {

            if (restURL.indexOf("?") == -1)             

                restURL += "?";

            restURL += urlParamaters; 

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        if(sessionid != null)

        {

            if(restURL.indexOf("?") == -1)

            {

                restURL += "?sessionid=" + sessionid;

            }

            else

            {

                restURL += "&sessionid=" + sessionid;

            }

        }

        URL u = null;

        InputStream content = null;

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        if (contentType!=null) {

            connection.setRequestProperty("Content-Type",contentType);

        }

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        if (!method.equals("GET")) {

            if (dataStream != null) {

                OutputStream out = connection.getOutputStream();

                IOUtils.copy(dataStream, out);

            }

        }

        return connection.getInputStream();   

    }

    /**

     * create a mime multipart message from object and sysmeta

     */

    private MimeMultipart createMimeMultipart(InputStream object)

      throws Exception

    {

        final MimeMultipart mmp = new MimeMultipart();

        MimeBodyPart objectPart = new MimeBodyPart();

        objectPart.addHeaderLine("Content-Transfer-Encoding: base64");

        objectPart.setFileName("doctext");

        DataSource ds = new InputStreamDataSource("doctext", object);

        DataHandler dh = new DataHandler(ds);

        objectPart.setDataHandler(dh);

        mmp.addBodyPart(objectPart);

        return mmp;

    }

    /**

     * parse a metacat query response and return a vector of docids

     * @param response

     * @return

     */

    private Vector<Document> parseResponse(String response)

    {

        Vector<Document> v = new Vector<Document>();

        int dstart = response.indexOf("<document>");

        int dend = response.indexOf("</document>", dstart);

        while(dstart != -1)

        {

            String doc = response.substring(dstart + "<document>".length(), dend);

            //System.out.println("adding " + docid);

            Document d = new Document(getFieldFromDoc(doc, "docid"),

                    getFieldFromDoc(doc, "doctype"),

                    getFieldFromDoc(doc, "createdate"),

                    getFieldFromDoc(doc, "updatedate"));

            v.add(d);

            dstart = response.indexOf("<document>", dend);

            dend = response.indexOf("</document>", dstart);

        }

        return v;

    }

    private String getFieldFromDoc(String doc, String fieldname)

    {

        String field = "<" + fieldname + ">";

        String fieldend = "</" + fieldname + ">";

        int start = doc.indexOf(field);

        int end = doc.indexOf(fieldend);

        String s = doc.substring(start + field.length(), end);

        System.out.println("field: " + fieldname + " : " + s);

        return s;

    }

    /**

     * login the source

     * @return

     * @throws Exception

     */

    private String loginSource()

      throws Exception

    {

        return login(sourceUrl);

    }

    /**

     * login the destination

     * @return

     * @throws Exception

     */

    private String loginDest()

        throws Exception

    {

        return login(destinationUrl);

    }

    /**

     * returns a sessionid

     * @return

     */

    private String login(String sourceUrl)

      throws Exception

    {

        InputStream is = getResponse(sourceUrl, "/metacat", 

                "action=login&username=" + username + "&password=" + password + "&qformat=xml", 

        "POST");

        String response = streamToString(is);

        //System.out.println("response: " + response);

        String sessionid = response.substring(

                response.indexOf("<sessionId>") + "<sessionId>".length(), 

                response.indexOf("</sessionId>"));

        System.out.println("sessionid: " + sessionid);

        return sessionid;

    }

    /**

     * logout both the source and destination

     * @throws Exception

     */

    private void logout()

        throws Exception

    {

        getResponse(sourceUrl, "/metacat", "action=logout&username=" + username, "POST");

        getResponse(destinationUrl, "/metacat", "action=logout&username=" + username, "POST");

    }

    /**

     * get an http response

     * @param contextRootUrl

     * @param resource

     * @param urlParameters

     * @param method

     * @return

     * @throws Exception

     */

    private InputStream getResponse(String contextRootUrl, String resource, 

            String urlParameters, String method)

      throws Exception

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl+resource;

        if (urlParameters != null) {

            if (restURL.indexOf("?") == -1)             

                restURL += "?";

            restURL += urlParameters; 

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        URL u = null;

        InputStream content = null;            

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        content = connection.getInputStream();

        return content;

    }

    private String streamToString(InputStream is)

        throws Exception

    {

        byte b[] = new byte[1024];

        int numread = is.read(b, 0, 1024);

        String response = new String();

        while(numread != -1)

        {

            response += new String(b, 0, numread);

            numread = is.read(b, 0, 1024);

        }

        return response;

    }

    private InputStream stringToStream(String s)

      throws Exception

    {

        ByteArrayInputStream bais = new ByteArrayInputStream(s.getBytes());

        return bais;

    }

    private class Document

    {

        public String docid;

        public String doctype;

        public String createDate;

        public String updateDate;

        public String doctext;

        public Document(String docid, String doctype, String createDate, String updateDate)

        {

            this.docid = docid;

            this.doctype = doctype;

            this.createDate = createDate;

            this.updateDate = updateDate;

        }

    }

}

public class CrudService implements MemberNodeCrud

{

     * Set the context url that this service uses.  It is normally not necessary

     * to call this method unless you are trying to connect to a server other

     * than the one in which this service is installed.  Otherwise, this value is

     * taken from the metacat.properties file (server.name, server.port, application.context).

     */

    public void setContextUrl(String url)

    {

        metacatUrl = url;

    }

    /**

    public static SessionData getSessionData(AuthToken token) {