Metacat

/**

 *  '$RCSfile$'

 *    Purpose: A Class that implements administrative methods

 *  Copyright: 2010 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Michael Daigle

 *

 *   '$Author: berkley $'

 *     '$Date: 2010-06-08 12:34:30 -0700 (Tue, 08 Jun 2010) $'

 * '$Revision: 5374 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.util;

import java.io.InputStream;

import java.io.OutputStream;

import java.net.URL;

import java.util.Date;

import java.util.List;

import java.util.Map;

import javax.activation.DataHandler;

import javax.activation.DataSource;

import javax.mail.internet.MimeBodyPart;

import javax.mail.internet.MimeMultipart;

import org.apache.commons.io.IOUtils;

import org.dataone.client.MNode;

import org.dataone.service.types.v1.AccessRule;

import org.dataone.service.types.v1.Checksum;

import org.dataone.service.types.v1.Identifier;

import org.dataone.service.types.v1.NodeReference;

import org.dataone.service.types.v1.ObjectFormat;

import org.dataone.service.types.v1.Permission;

import org.dataone.service.types.v1.Session;

import org.dataone.service.types.v1.Subject;

import org.dataone.service.types.v1.SystemMetadata;

import org.dataone.service.util.Constants;

import org.ecoinformatics.datamanager.database.DatabaseConnectionPoolInterface;

import org.ecoinformatics.datamanager.parser.DataPackage;

import edu.ucsb.nceas.metacat.restservice.InputStreamDataSource;

/**

 * @author berkley

 * A class to populate a metacat instance based on documents returned from a query

 */

public class MetacatPopulator

{

    private String sourceUrl = null;

    private String destinationUrl = null;

    private String query = null;

    private String username = null;

    private String password = null;

    /**

     * create a new MetacatPopulator with given source and destination urls.

     * These should be

     * of the form "http://<url>/<metacat_instance>"

     * If username and/or password is null, the query will be run as public

     * @param sourceUrl

     * @param destUrl

     * @param query

     * @param username

     * @param password

     */

    public MetacatPopulator(String sourceUrl, String destUrl, String query, String username, String password)

    {

        this.sourceUrl = sourceUrl;

        this.query = query;

        this.username = username;

        this.password = password;

        this.destinationUrl = destUrl;

        this.session = null; //new Session();

    /**

     * populate from the source

     */

    public void populate()

      throws Exception

    {

                        "returndoctype=eml://ecoinformatics.org/eml-2.0.1&" +

                        "returndoctype=eml://ecoinformatics.org/eml-2.0.0&";

        params += "qformat=xml&";

        params += "anyfield=" + query;

        InputStream is = getResponse(sourceUrl, "/metacat", params, "POST");

        //System.out.println("response: " + response);

        Vector<Document> docs = parseResponse(response);

        	// for generating the ORE map

            Map<Identifier, List<Identifier>> idMap = new HashMap<Identifier, List<Identifier>>();

            List<Identifier> dataIds = new ArrayList<Identifier>();

            Document doc = docs.get(i);

            String docid = doc.docid;

            //get the doc from source

            is = getResponse(sourceUrl, "/metacat", params, "POST");

            String doctext = streamToString(is);

        	DataManager dataManager = DataManager.getInstance(connectionPool, connectionPool.getDBAdapterName());

        	DataPackage dataPackage = dataManager.parseMetadata(is);

            }

            is = stringToStream(doctext);

            printHeader("creating document on destination " + destinationUrl);

            // iterate through the data objects

		                dataDocMimeType =

	                String dataDocLocalId = "";

	                            "ecogrid://knb/".length(), dataDocUrl.length());

	                    //get the file

	                    params = "action=read&qformat=xml&docid=" + dataDocLocalId;

	                    InputStream dataDocIs = getResponse(sourceUrl, "/metacat", params, "POST");

	                    String dataDocText = streamToString(dataDocIs);

	                    //set the id

	                    Identifier did = new Identifier();

	                    did.setValue(dataDocLocalId);

	                    dataIds.add(did);

	                    //create sysmeta for the data doc

	                    SystemMetadata dataDocSysMeta = generateSystemMetadata(doc);

	                    //overwrite the bogus values from the last call

	                    dataDocSysMeta.setIdentifier(did);

	                    ObjectFormat format = null;

	                    try {

	                    	format = ObjectFormatCache.getInstance().getFormat(dataDocMimeType);

							System.out.println(e.getMessage());

						}

	                    dataDocIs = stringToStream(dataDocText);

	                    	Long.toString(dataDocText.getBytes(MetaCatServlet.DEFAULT_ENCODING).length);

	                    dataDocSysMeta.setSize(new BigInteger(sizeStr));

	                    //create the data doc on d1

	                    }

	                        System.out.println("ERROR: Could not create data document with id " +

	                                dataDocSysMeta.getIdentifier().getValue() + " : " + e.getMessage());

	                    }

	                        if (error) {

	                                    "FAILED.");

	                        }

	                                " which is described by " + sysmeta.getIdentifier().getValue());

	                        }

	                    }

	                }

	                            dataDocUrl + " for document " + doc.docid +

	                    ".  Only ecogrid://knb/ urls are currently supported.");

	                }

	            }

              Identifier id =

            	  mn.create(session, sysmeta.getIdentifier(), IOUtils.toInputStream(doc.doctext), sysmeta);

              if (!dataIds.isEmpty()) {

	              // generate the ORE map for this datapackage

	              Identifier resourceMapId = new Identifier();

	              resourceMapId.setValue("resourceMap_" + sysmeta.getIdentifier().getValue());

	              idMap.put(sysmeta.getIdentifier(), dataIds);

	              ResourceMap rm = ResourceMapFactory.getInstance().createResourceMap(resourceMapId, idMap);

	              String resourceMapXML = ResourceMapFactory.getInstance().serializeResourceMap(rm);

	              Document rmDoc = new Document(resourceMapId.getValue(), "http://www.openarchives.org/ore/terms", "", "");

	              rmDoc.doctext = resourceMapXML;

	              SystemMetadata resourceMapSysMeta = generateSystemMetadata(rmDoc);

	              mn.create(session, resourceMapId, IOUtils.toInputStream(resourceMapXML), resourceMapSysMeta);

	              // clean up the permissions (FORCE public read)

	              for (Identifier dataId: dataIds) {

	            	  mn.setAccessPolicy(session, dataId, sysmeta.getAccessPolicy());

	            	  System.out.println("Set public access policy for: " + dataId.getValue());

	              }

            }

            catch(Exception e) {

                        sysmeta.getIdentifier().getValue() + " : " + e.getMessage());

            }

                printHeader("Done processing document " + sysmeta.getIdentifier().getValue());

    /**

     * @param doc

     * @return

     */

    private SystemMetadata generateSystemMetadata(Document doc)

        Identifier id = new Identifier();

        //set the object format

            if (doc.doctype.trim().equals("BIN")) {

        }

        //create the checksum

        //set the size

        sm.setSize(new BigInteger(sizeStr));

        sm.setRightsHolder(p);

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = parseMetacatDate(doc.updateDate);

            sm.setDateSysMetadataModified(dateUpdated);

        }

            Date dateCreated = new Date();

            sm.setDateUploaded(dateCreated);

            Date dateUpdated = new Date();

            sm.setDateSysMetadataModified(dateUpdated);

        }

        NodeReference nr = new NodeReference();

        sm.setAuthoritativeMemberNode(nr);

        AccessPolicy accessPolicy = new AccessPolicy();

        AccessRule accessRule = new AccessRule();

		accessRule.addPermission(Permission.READ);

        Subject subject = new Subject();

        subject.setValue(Constants.SUBJECT_PUBLIC);

		accessRule.addSubject(subject);

		accessPolicy.addAllow(accessRule);

    }

    }

    /**

     * a proper date object

     * @param date

     * @return

     */

    private Date parseMetacatDate(String date)

    {

        String year = date.substring(0, 4);

        String month = date.substring(5, 7);

        String day = date.substring(8, 10);

        Calendar c = Calendar.getInstance();

        c.set(new Integer(year).intValue(),

              new Integer(month).intValue(),

              new Integer(day).intValue());

        return c.getTime();

    }

    /**

     * send a request to the resource

     */

    private InputStream sendRequest(String contextRootUrl, String resource,

            String sessionid, String method, String urlParamaters,

            String contentType, InputStream dataStream)

        throws Exception

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl + resource;

        if (urlParamaters != null) {

            if (restURL.indexOf("?") == -1)

                restURL += "?";

            restURL += urlParamaters;

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        if(sessionid != null)

        {

            if(restURL.indexOf("?") == -1)

            {

                restURL += "?sessionid=" + sessionid;

            }

            else

            {

                restURL += "&sessionid=" + sessionid;

            }

        }

        URL u = null;

        InputStream content = null;

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        if (contentType!=null) {

            connection.setRequestProperty("Content-Type",contentType);

        }

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        if (!method.equals("GET")) {

            if (dataStream != null) {

                OutputStream out = connection.getOutputStream();

                IOUtils.copy(dataStream, out);

            }

        }

        return connection.getInputStream();

    }

    /**

     * create a mime multipart message from object and sysmeta

     */

    private MimeMultipart createMimeMultipart(InputStream object)

      throws Exception

    {

        final MimeMultipart mmp = new MimeMultipart();

        MimeBodyPart objectPart = new MimeBodyPart();

        objectPart.addHeaderLine("Content-Transfer-Encoding: base64");

        objectPart.setFileName("doctext");

        DataSource ds = new InputStreamDataSource("doctext", object);

        DataHandler dh = new DataHandler(ds);

        objectPart.setDataHandler(dh);

        mmp.addBodyPart(objectPart);

        return mmp;

    }

    /**

     * parse a metacat query response and return a vector of docids

     * @param response

     * @return

     */

    private Vector<Document> parseResponse(String response)

    {

        Vector<Document> v = new Vector<Document>();

        int dstart = response.indexOf("<document>");

        int dend = response.indexOf("</document>", dstart);

        while(dstart != -1)

        {

            String doc = response.substring(dstart + "<document>".length(), dend);

            //System.out.println("adding " + docid);

            Document d = new Document(getFieldFromDoc(doc, "docid"),

                    getFieldFromDoc(doc, "doctype"),

                    getFieldFromDoc(doc, "createdate"),

                    getFieldFromDoc(doc, "updatedate"));

            v.add(d);

            dstart = response.indexOf("<document>", dend);

            dend = response.indexOf("</document>", dstart);

        }

        return v;

    }

    private String getFieldFromDoc(String doc, String fieldname)

    {

        String field = "<" + fieldname + ">";

        String fieldend = "</" + fieldname + ">";

        int start = doc.indexOf(field);

        int end = doc.indexOf(fieldend);

        String s = doc.substring(start + field.length(), end);

    }

    /**

     * login the source

     * @return

     * @throws Exception

     */

    private String loginSource()

      throws Exception

    {

        return login(sourceUrl);

    }

    /**

     * returns a sessionid

     * @return

     */

    private String login(String sourceUrl)

      throws Exception

    {

        InputStream is = getResponse(sourceUrl, "/metacat",

        //System.out.println("response: " + response);

        {

            throw new Exception("Error logging into " + sourceUrl);

        }

                response.indexOf("<sessionId>") + "<sessionId>".length(),

                response.indexOf("</sessionId>"));

        System.out.println("sessionid: " + sessionid);

        return sessionid;

    }

    /**

     * logout both the source and destination

     * @throws Exception

     */

    private void logout()

        throws Exception

    {

        getResponse(sourceUrl, "/metacat", "action=logout&username=" + username, "POST");

    }

    /**

     * get an http response

     * @param contextRootUrl

     * @param resource

     * @param urlParameters

     * @param method

     * @return

     * @throws Exception

     */

    private InputStream getResponse(String contextRootUrl, String resource,

            String urlParameters, String method)

      throws Exception

    {

        HttpURLConnection connection = null ;

        String restURL = contextRootUrl+resource;

        if (urlParameters != null) {

            if (restURL.indexOf("?") == -1)

                restURL += "?";

            restURL += urlParameters;

            if(restURL.indexOf(" ") != -1)

            {

                restURL = restURL.replaceAll("\\s", "%20");

            }

        }

        URL u = null;

        InputStream content = null;

        System.out.println("url: " + restURL);

        System.out.println("method: " + method);

        u = new URL(restURL);

        connection = (HttpURLConnection) u.openConnection();

        connection.setDoOutput(true);

        connection.setDoInput(true);

        connection.setRequestMethod(method);

        content = connection.getInputStream();

        return content;

    }

    private String streamToString(InputStream is)

        throws Exception

    {

        byte b[] = new byte[1024];

        int numread = is.read(b, 0, 1024);

        String response = new String();

        while(numread != -1)

        {

            response += new String(b, 0, numread);

            numread = is.read(b, 0, 1024);

        }

        return response;

    }

    private InputStream stringToStream(String s)

      throws Exception

    {

    }

    private class Document

    {

        public String docid;

        public String doctype;

        public String createDate;

        public String updateDate;

        public String doctext;

        public Document(String docid, String doctype, String createDate, String updateDate)

        {

            this.doctype = doctype.trim();

            this.createDate = createDate.trim();

            this.updateDate = updateDate.trim();

    }

}