Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2000 Regents of the University of California and the

 *              National Center for Ecological Analysis and Synthesis

 *

 *   '$Author: $'

 *     '$Date: 2009-06-13 15:28:13 +0300  $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.dataone;

import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.io.PrintWriter;

import java.io.StringBufferInputStream;

import java.security.MessageDigest;

import java.sql.SQLException;

import java.util.*;

import java.text.DateFormat;

import javax.servlet.ServletContext;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.apache.commons.io.IOUtils;

import org.apache.log4j.Logger;

import org.dataone.service.exceptions.IdentifierNotUnique;

import org.dataone.service.exceptions.InsufficientResources;

import org.dataone.service.exceptions.InvalidRequest;

import org.dataone.service.exceptions.InvalidSystemMetadata;

import org.dataone.service.exceptions.InvalidToken;

import org.dataone.service.exceptions.NotAuthorized;

import org.dataone.service.exceptions.NotFound;

import org.dataone.service.exceptions.NotImplemented;

import org.dataone.service.exceptions.ServiceFailure;

import org.dataone.service.exceptions.UnsupportedType;

import org.dataone.service.mn.MemberNodeCrud;

import org.dataone.service.types.*;

import org.jibx.runtime.BindingDirectory;

import org.jibx.runtime.IBindingFactory;

import org.jibx.runtime.IMarshallingContext;

import org.jibx.runtime.IUnmarshallingContext;

import org.jibx.runtime.JiBXException;

import com.gc.iotools.stream.is.InputStreamFromOutputStream;

import edu.ucsb.nceas.metacat.AccessionNumberException;

import edu.ucsb.nceas.metacat.MetacatResultSet;

import edu.ucsb.nceas.metacat.MetacatResultSet.Document;

import edu.ucsb.nceas.metacat.DocumentImpl;

import edu.ucsb.nceas.metacat.EventLog;

import edu.ucsb.nceas.metacat.IdentifierManager;

import edu.ucsb.nceas.metacat.McdbDocNotFoundException;

import edu.ucsb.nceas.metacat.McdbException;

import edu.ucsb.nceas.metacat.MetacatHandler;

import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;

import edu.ucsb.nceas.metacat.client.rest.MetacatRestClient;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;

import edu.ucsb.nceas.metacat.service.SessionService;

import edu.ucsb.nceas.metacat.util.DocumentUtil;

import edu.ucsb.nceas.metacat.util.SessionData;

import edu.ucsb.nceas.utilities.ParseLSIDException;

import edu.ucsb.nceas.utilities.PropertyNotFoundException;

/**

 * 

 * Implements DataONE MemberNode CRUD API for Metacat. 

 * 

 * @author Matthew Jones

 */

public class CrudService implements MemberNodeCrud

{

    private static CrudService crudService = null;

    private MetacatHandler handler;

    private Hashtable<String, String[]> params;

    private Logger logMetacat = null;

    private Logger logCrud = null;

    private String metacatUrl;

    /**

     * singleton accessor

     */

    public static CrudService getInstance()

    {

      if(crudService == null)

      {

        crudService = new CrudService();

      }

      return crudService;

    }

    /**

     * Initializes new instance by setting servlet context,request and response.

     */

    public CrudService() {

    //change crud service into a singleton.  dont pass servlet data structures here

        logMetacat = Logger.getLogger(CrudService.class);

        logCrud = Logger.getLogger("DataOneLogger");

        try

        {

            String server = PropertyService.getProperty("server.name");

            String port = PropertyService.getProperty("server.httpPort");

            String context = PropertyService.getProperty("application.context");

            metacatUrl = "http://" + server + ":" + port + "/" + context;

            logMetacat.debug("Initializing CrudService with url " + metacatUrl);

        }

        catch(Exception e)

        {

            logMetacat.error("Could not find servlet url in CrudService: " + e.getMessage());

            e.printStackTrace();

            throw new RuntimeException("Error getting servlet url in CrudService: " + e.getMessage());

        }

        /*this.servletContext = servletContext;

        this.request = request;

        this.response = response;*/

        params = new Hashtable<String, String[]>();

        handler = new MetacatHandler(new Timer());

    }

    /**

     * return the context url CrudService is using.

     */

    public String getContextUrl()

    {

        return metacatUrl;

    }

    /**

     * Set the context url that this service uses.  It is normally not necessary

     * to call this method unless you are trying to connect to a server other

     * than the one in which this service is installed.  Otherwise, this value is

     * taken from the metacat.properties file (server.name, server.port, application.context).

     */

    public void setContextUrl(String url)

    {

        metacatUrl = url;

    }

    /**

     * set the params for this service from an HttpServletRequest param list

     */

    public void setParamsFromRequest(HttpServletRequest request)

    {

        Enumeration paramlist = request.getParameterNames();

        while (paramlist.hasMoreElements()) {

            String name = (String) paramlist.nextElement();

            String[] value = (String[])request.getParameterValues(name);

            params.put(name, value);

        }

    }

    /**

     * Authenticate against metacat and get a token.

     * @param username

     * @param password

     * @return

     * @throws ServiceFailure

     */

    public AuthToken authenticate(String username, String password)

      throws ServiceFailure

    {

        try

        {

            MetacatRestClient restClient = new MetacatRestClient(getContextUrl());   

            String response = restClient.login(username, password);

            String sessionid = restClient.getSessionId();

            SessionService sessionService = SessionService.getInstance();

            sessionService.registerSession(new SessionData(sessionid, username, new String[0], password, "CrudServiceLogin"));

            AuthToken token = new AuthToken(sessionid);

            EventLog.getInstance().log(metacatUrl,

                    username, null, "authenticate");

            logCrud.info("authenticate");

            return token;

        }

        catch(Exception e)

        {

            throw new ServiceFailure("1000", "Error authenticating with metacat: " + e.getMessage());

        }

    }

    /**

     * set the parameter values needed for this request

     */

    public void setParameter(String name, String[] value)

    {

        params.put(name, value);

    }

    /**

     * Generate SystemMetadata for any object in the object store that does

     * not already have it.  SystemMetadata documents themselves, are, of course,

     * exempt.  This is a utility method for migration of existing object 

     * stores to DataONE where SystemMetadata is required for all objects.  See 

     * https://trac.dataone.org/ticket/591

     * 

     * @param token an authtoken with appropriate permissions to read all 

     * documents in the object store.  To work correctly, this should probably

     * be an adminstrative credential.

     */

    public void generateMissingSystemMetadata(AuthToken token)

    {

        IdentifierManager im = IdentifierManager.getInstance();

        //get the list of ids with no SM

        List<String> l = im.getLocalIdsWithNoSystemMetadata();

        for(int i=0; i<l.size(); i++)

        { //for each id, add a system metadata doc

            String localId = l.get(i);

            //System.out.println("Creating SystemMetadata for localId " + localId);

            //get the document

            try

            {

                //generate required system metadata fields from the document

                SystemMetadata sm = createSystemMetadata(localId, token);

                //insert the systemmetadata object

                SessionData sessionData = getSessionData(token);

                insertSystemMetadata(sm, sessionData);

                String username = sessionData.getUserName();

                EventLog.getInstance().log(metacatUrl,

                        username, localId, "generateMissingSystemMetadata");

            }

            catch(Exception e)

            {

                //e.printStackTrace();

                System.out.println("Exception generating missing system metadata: " + e.getMessage());

                logMetacat.error("Could not generate missing system metadata: " + e.getMessage());

            }

        }

        logCrud.info("generateMissingSystemMetadata");

    }

    /**

     * create an object via the crud interface

     */

    public Identifier create(AuthToken token, Identifier guid, 

            InputStream object, SystemMetadata sysmeta) throws InvalidToken, 

            ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, 

            InsufficientResources, InvalidSystemMetadata, NotImplemented {

        logMetacat.debug("Starting CrudService.create()...");

        // authenticate & get user info

        SessionData sessionData = getSessionData(token);

        String username = sessionData.getUserName();

        String[] groups = sessionData.getGroupNames();

        String localId = null;

        if (username == null || username.equals("public"))

        {

            throw new NotAuthorized("1000", "User " + username + " is not authorized to create content." +

                    "  If you are not logged in, please do so and retry the request.");

        }

        // verify that guid == SystemMetadata.getIdentifier()

        logMetacat.debug("Comparing guid|sysmeta_guid: " + guid.getValue() + "|" + sysmeta.getIdentifier().getValue());

        if (!guid.getValue().equals(sysmeta.getIdentifier().getValue())) {

            throw new InvalidSystemMetadata("1180", 

                "GUID in method call does not match GUID in system metadata.");

        }

        logMetacat.debug("Checking if identifier exists...");

        // Check that the identifier does not already exist

        IdentifierManager im = IdentifierManager.getInstance();

        if (im.identifierExists(guid.getValue())) {

            throw new IdentifierNotUnique("1120", 

                "GUID is already in use by an existing object.");

        }

        // Check if we are handling metadata or data

        boolean isScienceMetadata = isScienceMetadata(sysmeta);

        if (isScienceMetadata) {

            // CASE METADATA:

            try {

                this.insertDocument(object, guid, sessionData);

                localId = im.getLocalId(guid.getValue());

            } catch (IOException e) {

                String msg = "Could not create string from XML stream: " +

                    " " + e.getMessage();

                logMetacat.debug(msg);

                throw new ServiceFailure("1190", msg);

            } catch(Exception e) {

                String msg = "Unexpected error in CrudService.create: " + e.getMessage();

                logMetacat.debug(msg);

                throw new ServiceFailure("1190", msg);

            }

        } else {

            // DEFAULT CASE: DATA (needs to be checked and completed)

            insertDataObject(object, guid, sessionData);

        }

        // For Metadata and Data, insert the system metadata into the object store too

        insertSystemMetadata(sysmeta, sessionData);

        logMetacat.debug("Returning from CrudService.create()");

        EventLog.getInstance().log(metacatUrl,

                username, localId, "create");

        logCrud.info("create localId:" + localId + " guid:" + guid.getValue());

        return guid;

    }

    /**

     * update an existing object with a new object.  Change the system metadata

     * to reflect the changes and update it as well.

     */

    public Identifier update(AuthToken token, Identifier guid, 

            InputStream object, Identifier obsoletedGuid, SystemMetadata sysmeta) 

            throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, 

            UnsupportedType, InsufficientResources, NotFound, InvalidSystemMetadata, 

            NotImplemented {

        try

        {

            SessionData sessionData = getSessionData(token);

            //find the old systemmetadata (sm.old) document id (the one linked to obsoletedGuid)

            SystemMetadata sm = getSystemMetadata(token, obsoletedGuid);

            //change sm.old's obsoletedBy field 

            List l = sm.getObsoletedByList();

            l.add(guid);

            sm.setObsoletedByList(l);

            //update sm.old

            updateSystemMetadata(sm, sessionData);

            //change the obsoletes field of the new systemMetadata (sm.new) to point to the id of the old one

            sysmeta.addObsolete(obsoletedGuid);

            //insert sm.new

            insertSystemMetadata(sysmeta, sessionData);

            boolean isScienceMetadata = isScienceMetadata(sysmeta);

            if(isScienceMetadata)

            {

                //update the doc

                updateDocument(object, obsoletedGuid, guid, sessionData);

            }

            else

            {

                //update a data file, not xml

                insertDataObject(object, guid, sessionData);

            }

            IdentifierManager im = IdentifierManager.getInstance();

            String username = sessionData.getUserName();

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(guid.getValue()), "update");

            logCrud.info("update localId:" + im.getLocalId(guid.getValue()) + " guid:" + guid.getValue());

            return guid;

        }

        catch(Exception e)

        {

            throw new ServiceFailure("1030", "Error updating document in CrudService: " + e.getMessage());

        }

    }

    /**

     * set the permission on the document

     * @param token

     * @param principal

     * @param permission

     * @param permissionType

     * @param permissionOrder

     * @return

     */

    public void setAccess(AuthToken token, Identifier id, String principal, String permission,

            String permissionType, String permissionOrder)

      throws ServiceFailure

    {

        try

        {

            IdentifierManager im = IdentifierManager.getInstance();

            String docid = im.getLocalId(id.getValue());

            final SessionData sessionData = getSessionData(token);

            String permNum = "0";

            if(permission.equals("read"))

            {

                permNum = "4";

            }

            else if(permission.equals("write"))

            {

                permNum = "6";

            }

            handler.setAccess(metacatUrl, sessionData.getUserName(), docid, 

                    principal, permNum, permissionType, permissionOrder);

            String username = sessionData.getUserName();

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(id.getValue()), "setAccess");

            logCrud.info("setAccess");

        }

        catch(Exception e)

        {

            e.printStackTrace();

            throw new ServiceFailure("1000", "Could not set access on the document with id " + id.getValue());

        }

    }

    /**

     *  Retrieve the list of objects present on the MN that match the calling 

     *  parameters. This method is required to support the process of Member 

     *  Node synchronization. At a minimum, this method should be able to 

     *  return a list of objects that match:

     *  startTime <= SystemMetadata.dateSysMetadataModified

     *  but is expected to also support date range (by also specifying endTime), 

     *  and should also support slicing of the matching set of records by 

     *  indicating the starting index of the response (where 0 is the index 

     *  of the first item) and the count of elements to be returned.

     *  

     *  If startTime or endTime is null, the query is not restricted by that parameter.

     *  

     * @see http://mule1.dataone.org/ArchitectureDocs/mn_api_replication.html#MN_replication.listObjects

     * @param token

     * @param startTime

     * @param endTime

     * @param objectFormat

     * @param replicaStatus

     * @param start

     * @param count

     * @return ObjectList

     * @throws NotAuthorized

     * @throws InvalidRequest

     * @throws NotImplemented

     * @throws ServiceFailure

     * @throws InvalidToken

     */

    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime, 

        ObjectFormat objectFormat, boolean replicaStatus, int start, int count)

      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken

    {

      ObjectList ol = new ObjectList();

      final SessionData sessionData = getSessionData(token);

      try

      {

          params.clear();

          params.put("returndoctype", new String[] {"http://dataone.org/service/types/SystemMetadata/0.1"});

          params.put("qformat", new String[] {"xml"});

          params.put("returnfield", new String[] {"size", "originMemberNode", 

                  "identifier", "objectFormat", "dateSysMetadataModified", "checksum", "checksum/@algorithm"});

          params.put("anyfield", new String[] {"%"});

          //System.out.println("query is: metacatUrl: " + metacatUrl + " user: " + sessionData.getUserName() +

          //    " sessionid: " + sessionData.getId() + " params: " + params.toString());

          MetacatResultSet rs = handler.query(metacatUrl, params, sessionData.getUserName(), 

                  sessionData.getGroupNames(), sessionData.getId());

          List docs = rs.getDocuments();

          if(count == 1000)

          {

              count = docs.size();

          }

          //System.out.println("query returned " + count + " documents.");

          for(int i=start; i<count; i++)

          {

              //get the document from the result

              Document d = (Document)docs.get(i);

              //System.out.println("processing doc " + d.docid);

              ObjectFormat returnedObjectFormat = ObjectFormat.convert(d.getField("objectFormat"));

              if(returnedObjectFormat == null)

              {

                  continue;

              }

              if(objectFormat != null && !objectFormat.toString().trim().equals(returnedObjectFormat.toString().trim()))

              { //make sure the objectFormat is the one specified

                  continue;

              }

              String dateSMM = d.getField("dateSysMetadataModified");

              if(dateSMM == null)

              {

                  continue;

              }

              Date dateSysMetadataModified = parseDate(dateSMM);

              int startDateComparison = 0;

              int endDateComparison = 0;

              if(startTime != null)

              {

                  startDateComparison = dateSysMetadataModified.compareTo(startTime);

              }

              if(endTime != null)

              {

                  endDateComparison = dateSysMetadataModified.compareTo(endTime);

              }

              if(startDateComparison < 0 || endDateComparison > 0)

              { //this date falls outside of the startTime and endTime params, so

                //skip it

                  continue;                  

              }

              ObjectInfo info = new ObjectInfo();

              //add the fields to the info object

              Checksum cs = new Checksum();

              cs.setValue(d.getField("checksum"));

              String csalg = d.getField("algorithm");

              if(csalg == null)

              {

                  csalg = "MD5";

              }

              ChecksumAlgorithm ca = ChecksumAlgorithm.convert(csalg);

              cs.setAlgorithm(ca);

              info.setChecksum(cs);

              info.setDateSysMetadataModified(dateSysMetadataModified);

              Identifier id = new Identifier();

              id.setValue(d.getField("identifier"));

              info.setIdentifier(id);

              info.setObjectFormat(returnedObjectFormat);

              info.setSize(new Long(d.getField("size").trim()).longValue());

              //add the ObjectInfo to the ObjectList

              ol.addObjectInfo(info);

          }

      }

      catch(Exception e)

      {

          e.printStackTrace();

          throw new ServiceFailure("1580", "Error retrieving ObjectList: " + e.getMessage());

      }

      String username = sessionData.getUserName();

      EventLog.getInstance().log(metacatUrl,

              username, null, "read");

      logCrud.info("listObjects");

      //System.out.println("ol.size: " + ol.sizeObjectInfoList());

      ol.setCount(ol.sizeObjectInfoList());

      ol.setStart(0);

      ol.setTotal(ol.sizeObjectInfoList());

      return ol;

    }

    /**

     * Call listObjects with the default values for replicaStatus (true), start (0),

     * and count (1000).

     * @param token

     * @param startTime

     * @param endTime

     * @param objectFormat

     * @return

     * @throws NotAuthorized

     * @throws InvalidRequest

     * @throws NotImplemented

     * @throws ServiceFailure

     * @throws InvalidToken

     */

    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime, 

        ObjectFormat objectFormat)

      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken

    {

       return listObjects(token, startTime, endTime, objectFormat, true, 0, 1000);

    }

    /**

     * Delete a document.  NOT IMPLEMENTED

     */

    public Identifier delete(AuthToken token, Identifier guid)

            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

            NotImplemented {

        logCrud.info("delete");

        throw new NotImplemented("1000", "This method not yet implemented.");

    }

    /**

     * describe a document.  NOT IMPLEMENTED

     */

    public DescribeResponse describe(AuthToken token, Identifier guid)

            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

            NotImplemented {

        logCrud.info("describe");

        throw new NotImplemented("1000", "This method not yet implemented.");

    }

    /**

     * get a document with a specified guid.

     */

    public InputStream get(AuthToken token, Identifier guid)

            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

            NotImplemented {

        // Retrieve the session information from the AuthToken

        // If the session is expired, then the user is 'public'

        final SessionData sessionData = getSessionData(token);

        // Look up the localId for this global identifier

        IdentifierManager im = IdentifierManager.getInstance();

        try {

            final String localId = im.getLocalId(guid.getValue());

            final InputStreamFromOutputStream<String> objectStream = 

                new InputStreamFromOutputStream<String>() {

                @Override

                public String produce(final OutputStream dataSink) throws Exception {

                    try {

                        handler.readFromMetacat(metacatUrl, null, 

                                dataSink, localId, "xml",

                                sessionData.getUserName(), 

                                sessionData.getGroupNames(), true, params);

                    } catch (PropertyNotFoundException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "Error getting property from metacat: " + e.getMessage());

                    } catch (ClassNotFoundException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "Class not found error when reading from metacat: " + e.getMessage());

                    } catch (IOException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "IOException while reading from metacat: " + e.getMessage());

                    } catch (SQLException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "SQLException while reading from metacat: " + e.getMessage());

                    } catch (McdbException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "Metacat DB exception while reading from metacat: " + e.getMessage());

                    } catch (ParseLSIDException e) {

                        e.printStackTrace();

                        throw new NotFound("1020", "LSID parsing exception while reading from metacat: " + e.getMessage());

                    } catch (InsufficientKarmaException e) {

                        e.printStackTrace();

                        throw new NotAuthorized("1000", "User not authorized for get(): " + e.getMessage());

                    }

                    return "Completed";

                }

            };

            String username = sessionData.getUserName();

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(guid.getValue()), "read");

            logCrud.info("get localId:" + localId + " guid:" + guid.getValue());

            return objectStream;

        } catch (McdbDocNotFoundException e) {

            throw new NotFound("1020", e.getMessage());

        }

    }

    /**

     * get the checksum for a document.  NOT IMPLEMENTED

     */

    public Checksum getChecksum(AuthToken token, Identifier guid)

            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

            InvalidRequest, NotImplemented {

        logCrud.info("getChecksum");

        throw new NotImplemented("1000", "This method not yet implemented.");

    }

    /**

     * get the checksum for a document.  NOT IMPLEMENTED

     */

    public Checksum getChecksum(AuthToken token, Identifier guid, 

            String checksumAlgorithm) throws InvalidToken, ServiceFailure, 

            NotAuthorized, NotFound, InvalidRequest, NotImplemented {

        logCrud.info("getChecksum");

        throw new NotImplemented("1000", "This method not yet implemented.");

    }

    /**

     * get log records.  

     */

    public Log getLogRecords(AuthToken token, Date fromDate, Date toDate, Event event)

            throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, 

            NotImplemented 

    {

        Log log = new Log();

        Vector<LogEntry> logs = new Vector<LogEntry>();

        IdentifierManager im = IdentifierManager.getInstance();

        EventLog el = EventLog.getInstance();

        if(fromDate == null)

        {

            fromDate = new Date(1);

        }

        if(toDate == null)

        {

            toDate = new Date();

        }

        String report = el.getReport(null, null, null, null, 

                new java.sql.Timestamp(fromDate.getTime()), 

                new java.sql.Timestamp(toDate.getTime()));

        String logEntry = "<logEntry>";

        String endLogEntry = "</logEntry>";

        int startIndex = 0;

        int foundIndex = report.indexOf(logEntry, startIndex);

        while(foundIndex != -1)

        {

            //parse out each entry

            int endEntryIndex = report.indexOf(endLogEntry, foundIndex);

            String entry = report.substring(foundIndex, endEntryIndex);

            //System.out.println("entry: " + entry);

            startIndex = endEntryIndex + endLogEntry.length();

            foundIndex = report.indexOf(logEntry, startIndex);

            String entryId = getLogEntryField("entryid", entry);

            String ipAddress = getLogEntryField("ipAddress", entry);

            String principal = getLogEntryField("principal", entry);

            String docid = getLogEntryField("docid", entry);

            String eventS = getLogEntryField("event", entry);

            String dateLogged = getLogEntryField("dateLogged", entry);

            LogEntry le = new LogEntry();

            Event e = Event.convert(eventS);

            if(e == null)

            { //skip any events that are not Dataone Crud events

                continue;

            }

            le.setEvent(e);

            Identifier entryid = new Identifier();

            entryid.setValue(entryId);

            le.setEntryId(entryid);

            Identifier identifier = new Identifier();

            try

            {

                identifier.setValue(im.getGUID(docid, im.getLatestRevForLocalId(docid)));

            }

            catch(Exception ex)

            { //try to get the guid, if that doesn't work, just use the local id

                identifier.setValue(docid);

            }

            le.setIdentifier(identifier);

            le.setIpAddress(ipAddress);

            Calendar c = Calendar.getInstance();

            String year = dateLogged.substring(0, 4);

            String month = dateLogged.substring(5, 7);

            String date = dateLogged.substring(8, 10);

            //System.out.println("year: " + year + " month: " + month + " day: " + date);

            c.set(new Integer(year).intValue(), new Integer(month).intValue(), new Integer(date).intValue());

            Date logDate = c.getTime();

            le.setDateLogged(logDate);

            NodeReference memberNode = new NodeReference();

            memberNode.setValue(ipAddress);

            le.setMemberNode(memberNode);

            Principal princ = new Principal();

            princ.setValue(principal);

            le.setPrincipal(princ);

            le.setUserAgent("metacat/RESTService");

            if(event == null)

            {

                logs.add(le);

            }

            if(event != null &&

               e.toString().toLowerCase().trim().equals(event.toString().toLowerCase().trim()))

            {

              logs.add(le);

            }

        }

        log.setLogEntryList(logs);

        logCrud.info("getLogRecords");

        return log;

    }

    /**

     * parse a logEntry and get the relavent field from it

     * @param fieldname

     * @param entry

     * @return

     */

    private String getLogEntryField(String fieldname, String entry)

    {

        String begin = "<" + fieldname + ">";

        String end = "</" + fieldname + ">";

        //System.out.println("looking for " + begin + " and " + end + " in entry " + entry);

        String s = entry.substring(entry.indexOf(begin) + begin.length(), entry.indexOf(end));

        //System.out.println("entry " + fieldname + " : " + s);

        return s;

    }

    /**

     * get the system metadata for a document with a specified guid.

     */

    public SystemMetadata getSystemMetadata(AuthToken token, Identifier guid)

            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

            InvalidRequest, NotImplemented {

        logMetacat.debug("CrudService.getSystemMetadata - for guid: " + guid.getValue());

        // Retrieve the session information from the AuthToken

        // If the session is expired, then the user is 'public'

        final SessionData sessionData = getSessionData(token);

        try {

            IdentifierManager im = IdentifierManager.getInstance();

            final String localId = im.getSystemMetadataId(guid.getValue());

            // Read system metadata from metacat's db

            final InputStreamFromOutputStream<String> objectStream = 

                new InputStreamFromOutputStream<String>() {

                @Override

                public String produce(final OutputStream dataSink) throws Exception {

                    try {

                        handler.readFromMetacat(metacatUrl, null, 

                                dataSink, localId, "xml",

                                sessionData.getUserName(), 

                                sessionData.getGroupNames(), true, params);

                    } catch (PropertyNotFoundException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "Property not found while reading system metadata from metacat: " + e.getMessage());

                    } catch (ClassNotFoundException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "Class not found while reading system metadata from metacat: " + e.getMessage());

                    } catch (IOException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "IOException while reading system metadata from metacat: " + e.getMessage());

                    } catch (SQLException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "SQLException while reading system metadata from metacat: " + e.getMessage());

                    } catch (McdbException e) {

                        e.printStackTrace();

                        throw new ServiceFailure("1030", "Metacat DB Exception while reading system metadata from metacat: " + e.getMessage());

                    } catch (ParseLSIDException e) {

                        e.printStackTrace();

                        throw new NotFound("1020", "Error parsing LSID while reading system metadata from metacat: " + e.getMessage());

                    } catch (InsufficientKarmaException e) {

                        e.printStackTrace();

                        throw new NotAuthorized("1000", "User not authorized for get() on system metadata: " + e.getMessage());

                    }

                    return "Completed";

                }

            };

            // Deserialize the xml to create a SystemMetadata object

            SystemMetadata sysmeta = deserializeSystemMetadata(objectStream);

            String username = sessionData.getUserName();

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(guid.getValue()), "read");

            logCrud.info("getSystemMetadata localId: " + localId + " guid:" + guid.getValue());

            return sysmeta;

        } catch (McdbDocNotFoundException e) {

            //e.printStackTrace();