Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2000 Regents of the University of California and the

 *              National Center for Ecological Analysis and Synthesis

 *

 *   '$Author: $'

 *     '$Date: 2009-06-13 15:28:13 +0300  $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.dataone;

import java.io.ByteArrayInputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.security.NoSuchAlgorithmException;

import java.sql.SQLException;

import java.text.DateFormat;

import java.util.Calendar;

import java.util.Date;

import java.util.Enumeration;

import java.util.Hashtable;

import java.util.List;

import java.util.TimeZone;

import java.util.Timer;

import java.util.TimerTask;

import java.util.Vector;

import javax.servlet.http.HttpServletRequest;

import org.apache.commons.io.IOUtils;

import org.apache.log4j.Logger;

import org.dataone.service.exceptions.IdentifierNotUnique;

import org.dataone.service.exceptions.InsufficientResources;

import org.dataone.service.exceptions.InvalidRequest;

import org.dataone.service.exceptions.InvalidSystemMetadata;

import org.dataone.service.exceptions.InvalidToken;

import org.dataone.service.exceptions.NotAuthorized;

import org.dataone.service.exceptions.NotFound;

import org.dataone.service.exceptions.NotImplemented;

import org.dataone.service.exceptions.ServiceFailure;

import org.dataone.service.exceptions.UnsupportedType;

import org.dataone.service.mn.MemberNodeCrud;

import org.dataone.service.types.AuthToken;

import org.dataone.service.types.Checksum;

import org.dataone.service.types.ChecksumAlgorithm;

import org.dataone.service.types.DescribeResponse;

import org.dataone.service.types.Event;

import org.dataone.service.types.Identifier;

import org.dataone.service.types.Log;

import org.dataone.service.types.LogEntry;

import org.dataone.service.types.NodeReference;

import org.dataone.service.types.ObjectFormat;

import org.dataone.service.types.ObjectList;

import org.dataone.service.types.Subject;

import org.dataone.service.types.SystemMetadata;

import org.dataone.service.types.util.ServiceTypeUtil;

import edu.ucsb.nceas.metacat.AccessionNumberException;

import edu.ucsb.nceas.metacat.DocumentImpl;

import edu.ucsb.nceas.metacat.EventLog;

import edu.ucsb.nceas.metacat.IdentifierManager;

import edu.ucsb.nceas.metacat.McdbDocNotFoundException;

import edu.ucsb.nceas.metacat.McdbException;

import edu.ucsb.nceas.metacat.MetacatHandler;

import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;

import edu.ucsb.nceas.metacat.client.rest.MetacatRestClient;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;

import edu.ucsb.nceas.metacat.service.SessionService;

import edu.ucsb.nceas.metacat.util.DocumentUtil;

import edu.ucsb.nceas.metacat.util.SessionData;

import edu.ucsb.nceas.utilities.ParseLSIDException;

import edu.ucsb.nceas.utilities.PropertyNotFoundException;

/**

 * 

 * Implements DataONE MemberNode CRUD API for Metacat. 

 * 

 * @author Matthew Jones

 */

public class CrudService implements MemberNodeCrud

{

    private static CrudService crudService = null;

    private MetacatHandler handler;

    private Hashtable<String, String[]> params;

    private Logger logMetacat = null;

    private Logger logCrud = null;

    private String metacatUrl;

    /**

     * singleton accessor

     */

    public static CrudService getInstance() 

    {

      if(crudService == null)

      {

        crudService = new CrudService();

      }

      return crudService;

    }

    /**

     * Constructor, private for singleton access

     */

    private CrudService() {

        logMetacat = Logger.getLogger(CrudService.class);

        logCrud = Logger.getLogger("DataOneLogger");

        try

        {

            String server = PropertyService.getProperty("server.name");

            String port = PropertyService.getProperty("server.httpPort");

            String context = PropertyService.getProperty("application.context");

            metacatUrl = "http://" + server + ":" + port + "/" + context + "/d1";

            logMetacat.debug("Initializing CrudService with url " + metacatUrl);

        }

        catch(Exception e)

        {

            logMetacat.error("Could not find servlet url in CrudService: " + e.getMessage());

            e.printStackTrace();

            throw new RuntimeException("Error getting servlet url in CrudService: " + e.getMessage());

        }

        params = new Hashtable<String, String[]>();

        handler = new MetacatHandler(new Timer());

    }

    /**

     * return the context url CrudService is using.

     */

    public String getContextUrl()

    {

        return metacatUrl;

    }

    /**

     * Set the context url that this service uses.  It is normally not necessary

     * to call this method unless you are trying to connect to a server other

     * than the one in which this service is installed.  Otherwise, this value is

     * taken from the metacat.properties file (server.name, server.port, application.context).

     */

    public void setContextUrl(String url)

    {

        metacatUrl = url;

    }

    /**

     * set the params for this service from an HttpServletRequest param list

     */

    public void setParamsFromRequest(HttpServletRequest request)

    {

        @SuppressWarnings("unchecked")

        Enumeration<String> paramlist = request.getParameterNames();

        while (paramlist.hasMoreElements()) {

            String name = (String) paramlist.nextElement();

            String[] value = (String[])request.getParameterValues(name);

            params.put(name, value);

        }

    }

    /**

     * Authenticate against metacat and get a token.

     * @param username

     * @param password

     * @return

     * @throws ServiceFailure

     */

    public AuthToken authenticate(String username, String password)

      throws ServiceFailure

    {

        /* TODO:

         * This method is not in the original D1 crud spec.  It is highly

         * metacat centric.  Higher level decisions need to be made on authentication

         * interfaces for D1 nodes.

         */

        try

        {

            MetacatRestClient restClient = new MetacatRestClient(getContextUrl());   

            String response = restClient.login(username, password);

            String sessionid = restClient.getSessionId();

            SessionService sessionService = SessionService.getInstance();

            sessionService.registerSession(new SessionData(sessionid, username, new String[0], password, "CrudServiceLogin"));

            AuthToken token = new AuthToken(sessionid);

            EventLog.getInstance().log(metacatUrl,

                    username, null, "authenticate");

            logCrud.info("authenticate");

            return token;

        }

        catch(Exception e)

        {

            throw new ServiceFailure("1620", "Error authenticating with metacat: " + e.getMessage());

        }

    }

    /**

     * set the parameter values needed for this request

     */

    public void setParameter(String name, String[] value)

    {

        params.put(name, value);

    }

    /**

     * create an object via the crud interface

     */

    public Identifier create(AuthToken token, Identifier guid, 

            InputStream object, SystemMetadata sysmeta) throws InvalidToken, 

            ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, 

            InsufficientResources, InvalidSystemMetadata, NotImplemented {

        logMetacat.debug("Starting CrudService.create()...");

        // authenticate & get user info

        SessionData sessionData = getSessionData(token);

        String username = "public";

        String[] groups = null;

        if(sessionData != null)

        {

            username = sessionData.getUserName();

            groups = sessionData.getGroupNames();

        }

        String localId = null;

        if (username == null || username.equals("public"))

        {

            //TODO: many of the thrown exceptions do not use the correct error codes

            //check these against the docs and correct them

            throw new NotAuthorized("1100", "User " + username + " is not authorized to create content." +

                    "  If you are not logged in, please do so and retry the request.");

        }

        // verify that guid == SystemMetadata.getIdentifier()

        logMetacat.debug("Comparing guid|sysmeta_guid: " + guid.getValue() + "|" + sysmeta.getIdentifier().getValue());

        if (!guid.getValue().equals(sysmeta.getIdentifier().getValue())) {

            throw new InvalidSystemMetadata("1180", 

                "GUID in method call (" + guid.getValue() + ") does not match GUID in system metadata (" +

                sysmeta.getIdentifier().getValue() + ").");

        }

        logMetacat.debug("Checking if identifier exists...");

        // Check that the identifier does not already exist

        IdentifierManager im = IdentifierManager.getInstance();

        if (im.identifierExists(guid.getValue())) {

            throw new IdentifierNotUnique("1120", 

                "GUID is already in use by an existing object.");

        }

        // Check if we are handling metadata or data

        boolean isScienceMetadata = isScienceMetadata(sysmeta);

        if (isScienceMetadata) {

            // CASE METADATA:

            try {

                //logCrud.debug("CrudService: inserting document with guid " + guid.getValue());

                this.insertDocument(object, guid, sessionData);

                localId = im.getLocalId(guid.getValue());

            } catch (IOException e) {

                String msg = "Could not create string from XML stream: " +

                    " " + e.getMessage();

                logMetacat.debug(msg);

                throw new ServiceFailure("1190", msg);

            } catch(Exception e) {

                String msg = "Unexpected error in CrudService.create: " + e.getMessage();

                logMetacat.debug(msg);

                throw new ServiceFailure("1190", msg);

            }

        } else {

            // DEFAULT CASE: DATA (needs to be checked and completed)

            localId = insertDataObject(object, guid, sessionData);

        }

        // For Metadata and Data, insert the system metadata into the object store too

        insertSystemMetadata(sysmeta, sessionData);

        //get the document info.  add any access params for the sysmeta too

        //logCrud.debug("looking for access records to add for system " +

        //    "metadata who's parent doc's  local id is " + localId);

        try

        {

            Hashtable<String, Object> h = im.getDocumentInfo(localId.substring(0, localId.lastIndexOf(".")));

            Vector v = (Vector)h.get("access");

            for(int i=0; i<v.size(); i++)

            {

                @SuppressWarnings("unchecked")

                Hashtable<String, String> ah = (Hashtable<String, String>)v.elementAt(i);

                String principal = (String)ah.get("principal_name");

                String permission = (String)ah.get("permission");

                String permissionType = (String)ah.get("permission_type");

                String permissionOrder = (String)ah.get("permission_order");

                int perm = new Integer(permission).intValue();

                //logCrud.debug("found access record for principal " + principal);

                //logCrud.debug("permission: " + perm + " perm_type: " + permissionType + 

                //    " perm_order: " + permissionOrder);

                this.setAccess(token, guid, principal, perm, permissionType, permissionOrder, true);

            }

        }

        catch(Exception e)

        {

            logMetacat.error("Error setting permissions on System Metadata object: " +

            		e.getMessage());

            //TODO: decide if this error should cancel the entire create or

            //if it should continue with just a logged error.

        }

        logMetacat.debug("Returning from CrudService.create()");

        EventLog.getInstance().log(metacatUrl,

                username, localId, "create");

        logCrud.info("create D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId);

        return guid;

    }

    /**

     * update an existing object with a new object.  Change the system metadata

     * to reflect the changes and update it as well.

     */

    public Identifier update(AuthToken token, Identifier guid, 

            InputStream object, Identifier obsoletedGuid, SystemMetadata sysmeta) 

            throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, 

            UnsupportedType, InsufficientResources, NotFound, InvalidSystemMetadata, 

            NotImplemented {

    	try

    	{

    		SessionData sessionData = getSessionData(token);

            //find the old systemmetadata (sm.old) document id (the one linked to obsoletedGuid)

            SystemMetadata sm = getSystemMetadata(token, obsoletedGuid);

            //change sm.old's obsoletedBy field 

            List<Identifier> l = sm.getObsoletedByList();

            l.add(guid);

            sm.setObsoletedByList(l);

            //update sm.old

            updateSystemMetadata(sm, sessionData);

            //change the obsoletes field of the new systemMetadata (sm.new) to point to the id of the old one

            sysmeta.addObsolete(obsoletedGuid);

            //insert sm.new

            insertSystemMetadata(sysmeta, sessionData);

            String localId;

            boolean isScienceMetadata = isScienceMetadata(sysmeta);

            if(isScienceMetadata)

            {

                //update the doc

                localId = updateDocument(object, obsoletedGuid, guid, sessionData, false);

            }

            else

            {

                //update a data file, not xml

                localId = insertDataObject(object, guid, sessionData);

            }

            IdentifierManager im = IdentifierManager.getInstance();

            String username = "public";

            if(sessionData != null)

            {

                username = sessionData.getUserName();

            }

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(guid.getValue()), "update");

            logCrud.info("update D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId);

            return guid;

        }

    	catch(IOException e) {

            throw new ServiceFailure("1310", "Error updating document in CrudService: " + e.getMessage());

        }

    	catch( McdbDocNotFoundException e) {

    		throw new ServiceFailure("1310", "Error updating document in CrudService: " + e.getMessage());

    	}

    	catch( InvalidRequest e) {

    		throw new ServiceFailure("1310", "Error updating document in CrudService: " + e.getMessage());

    	}

    }

    /**

     * set access permissions on both the science metadata and system metadata

     */

    public void setAccess(AuthToken token, Identifier id, String principal, String permission,

            String permissionType, String permissionOrder)

      throws ServiceFailure

    {

        setAccess(token, id, principal, permission, permissionType, permissionOrder, true);

    }

    /**

     * set access control on the doc

     * @param token

     * @param id

     * @param principal

     * @param permission

     */

    public void setAccess(AuthToken token, Identifier id, String principal, int permission,

      String permissionType, String permissionOrder, boolean setSystemMetadata)

      throws ServiceFailure

    {

        String perm = "";

        if(permission >= 4)

        {

            perm = "read";

        }

        if(permission >= 6)

        {

            perm = "write";

        }

        //logCrud.debug("perm in setAccess: " + perm);

        //logCrud.debug("permission in setAccess: " + permission);

        setAccess(token, id, principal, perm, permissionType, permissionOrder,

                setSystemMetadata);

    }

    /**

     * set the permission on the document

     * @param token

     * @param principal

     * @param permission

     * @param permissionType

     * @param permissionOrder

     * @return

     */

    public void setAccess(AuthToken token, Identifier id, String principal, String permission,

            String permissionType, String permissionOrder, boolean setSystemMetadata)

      throws ServiceFailure

    {

        /* TODO:

         * This is also not part of the D1 Crud spec.  This method is needed for

         * systems such as metacat where access to objects is controlled by

         * and ACL.  Higher level decisions need to be made about how this

         * should work within D1.

         */

        try

        {

            final SessionData sessionData = getSessionData(token);

            if(sessionData == null)

            {

                throw new ServiceFailure("1000", "User must be logged in to set access.");

            }

            IdentifierManager im = IdentifierManager.getInstance();

            String docid = im.getLocalId(id.getValue());

            String permNum = "0";

            if(permission.equals("read"))

            {

                permNum = "4";

            }

            else if(permission.equals("write"))

            {

                permNum = "6";

            }

            logCrud.debug("user " + sessionData.getUserName() + 

                    " is setting access level " + permNum + " for permission " + 

                    permissionType + " on doc with localid " + docid);

            handler.setAccess(metacatUrl, sessionData.getUserName(), docid, 

                    principal, permNum, permissionType, permissionOrder);

            String username = "public";

            if(sessionData != null)

            {

                username = sessionData.getUserName();

            }

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(id.getValue()), "setAccess");

            logCrud.info("setAccess");

        }

        catch(Exception e)

        {

            e.printStackTrace();

            throw new ServiceFailure("1000", "Could not set access on the document with id " + id.getValue());

        }

    }

    /**

     *  Retrieve the list of objects present on the MN that match the calling 

     *  parameters. This method is required to support the process of Member 

     *  Node synchronization. At a minimum, this method should be able to 

     *  return a list of objects that match:

     *  startTime <= SystemMetadata.dateSysMetadataModified

     *  but is expected to also support date range (by also specifying endTime), 

     *  and should also support slicing of the matching set of records by 

     *  indicating the starting index of the response (where 0 is the index 

     *  of the first item) and the count of elements to be returned.

     *  

     *  If startTime or endTime is null, the query is not restricted by that parameter.

     *  

     * @see http://mule1.dataone.org/ArchitectureDocs/mn_api_replication.html#MN_replication.listObjects

     * @param token

     * @param startTime

     * @param endTime

     * @param objectFormat

     * @param replicaStatus

     * @param start

     * @param count

     * @return ObjectList

     * @throws NotAuthorized

     * @throws InvalidRequest

     * @throws NotImplemented

     * @throws ServiceFailure

     * @throws InvalidToken

     */

    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime, 

        ObjectFormat objectFormat, boolean replicaStatus, int start, int count)

      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken

    {

      //ObjectList ol = new ObjectList();

      //final SessionData sessionData = getSessionData(token);

      //int totalAfterQuery = 0;

      return IdentifierManager.getInstance().querySystemMetadata(startTime, endTime,

              objectFormat, replicaStatus, start, count);

      /////////////////////////////////////////////////////////////////////////

      ///////////////// OLD CODE //////////////////////////////////////////////

      /////////////////////////////////////////////////////////////////////////

//      try

//      {

//          if (PropertyService.getProperty("database.queryCacheOn").equals("true"))

//          {

//              //System.out.println("the string stored into cache is "+ resultsetBuffer.toString());

//              DBQuery.clearQueryResultCache();

//          }

//      }

//      catch (PropertyNotFoundException e1)

//      {

//          //just don't do anything

//      }

//      

//      try

//      {

//          //TODO: Milliseconds need to be added to the dateFormat

//          System.out.println("=========== Listing Objects =============");

//          System.out.println("Current server time is: " + new Date());

//          if(startTime != null)

//          {

//              System.out.println("query start time is " + startTime);

//          }

//          if(endTime != null)

//          {

//              System.out.println("query end time is " + endTime);

//          }

//          params.clear();

//          params.put("qformat", new String[] {PropertyService.getProperty("crudService.listObjects.QFormat")});

//          params.put("action", new String[] {"squery"});

//          params.put("query", new String[] {createListObjectsPathQueryDocument()});

//          

//          /*System.out.println("query is: metacatUrl: " + metacatUrl + " user: " + sessionData.getUserName() +

//                  " sessionid: " + sessionData.getId() + " params: ");

//          String url = metacatUrl + "/metacat?action=query&sessionid=" + sessionData.getId();

//          Enumeration keys = params.keys();

//          while(keys.hasMoreElements())

//          {

//              String key = (String)keys.nextElement();

//              String[] parr = params.get(key);

//              for(int i=0; i<parr.length; i++)

//              {

//                  System.out.println("param " + key + ": " + parr[i]);

//                  url += "&" + key + "=" + parr[i] ;

//              }

//          }

//          System.out.println("query url: " + url);

//          */

//          String username = "public";

//          String[] groups = null;

//          String sessionid = "";

//          if(sessionData != null)

//          {

//              username = sessionData.getUserName();

//              groups = sessionData.getGroupNames();

//              sessionid = sessionData.getId();

//          }

//          

//          MetacatResultSet rs = handler.query(metacatUrl, params, username, 

//                  groups, sessionid);

//          List docs = rs.getDocuments();

//          

//          System.out.println("query returned " + docs.size() + " documents.");

//          Vector<Document> docCopy = new Vector<Document>();

//          

//          //preparse the list to remove any that don't match the query params

//          /* TODO: this type of query/subquery processing is probably not scalable

//           * to larger object stores.  This code should be revisited.  The metacat

//           * query handler should probably be altered to handle the type of query 

//           * done here.

//           */ 

//          for(int i=0; i<docs.size(); i++)

//          {

//              Document d = (Document)docs.get(i);

//              

//              ObjectFormat returnedObjectFormat = ObjectFormat.convert(d.getField("objectFormat"));

//              

//              if(returnedObjectFormat != null && 

//                 objectFormat != null && 

//                 !objectFormat.toString().trim().equals(returnedObjectFormat.toString().trim()))

//              { //make sure the objectFormat is the one specified

//                  continue;

//              }

//              

//              String dateSMM = d.getField("dateSysMetadataModified");

//              if((startTime != null || endTime != null) && dateSMM == null)

//              {  //if startTime or endTime are not null, we need a date to compare to

//                  continue;

//              }

//              

//              //date parse

//              Date dateSysMetadataModified = null;

//              if(dateSMM != null)

//              {

//

//                  /*                  

//                  if(dateSMM.indexOf(".") != -1)

//                  {  //strip the milliseconds

//                      //TODO: don't do this. we need milliseconds now.

//                      //TODO: explore ISO 8601 to figure out milliseconds

//                      dateSMM = dateSMM.substring(0, dateSMM.indexOf(".")) + 'Z';

//                  }

//                  */

//                  //System.out.println("dateSMM: " + dateSMM);

//                  //dateFormat.setTimeZone(TimeZone.getTimeZone("GMT-0"));

//                  try

//                  {   //the format we want

//                      dateSysMetadataModified = dateFormat.parse(dateSMM);

//                  }

//                  catch(java.text.ParseException pe)

//                  {   //try another legacy format

//                      try

//                      {

//                          DateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss.S'Z'");

//                          dateFormat2.setTimeZone(TimeZone.getTimeZone("GMT-0"));

//                          dateSysMetadataModified = dateFormat2.parse(dateSMM);

//                      }

//                      catch(java.text.ParseException pe2)

//                      {

//                          //try another legacy format

//                          DateFormat dateFormat3 = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'");

//                          dateFormat3.setTimeZone(TimeZone.getTimeZone("GMT-0"));

//                          dateSysMetadataModified = dateFormat3.parse(dateSMM);

//                      }

//                      

//                  }                  

//              }

//              

//              /*System.out.println("====================================");

//              System.out.println("doc number " + i);

//              System.out.println("docid: " + d.docid);

//              System.out.println("guid: " + d.getField("identifier").trim());

//              System.out.println("dateSMM: " + dateSMM);

//              System.out.println("dateSysMetadataModified: " + dateSysMetadataModified);

//              System.out.println("startTime: " + startTime);

//              System.out.println("endtime: " + endTime);*/

//              

//              int startDateComparison = 0;

//              int endDateComparison = 0;

//              if(startTime != null)

//              {

//                  Calendar zTime = Calendar.getInstance(TimeZone.getTimeZone("GMT-0"));

//                  zTime.setTime(startTime);

//                  startTime = zTime.getTime();

//                  

//                  if(dateSysMetadataModified == null)

//                  {

//                      startDateComparison = -1;

//                  }

//                  else

//                  {

//                      startDateComparison = dateSysMetadataModified.compareTo(startTime);

//                  }

//                  //System.out.println("startDateCom: " + startDateComparison);

//              }

//              else

//              {

//                  startDateComparison = 1;

//              }

//              

//              if(endTime != null)

//              {

//                  Calendar zTime = Calendar.getInstance(TimeZone.getTimeZone("GMT-0"));

//                  zTime.setTime(endTime);

//                  endTime = zTime.getTime();

//                  

//                  if(dateSysMetadataModified == null)

//                  {

//                      endDateComparison = 1;

//                  }

//                  else

//                  {

//                      endDateComparison = dateSysMetadataModified.compareTo(endTime);

//                  }

//                  //System.out.println("endDateCom: " + endDateComparison);

//              }

//              else

//              {

//                  endDateComparison = -1;

//              }

//              

//              

//              if(startDateComparison < 0 || endDateComparison > 0)

//              { 

//                  continue;                  

//              }

//              

//              docCopy.add((Document)docs.get(i));

//          } //end pre-parse

//          

//          docs = docCopy;

//          totalAfterQuery = docs.size();

//          //System.out.println("total after subquery: " + totalAfterQuery);

//          

//          //make sure we don't run over the end

//          int end = start + count;

//          if(end > docs.size())

//          {

//              end = docs.size();

//          }

//          

//          for(int i=start; i<end; i++)

//          {

//              //get the document from the result

//              Document d = (Document)docs.get(i);

//              //System.out.println("processing doc " + d.docid);

//              

//              String dateSMM = d.getField("dateSysMetadataModified");

//              //System.out.println("dateSMM: " + dateSMM);

//              //System.out.println("parsed date: " + parseDate(dateSMM));

//              Date dateSysMetadataModified = null;

//              if(dateSMM != null)

//              {

//                  try

//                  {

//                      dateSysMetadataModified = parseDate(dateSMM);

//                  }

//                  catch(Exception e)

//                  { //if we fail to parse the date, just ignore the value

//                      dateSysMetadataModified = null;

//                  }

//              }

//              ObjectFormat returnedObjectFormat = ObjectFormat.convert(d.getField("objectFormat"));

//                

//              

//              ObjectInfo info = new ObjectInfo();

//              //add the fields to the info object

//              Checksum cs = new Checksum();

//              cs.setValue(d.getField("checksum"));

//              String csalg = d.getField("algorithm");

//              if(csalg == null)

//              {

//                  csalg = "MD5";

//              }

//              ChecksumAlgorithm ca = ChecksumAlgorithm.convert(csalg);

//              cs.setAlgorithm(ca);

//              info.setChecksum(cs);

//              info.setDateSysMetadataModified(dateSysMetadataModified);

//              Identifier id = new Identifier();

//              id.setValue(d.getField("identifier").trim());

//              info.setIdentifier(id);

//              info.setObjectFormat(returnedObjectFormat);

//              String size = d.getField("size");

//              if(size != null)

//              {

//                  info.setSize(new Long(size.trim()).longValue());

//              }

//              //add the ObjectInfo to the ObjectList

//              //logCrud.info("objectFormat: " + info.getObjectFormat().toString());

//              //logCrud.info("id: " + info.getIdentifier().getValue());

//              

//              if(info.getIdentifier().getValue() != null)

//              { //id can be null from tests.  should not happen in production.

//                  if((info.getObjectFormat() != null && !info.getObjectFormat().toString().trim().equals("")))

//                  { //objectFormat needs to not be null and not be an empty string

//                    ol.addObjectInfo(info);

//                  }

//                  else

//                  {

//                      logCrud.info("Not adding object with null objectFormat" + info.getIdentifier().getValue().toString());

//                  }

//              }

//             

//          }

//      }

//      catch(Exception e)

//      {

//          e.printStackTrace();

//          logCrud.error("Error creating ObjectList: " + e.getMessage() + " cause: " + e.getCause());

//          throw new ServiceFailure("1580", "Error retrieving ObjectList: " + e.getMessage());

//      }

//      String username = "public";

//      if(sessionData != null)

//      {

//          username = sessionData.getUserName();

//      }

//      EventLog.getInstance().log(metacatUrl,

//              username, null, "read");

//      logCrud.info("listObjects");

//      if(totalAfterQuery < count)

//      {

//          count = totalAfterQuery;

//      }

//      ol.setCount(count);

//      ol.setStart(start);

//      ol.setTotal(totalAfterQuery);

//      return ol;

    }

    /**

     * Call listObjects with the default values for replicaStatus (true), start (0),

     * and count (1000).

     * @param token

     * @param startTime

     * @param endTime

     * @param objectFormat

     * @return

     * @throws NotAuthorized

     * @throws InvalidRequest

     * @throws NotImplemented

     * @throws ServiceFailure

     * @throws InvalidToken

     */

    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime, 

        ObjectFormat objectFormat)

      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken

    {

       return listObjects(token, startTime, endTime, objectFormat, true, 0, 1000);

    }

    /**

     * Delete a document. 

     */

    public Identifier delete(AuthToken token, Identifier guid)

            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

            NotImplemented, InvalidRequest {

        logCrud.info("delete");

        if(token == null || token.getToken().equals("publid"))

        {

            throw new NotAuthorized("1320", "You must be logged in to delete records.");

        }

        if(guid == null || guid.getValue().trim().equals(""))

        {

            throw new InvalidRequest("1322", "No GUID specified in CrudService.delete()");

        }

        final SessionData sessionData = getSessionData(token);

        IdentifierManager manager = IdentifierManager.getInstance();

        String docid;

        try

        {

            docid = manager.getLocalId(guid.getValue());

        }

        catch(McdbDocNotFoundException mnfe)

        {

            throw new InvalidRequest("1322", "GUID " + guid + " not found.");

        }

        try

        {

            DocumentImpl.delete(docid, sessionData.getUserName(), sessionData.getGroupNames(), null);

        }

        catch(SQLException e)

        {

            throw new ServiceFailure("1350", "Could not delete document: " + e.getMessage());

        }

        catch(McdbDocNotFoundException e)

        {

          throw new ServiceFailure("1350", "Could not delete document: " + e.getMessage());

        }

        catch(InsufficientKarmaException e)

        {

          throw new ServiceFailure("1350", "Could not delete document: " + e.getMessage());

        }

        catch(Exception e)

        {

          throw new ServiceFailure("1350", "Could not delete document: " + e.getMessage());

        }

        return guid;

    }

    /**

     * describe a document.  

     */

    public DescribeResponse describe(AuthToken token, Identifier guid)

            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

            NotImplemented, InvalidRequest {

        logCrud.info("describe");

        if(token == null)

        {

            throw new InvalidToken("1370", "Authentication token is null");

        }

        if(guid == null || guid.getValue().trim().equals(""))

        {

            throw new InvalidRequest("1362", "Guid is null.  A valid guid is required.");

        }

        SystemMetadata sm = getSystemMetadata(token, guid);

        DescribeResponse dr = new DescribeResponse(sm.getObjectFormat(), 

                sm.getSize(), sm.getDateSysMetadataModified(), sm.getChecksum());

        return dr;

    }

    /**

     * get a document with a specified guid.

     */

    public InputStream get(AuthToken token, Identifier guid)

            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

            NotImplemented {

        // Retrieve the session information from the AuthToken

        // If the session is expired, then the user is 'public'

        if(token == null)

        {

            token = new AuthToken("Public");

        }

        final SessionData sessionData = getSessionData(token);

        // Look up the localId for this global identifier

        IdentifierManager im = IdentifierManager.getInstance();

        try 

        {

            final String localId = im.getLocalId(guid.getValue());

            InputStream objectStream;

            try 

            {

                String username = "public";

                String[] groups = new String[0];

                if(sessionData != null)

                {

                    username = sessionData.getUserName();

                    groups = sessionData.getGroupNames();

                }

                objectStream = readFromMetacat(localId, username, groups);

            } catch (PropertyNotFoundException e) {

                e.printStackTrace();

                throw new ServiceFailure("1030", "Error getting property from metacat: " + e.getMessage());

            } catch (ClassNotFoundException e) {

                e.printStackTrace();

                throw new ServiceFailure("1030", "Class not found error when reading from metacat: " + e.getMessage());

            } catch (IOException e) {

                e.printStackTrace();

                throw new ServiceFailure("1030", "IOException while reading from metacat: " + e.getMessage());

            } catch (SQLException e) {

                e.printStackTrace();

                throw new ServiceFailure("1030", "SQLException while reading from metacat: " + e.getMessage());

            } catch (McdbException e) {

                e.printStackTrace();

                throw new ServiceFailure("1030", "Metacat DB exception while reading from metacat: " + e.getMessage());

            } catch (ParseLSIDException e) {

                e.printStackTrace();

                throw new NotFound("1020", "LSID parsing exception while reading from metacat: " + e.getMessage());

            } catch (InsufficientKarmaException e) {

                e.printStackTrace();

                throw new NotAuthorized("1000", "User not authorized for get(): " + e.getMessage());

            }

            String username = "public";

            if(sessionData != null)

            {

                username = sessionData.getUserName();