Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2010 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *

 *   '$Author: jones $'

 *     '$Date: 2010-02-03 17:58:12 -0900 (Wed, 03 Feb 2010) $'

 * '$Revision: 5211 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat;

import java.io.File;

import java.math.BigInteger;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Timestamp;

import java.util.ArrayList;

import java.util.Date;

import java.util.HashMap;

import java.util.Hashtable;

import java.util.List;

import java.util.Vector;

import org.apache.log4j.Logger;

import org.dataone.client.v2.formats.ObjectFormatCache;

import org.dataone.configuration.Settings;

import org.dataone.service.exceptions.BaseException;

import org.dataone.service.exceptions.InvalidSystemMetadata;

import org.dataone.service.types.v1.AccessPolicy;

import org.dataone.service.types.v1.AccessRule;

import org.dataone.service.types.v1.Checksum;

import org.dataone.service.types.v1.Identifier;

import org.dataone.service.types.v1.NodeReference;

import org.dataone.service.types.v1.ObjectFormatIdentifier;

import org.dataone.service.types.v1.ObjectInfo;

import org.dataone.service.types.v1.ObjectList;

import org.dataone.service.types.v1.Permission;

import org.dataone.service.types.v1.Replica;

import org.dataone.service.types.v1.ReplicationPolicy;

import org.dataone.service.types.v1.ReplicationStatus;

import org.dataone.service.types.v1.Subject;

import org.dataone.service.types.v2.MediaType;

import org.dataone.service.types.v2.MediaTypeProperty;

import org.dataone.service.types.v2.SystemMetadata;

import edu.ucsb.nceas.metacat.accesscontrol.XMLAccessAccess;

import edu.ucsb.nceas.metacat.database.DBConnection;

import edu.ucsb.nceas.metacat.database.DBConnectionPool;

import edu.ucsb.nceas.metacat.database.DatabaseService;

import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.shared.AccessException;

import edu.ucsb.nceas.metacat.shared.ServiceException;

import edu.ucsb.nceas.metacat.util.DocumentUtil;

import edu.ucsb.nceas.utilities.FileUtil;

import edu.ucsb.nceas.utilities.PropertyNotFoundException;

import edu.ucsb.nceas.utilities.access.AccessControlInterface;

import edu.ucsb.nceas.utilities.access.XMLAccessDAO;

/**

 * Manage the relationship between Metacat local identifiers (LocalIDs) that are

 * codified as the (docid, rev) pair with globally unique string identifiers

 * (GUIDs) that are opaque strings.  This class provides methods to manage these

 * identifiers, and to search for and look up LocalIDs based on their GUID and

 * vice versa. IdentifierManager is a singleton.

 * 

 * @author Matthew Jones

 */

public class IdentifierManager {

    public static final String TYPE_SYSTEM_METADATA = "systemmetadata";

    public static final String TYPE_IDENTIFIER = "identifier";

    private static boolean filterWhiteSpaces = Settings.getConfiguration().getBoolean("dataone.listingidentifier.filteringwhitespaces", true);

    /**

     * The single instance of the manager that is always returned.

     */

    private static IdentifierManager self = null;

    private Logger logMetacat = Logger.getLogger(IdentifierManager.class);

    /**

     * A private constructor that initializes the class when getInstance() is

     * called.

     */

    private IdentifierManager() {}

    /**

     * Return the single instance of the manager after initializing it if it

     * wasn't previously initialized.

     * 

     * @return the single IdentifierManager instance

     */

    public static IdentifierManager getInstance()

    {

        if (self == null) {

            self = new IdentifierManager();

        }

        return self;

    }

    /*public SystemMetadata asSystemMetadata(Date dateUploaded, String rightsHolder,

            String checksum, String checksumAlgorithm, String originMemberNode,

            String authoritativeMemberNode, Date dateModified, String submitter, 

            String guid, String fmtidStr, BigInteger size, BigInteger serialVersion) {

        SystemMetadata sysMeta = new SystemMetadata();

        Identifier sysMetaId = new Identifier();

        sysMetaId.setValue(guid);

        sysMeta.setIdentifier(sysMetaId);

        sysMeta.setDateUploaded(dateUploaded);

        Subject rightsHolderSubject = new Subject();

        rightsHolderSubject.setValue(rightsHolder);

        sysMeta.setRightsHolder(rightsHolderSubject);

        Checksum checksumObject = new Checksum();

        checksumObject.setValue(checksum);

        checksumObject.setAlgorithm(checksumAlgorithm);

        sysMeta.setChecksum(checksumObject);

        NodeReference omn = new NodeReference();

        omn.setValue(originMemberNode);

        sysMeta.setOriginMemberNode(omn);

        NodeReference amn = new NodeReference();

        amn.setValue(authoritativeMemberNode);

        sysMeta.setAuthoritativeMemberNode(amn);

        sysMeta.setDateSysMetadataModified(dateModified);

        Subject submitterSubject = new Subject();

        submitterSubject.setValue(submitter);

        sysMeta.setSubmitter(submitterSubject);

        ObjectFormatIdentifier fmtid = null;

        try {

        	ObjectFormatIdentifier formatId = new ObjectFormatIdentifier();

        	formatId.setValue(fmtidStr);

        	fmtid = ObjectFormatCache.getInstance().getFormat(formatId).getFormatId();

        	sysMeta.setFormatId(fmtid);

        } catch (BaseException nfe) {

            logMetacat.error("The objectFormat " + fmtidStr +

          	" is not registered. Setting the default format id.");

            fmtid = new ObjectFormatIdentifier();

            fmtid.setValue("application/octet-stream");

            sysMeta.setFormatId(fmtid);

        }

        sysMeta.setSize(size);

        sysMeta.setSerialVersion(serialVersion);

        return sysMeta;

    }*/

    /**

     * return a hash of all of the info that is in the systemmetadata table

     * @param localId

     * @return

     */

    /*public Hashtable<String, String> getSystemMetadataInfo(String localId)

    throws McdbDocNotFoundException

    {

        try

        {

            AccessionNumber acc = new AccessionNumber(localId, "NONE");

            localId = acc.getDocid();

        }

        catch(Exception e)

        {

            //do nothing. just try the localId as it is

        }

        Hashtable<String, String> h = new Hashtable<String, String>();

        String sql = "select guid, date_uploaded, rights_holder, checksum, checksum_algorithm, " +

          "origin_member_node, authoritive_member_node, date_modified, submitter, object_format, size " +

          "from systemmetadata where docid = ?";

        DBConnection dbConn = null;

        int serialNumber = -1;

        try 

        {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getDocumentInfo");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the insert statement

            PreparedStatement stmt = dbConn.prepareStatement(sql);

            stmt.setString(1, localId);

            ResultSet rs = stmt.executeQuery();

            if (rs.next()) 

            {

                String guid = rs.getString(1);

                Timestamp dateUploaded = rs.getTimestamp(2);

                String rightsHolder = rs.getString(3);

                String checksum = rs.getString(4);

                String checksumAlgorithm = rs.getString(5);

                String originMemberNode = rs.getString(6);

                String authoritativeMemberNode = rs.getString(7);

                Timestamp dateModified = rs.getTimestamp(8);

                String submitter = rs.getString(9);

                String objectFormat = rs.getString(10);

                long size = new Long(rs.getString(11)).longValue();

                h.put("guid", guid);

                h.put("date_uploaded", new Long(dateUploaded.getTime()).toString());

                h.put("rights_holder", rightsHolder);

                h.put("checksum", checksum);

                h.put("checksum_algorithm", checksumAlgorithm);

                h.put("origin_member_node", originMemberNode);

                h.put("authoritative_member_node", authoritativeMemberNode);

                h.put("date_modified", new Long(dateModified.getTime()).toString());

                h.put("submitter", submitter);

                h.put("object_format", objectFormat);

                h.put("size", new Long(size).toString());

                stmt.close();

            } 

            else

            {

                stmt.close();

                DBConnectionPool.returnDBConnection(dbConn, serialNumber);

                throw new McdbDocNotFoundException("2Could not find document " + localId);

            }

        } 

        catch (SQLException e) 

        {

            e.printStackTrace();

            logMetacat.error("Error while getting system metadata info for localid " + localId + " : "  

                    + e.getMessage());

        } 

        finally 

        {

            // Return database connection to the pool

            DBConnectionPool.returnDBConnection(dbConn, serialNumber);

        }

        return h;

    }*/

    /**

     * return a hash of all of the info that is in the systemmetadata table

     * @param guid

     * @return

     * @throws McdbDocNotFoundException 

     */

    public SystemMetadata getSystemMetadata(String guid)

    	throws McdbDocNotFoundException

    {

        SystemMetadata sysMeta = new SystemMetadata();

        String sql = "select guid, date_uploaded, rights_holder, checksum, checksum_algorithm, " +

          "origin_member_node, authoritive_member_node, date_modified, submitter, object_format, size, " +

          "replication_allowed, number_replicas, obsoletes, obsoleted_by, serial_version, archived, series_id, file_name, media_type " +

          "from systemmetadata where guid = ?";

        DBConnection dbConn = null;

        int serialNumber = -1;

        Boolean replicationAllowed = new Boolean(false);

        BigInteger numberOfReplicas = new BigInteger("-1");

        BigInteger serialVersion = new BigInteger("-1");

        Boolean archived = new Boolean(false);

        try 

        {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getSystemMetadata");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the statement

            PreparedStatement stmt = dbConn.prepareStatement(sql);

            stmt.setString(1, guid);

            ResultSet rs = stmt.executeQuery();

            if (rs.next()) 

            {

                Timestamp dateUploaded = rs.getTimestamp(2);

                String rightsHolder = rs.getString(3);

                String checksum = rs.getString(4);

                String checksumAlgorithm = rs.getString(5);

                String originMemberNode = rs.getString(6);

                String authoritativeMemberNode = rs.getString(7);

                Timestamp dateModified = rs.getTimestamp(8);

                String submitter = rs.getString(9);

                String fmtidStr = rs.getString(10);

                BigInteger size = new BigInteger(rs.getString(11));

                replicationAllowed = new Boolean(rs.getBoolean(12));

                numberOfReplicas = new BigInteger(rs.getString(13));

                String obsoletes = rs.getString(14);

                String obsoletedBy = rs.getString(15);

                serialVersion = new BigInteger(rs.getString(16));

                archived = new Boolean(rs.getBoolean(17));

                String series_id = rs.getString(18);

                String file_name = rs.getString(19);

                String media_type = rs.getString(20);

                Identifier sysMetaId = new Identifier();

                sysMetaId.setValue(guid);

                sysMeta.setIdentifier(sysMetaId);

                sysMeta.setSerialVersion(serialVersion);

                sysMeta.setDateUploaded(dateUploaded);

                Subject rightsHolderSubject = new Subject();

                rightsHolderSubject.setValue(rightsHolder);

                sysMeta.setRightsHolder(rightsHolderSubject);

                Checksum checksumObject = new Checksum();

                checksumObject.setValue(checksum);

                checksumObject.setAlgorithm(checksumAlgorithm);

                sysMeta.setChecksum(checksumObject);

                if (originMemberNode != null) {

	                NodeReference omn = new NodeReference();

	                omn.setValue(originMemberNode);

	                sysMeta.setOriginMemberNode(omn);

                }

                if (authoritativeMemberNode != null) {

	                NodeReference amn = new NodeReference();

	                amn.setValue(authoritativeMemberNode);

	                sysMeta.setAuthoritativeMemberNode(amn);

                }

                sysMeta.setDateSysMetadataModified(dateModified);

                if (submitter != null) {

	                Subject submitterSubject = new Subject();

	                submitterSubject.setValue(submitter);

	                sysMeta.setSubmitter(submitterSubject);

                }

                ObjectFormatIdentifier fmtid = new ObjectFormatIdentifier();

                fmtid.setValue(fmtidStr);

            	sysMeta.setFormatId(fmtid);

                sysMeta.setSize(size);

                if (obsoletes != null) {

	                Identifier obsoletesId = new Identifier();

	                obsoletesId.setValue(obsoletes);

	                sysMeta.setObsoletes(obsoletesId);

                }

                if (obsoletedBy != null) {

		            Identifier obsoletedById = new Identifier();

		            obsoletedById.setValue(obsoletedBy);

		            sysMeta.setObsoletedBy(obsoletedById);

                }

                sysMeta.setArchived(archived);

                if(series_id != null) {

                    Identifier seriesId = new Identifier();

                    seriesId.setValue(series_id);

                    sysMeta.setSeriesId(seriesId);

                }

                if(file_name != null ) {

                    sysMeta.setFileName(file_name);

                }

                if(media_type != null ) {

                    MediaType mediaType = new MediaType();

                    mediaType.setName(media_type);

                    // get media type properties from another table.

                    String mediaTypePropertyQuery = "select name, value from smmediatypeproperties where guid = ?";

                    PreparedStatement stmt2 = dbConn.prepareStatement(mediaTypePropertyQuery);

                    stmt2.setString(1, guid);

                    ResultSet rs2 = stmt2.executeQuery();

                    while (rs2.next()) {

                        String name = rs2.getString(1);

                        String value = rs2.getString(2);

                        MediaTypeProperty property = new MediaTypeProperty();

                        property.setName(name);

                        property.setValue(value);

                        mediaType.addProperty(property);

                    }

                    sysMeta.setMediaType(mediaType);

                    rs2.close();

                    stmt2.close();

                }

                stmt.close();

            } 

            else

            {

                stmt.close();

                DBConnectionPool.returnDBConnection(dbConn, serialNumber);

                throw new McdbDocNotFoundException("Could not find " + guid);

            }

        } 

        catch (SQLException e) 

        {

            e.printStackTrace();

            logMetacat.error("Error while getting system metadata for guid " + guid + " : "  

                    + e.getMessage());

        } 

        finally 

        {

            // Return database connection to the pool

            DBConnectionPool.returnDBConnection(dbConn, serialNumber);

        }

        // populate the replication policy

        ReplicationPolicy replicationPolicy = new ReplicationPolicy();

        if ( numberOfReplicas != null  && numberOfReplicas.intValue() != -1 ) {

            replicationPolicy.setNumberReplicas(numberOfReplicas.intValue());

        }

        if ( replicationAllowed != null ) {

            replicationPolicy.setReplicationAllowed(replicationAllowed);

        }

        replicationPolicy.setBlockedMemberNodeList(getReplicationPolicy(guid, "blocked"));

        replicationPolicy.setPreferredMemberNodeList(getReplicationPolicy(guid, "preferred"));

		    sysMeta.setReplicationPolicy(replicationPolicy);

		    // look up replication status

		    sysMeta.setReplicaList(getReplicationStatus(guid));

		    // look up access policy

		    try {

		    	sysMeta.setAccessPolicy(getAccessPolicy(guid));

		    } catch (AccessException e) {

		    	throw new McdbDocNotFoundException(e);

		    }

        return sysMeta;

    }

    private List<NodeReference> getReplicationPolicy(String guid, String policy)

		throws McdbDocNotFoundException {

		List<NodeReference> nodes = new ArrayList<NodeReference>();

		String sql = "select guid, policy, member_node " +

			"from smReplicationPolicy where guid = ? and policy = ? order by policy_id ASC";

	    DBConnection dbConn = null;

	    int serialNumber = -1;

	    try {

	        // Get a database connection from the pool

	        dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getReplicationPolicy");

	        serialNumber = dbConn.getCheckOutSerialNumber();

	        // Execute the statement

	        PreparedStatement stmt = dbConn.prepareStatement(sql);

	        stmt.setString(1, guid);

	        stmt.setString(2, policy);

	        ResultSet rs = stmt.executeQuery();

	        while (rs.next()) 

	        {

	            String memberNode = rs.getString(3);

	            NodeReference node = new NodeReference();

	            node.setValue(memberNode);

	            nodes.add(node);

	        } 

	        stmt.close();

	    } catch (SQLException e) {

	        logMetacat.error("Error while getting system metadata replication policy for guid " + guid, e);

	    } 

	    finally {

	        // Return database connection to the pool

	        DBConnectionPool.returnDBConnection(dbConn, serialNumber);

	    }

	    return nodes;

	}

    private List<Replica> getReplicationStatus(String guid) throws McdbDocNotFoundException {

		List<Replica> replicas = new ArrayList<Replica>();

		String sql = "select guid, member_node, status, date_verified " +

			"from smReplicationStatus where guid = ?";

	    DBConnection dbConn = null;

	    int serialNumber = -1;

	    try {

	        // Get a database connection from the pool

	        dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getReplicas");

	        serialNumber = dbConn.getCheckOutSerialNumber();

	        // Execute the statement

	        PreparedStatement stmt = dbConn.prepareStatement(sql);

	        stmt.setString(1, guid);

	        ResultSet rs = stmt.executeQuery();

	        while (rs.next()) 

	        {

	            String memberNode = rs.getString(2);

	            String status = rs.getString(3);

	            java.sql.Timestamp verified = rs.getTimestamp(4);

	            Replica replica = new Replica();	            

	            NodeReference node = new NodeReference();

	            node.setValue(memberNode);

	            replica.setReplicaMemberNode(node);

	            replica.setReplicationStatus(ReplicationStatus.valueOf(status));

	            replica.setReplicaVerified(new Date(verified.getTime()));

	            replicas.add(replica);

	        } 

	        stmt.close();

	    } catch (SQLException e) {

	        logMetacat.error("Error while getting system metadata replication policy for guid " + guid, e);

	    } 

	    finally {

	        // Return database connection to the pool

	        DBConnectionPool.returnDBConnection(dbConn, serialNumber);

	    }

	    return replicas;

	}

    /**

     * return the newest rev for a given localId

     * @param localId

     * @return

     */

    public int getLatestRevForLocalId(String localId)

        throws McdbDocNotFoundException

    {

        try

        {

            AccessionNumber acc = new AccessionNumber(localId, "NONE");

            localId = acc.getDocid();

        }

        catch(Exception e)

        {

            //do nothing. just try the localId as it is

        }

        int rev = 0;

        String sql = "select rev from xml_documents where docid like ? ";

        DBConnection dbConn = null;

        int serialNumber = -1;

        try 

        {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getLatestRevForLocalId");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the insert statement

            PreparedStatement stmt = dbConn.prepareStatement(sql);

            stmt.setString(1, localId);

            ResultSet rs = stmt.executeQuery();

            if (rs.next()) 

            {

                rev = rs.getInt(1);

                stmt.close();

            } 

            else

            {

                stmt.close();

                DBConnectionPool.returnDBConnection(dbConn, serialNumber);

                throw new McdbDocNotFoundException("While trying to get the latest rev, could not find document " + localId);

            }

        } 

        catch (SQLException e) 

        {

            logMetacat.error("Error while looking up the guid: " 

                    + e.getMessage());

        } 

        finally 

        {

            // Return database connection to the pool

            DBConnectionPool.returnDBConnection(dbConn, serialNumber);

        }

        return rev;

    }

    /**

     * return all local ids in the object store that do not have associated

     * system metadata

     */

    public List<String> getLocalIdsWithNoSystemMetadata(boolean includeRevisions, int serverLocation)

    {

        Vector<String> ids = new Vector<String>();

        String sql = "select docid, rev from xml_documents " +

        		"where docid not in " +

        		"(select docid from identifier where guid in (select guid from systemmetadata))";

        if (serverLocation > 0) {

        	sql = sql + " and server_location = ? ";

        }

        String revisionSql = "select docid, rev from xml_revisions " +

				"where docid not in " +

				"(select docid from identifier where guid in (select guid from systemmetadata))";

        if (serverLocation > 0) {

        	revisionSql = revisionSql + " and server_location = ? ";

        }

        if (includeRevisions) {

        	sql = sql + " UNION ALL " + revisionSql;

        }

        DBConnection dbConn = null;

        int serialNumber = -1;

        try 

        {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getLocalIdsWithNoSystemMetadata");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the insert statement

            PreparedStatement stmt = dbConn.prepareStatement(sql);

            // set params based on what we have in the query string

            if (serverLocation > 0) {

            	stmt.setInt(1, serverLocation);

            	if (includeRevisions) {

            		stmt.setInt(2, serverLocation);

            	}

            }

            ResultSet rs = stmt.executeQuery();

            while (rs.next()) 

            {

                String localid = rs.getString(1);

                String rev = rs.getString(2);

                localid += "." + rev;

                logMetacat.debug("id to add SM for: " + localid);

                ids.add(localid);

            } 

            stmt.close();

        } 

        catch (SQLException e) 

        {

            logMetacat.error("Error while looking up the guid: " 

                    + e.getMessage());

        } 

        finally 

        {

            // Return database connection to the pool

            DBConnectionPool.returnDBConnection(dbConn, serialNumber);

        }

        return ids;

    }

    /**

     * return a listing of all local ids in the object store

     * @return a list of all local ids in metacat

     */

    public List<String> getAllLocalIds()

    // seems to be an unnecessary and restrictive throw -rnahf 13-Sep-2011

    //    throws Exception

    {

        Vector<String> ids = new Vector<String>();

        String sql = "select docid from xml_documents";

        DBConnection dbConn = null;

        int serialNumber = -1;

        try 

        {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getAllLocalIds");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the insert statement

            PreparedStatement stmt = dbConn.prepareStatement(sql);

            ResultSet rs = stmt.executeQuery();

            while (rs.next()) 

            {

                String localid = rs.getString(1);

                ids.add(localid);

            } 

            stmt.close();

        } 

        catch (SQLException e) 

        {

            logMetacat.error("Error while looking up the guid: " 

                    + e.getMessage());

        } 

        finally 

        {

            // Return database connection to the pool

            DBConnectionPool.returnDBConnection(dbConn, serialNumber);

        }

        return ids;

    }

    /**

     * return a listing of all guids in the object store

     * @return a list of all GUIDs in metacat

     */

    public List<String> getAllSystemMetadataGUIDs()

    {

        Vector<String> guids = new Vector<String>();

        String sql = "select guid from systemmetadata";

        DBConnection dbConn = null;

        int serialNumber = -1;

        try 

        {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getAllGUIDs");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the insert statement

            PreparedStatement stmt = dbConn.prepareStatement(sql);

            ResultSet rs = stmt.executeQuery();

            while (rs.next()) 

            {

                String guid = rs.getString(1);

                guids.add(guid);

            } 

            stmt.close();

        } 

        catch (SQLException e) 

        {

            logMetacat.error("Error while retrieving the guid: " 

                    + e.getMessage());

        } 

        finally 

        {

            // Return database connection to the pool

            DBConnectionPool.returnDBConnection(dbConn, serialNumber);

        }

        return guids;

    }

    /**

     * returns a list of system metadata-only guids since the given date

     * @return a list of system ids in metacat that do not correspond to objects

     * TODO: need to check which server they are on

     */

    public List<String> getUpdatedSystemMetadataIds(Date since)

       throws Exception

    {

        List<String> ids = new Vector<String>();

        String sql = 

        	"select guid from " + TYPE_SYSTEM_METADATA +

        	" where guid not in " +

        	" (select guid from " + TYPE_IDENTIFIER + ") " +

        	" and date_modified > ?";

        DBConnection dbConn = null;

        int serialNumber = -1;

        try 

        {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getUpdatedSystemMetadataIds");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the insert statement

            PreparedStatement stmt = dbConn.prepareStatement(sql);

            stmt.setDate(1, new java.sql.Date(since.getTime()));

            ResultSet rs = stmt.executeQuery();

            while (rs.next()) 

            {

                String guid = rs.getString(1);

                ids.add(guid);

            } 

            stmt.close();

        } 

        catch (SQLException e) 

        {

            logMetacat.error("Error while looking up the updated guids: " 

                    + e.getMessage());

        } 

        finally 

        {

            // Return database connection to the pool

            DBConnectionPool.returnDBConnection(dbConn, serialNumber);

        }

        return ids;

    }

    /**

     * returns a list of system metadata-only guids since the given date

     * @return a list of system ids in metacat that do not correspond to objects

     * TODO: need to check which server they are on

     */

    public Date getLastModifiedDate() throws Exception {

        Date maxDate = null;

        List<String> ids = new Vector<String>();

        String sql = 

        	"select max(date_modified) from " + TYPE_SYSTEM_METADATA;

        DBConnection dbConn = null;

        int serialNumber = -1;

        try 

        {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getLastModifiedDate");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the insert statement

            PreparedStatement stmt = dbConn.prepareStatement(sql);

            ResultSet rs = stmt.executeQuery();

            if (rs.next()) {

            	maxDate = rs.getDate(1);

            } 

            stmt.close();

        } 

        catch (SQLException e) 

        {

            logMetacat.error("Error while looking up the latest update date: " 

                    + e.getMessage());

        } 

        finally 

        {

            // Return database connection to the pool

            DBConnectionPool.returnDBConnection(dbConn, serialNumber);

        }

        return maxDate;

    }

    /**

     * Determine if an identifier exists already, returning true if so.

     * NOTE: looks in the identifier and system metadata table for a match

     * (in that order)

     * 

     * @param guid the global identifier to look up

     * @return boolean true if the identifier exists

     */

    public boolean identifierExists(String guid) throws SQLException

    {

        boolean idExists = false;

        try {

            String id = getLocalId(guid);

            if (id != null) {

                idExists = true;

            }

        } catch (McdbDocNotFoundException e) {

        	// try system metadata only

        	    //this will check if the guid field on the system metadata table has the id

        		idExists = systemMetadataPIDExists(guid);

        		if(!idExists) {

        		    //if the guid field of the system metadata table doesn't have the id,

        		    //we will check if the serial_id field of the system metadata table has it

        		    idExists=systemMetadataSIDExists(guid);

        		}

        }

        return idExists;

    }

    /**

     * Determine if an identifier mapping exists already, 

     * returning true if so.

     * 

     * @param guid the global identifier to look up

     * @return boolean true if the identifier exists

     */

    public boolean mappingExists(String guid) throws SQLException

    {

        boolean idExists = false;

        try {

            String id = getLocalId(guid);

            if (id != null) {

                idExists = true;

            }

        } catch (McdbDocNotFoundException e) {

        	// nope!

        }

        return idExists;

    }

    /**

     * 

     * @param guid

     * @param rev

     * @return

     */

    public String generateLocalId(String guid, int rev)

    {

        return generateLocalId(guid, rev, false);

    }

    /**

     * Given a global identifier (guid), create a suitable local identifier that

     * follows Metacat's docid semantics and format (scope.id.rev), and create

     * a mapping between these two identifiers.  This effectively reserves both

     * the global and the local identifier, as they will now be present in the

     * identifier mapping table.  

     * 

     * REMOVED feature: If the incoming guid has the syntax of a

     * Metacat docid (scope.id.rev), then simply use it.

     * WHY: because "test.1.001" becomes "test.1.1" which is not correct for DataONE

     * identifier use (those revision numbers are just chartacters and should not be interpreted)

     * 

     * @param guid the global string identifier

     * @param rev the revision number to be used in the localId

     * @return String containing the localId to be used for Metacat operations

     */

    public String generateLocalId(String guid, int rev, boolean isSystemMetadata) 

    {

        String localId = "";

        boolean conformsToDocidFormat = false;

        // BRL -- do not allow Metacat-conforming IDs to be used:

        // test.1.001 becomes test.1.1 which is NOT correct for DataONE identifiers

        // Check if the guid passed in is already in docid (scope.id.rev) format

//        try {

//            AccessionNumber acc = new AccessionNumber(guid, "NONE");

//            if (new Integer(acc.getRev()).intValue() > 0) {

//                conformsToDocidFormat = true;

//            }

//        } catch (NumberFormatException e) {

//            // No action needed, simply detecting invalid AccessionNumbers

//        } catch (AccessionNumberException e) {

//            // No action needed, simply detecting invalid AccessionNumbers

//        } catch (SQLException e) {

//            // No action needed, simply detecting invalid AccessionNumbers

//        }

        if (conformsToDocidFormat) {

            // if it conforms, use it for both guid and localId

            localId = guid;

        } else {

            // if not, then generate a new unique localId

            localId = DocumentUtil.generateDocumentId(rev);

        }

        // Register this new pair in the identifier mapping table

        logMetacat.debug("creating mapping in generateLocalId");

        if(!isSystemMetadata)

        { //don't do this if we're generating for system metadata

            createMapping(guid, localId);

        }

        return localId;

    }

    /**

     * given a local identifer, look up the guid.  Throw McdbDocNotFoundException

     * if the docid, rev is not found in the identifiers or systemmetadata tables

     *

     * @param docid the docid to look up

     * @param rev the revision of the docid to look up

     * @return String containing the mapped guid

     * @throws McdbDocNotFoundException if the docid, rev is not found

     */

    public String getGUID(String docid, int rev)

      throws McdbDocNotFoundException

    {

        logMetacat.debug("getting guid for " + docid);

        String query = "select guid from identifier where docid = ? and rev = ?";

        String guid = null;

        DBConnection dbConn = null;

        int serialNumber = -1;

        PreparedStatement stmt = null;

        try {

            // Get a database connection from the pool

            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getGUID");

            serialNumber = dbConn.getCheckOutSerialNumber();

            // Execute the insert statement

            stmt = dbConn.prepareStatement(query);

            stmt.setString(1, docid);

            stmt.setInt(2, rev);

            ResultSet rs = stmt.executeQuery();

            if (rs.next()) 

            {

                guid = rs.getString(1);

            } 

            else

            {

            	throw new McdbDocNotFoundException("No guid registered for docid " + docid + "." + rev);

            }

            if(rs != null) {

                rs.close();

            }

        } catch (SQLException e) {

            logMetacat.error("Error while looking up the guid: " 

                    + e.getMessage());

        } finally {