Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2010 Regents of the University of California and the

 *              National Center for Ecological Analysis and Synthesis

 *  Purpose: To test the Access Controls in metacat by JUnit

 *

 *   '$Author: berkley $'

 *     '$Date: 2010-05-03 14:26:08 -0700 (Fri, 14 Aug 2009) $'

 * '$Revision: 5027 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.dataone;

import java.util.*;

import java.io.*;

import java.security.MessageDigest;

import edu.ucsb.nceas.MCTestCase;

import edu.ucsb.nceas.metacat.IdentifierManager;

import edu.ucsb.nceas.metacat.client.MetacatAuthException;

import edu.ucsb.nceas.metacat.client.MetacatException;

import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;

import edu.ucsb.nceas.metacat.dataone.CrudService;

import junit.framework.Test;

import junit.framework.TestSuite;

import org.apache.commons.io.IOUtils;

import org.apache.log.LogEvent;

import org.dataone.service.exceptions.InvalidRequest;

import org.dataone.service.exceptions.InvalidToken;

import org.dataone.service.exceptions.NotAuthorized;

import org.dataone.service.exceptions.NotImplemented;

import org.dataone.service.exceptions.ServiceFailure;

import org.dataone.service.types.*;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.client.rest.MetacatRestClient;

import edu.ucsb.nceas.metacat.service.SessionService;

import edu.ucsb.nceas.metacat.util.SessionData;

import com.gc.iotools.stream.is.InputStreamFromOutputStream;

/**

 * A JUnit test for testing the dataone CrudService class

 */

public class CrudServiceTest extends MCTestCase 

{   

    private int createCount = 0;

    /**

    * consstructor for the test

    */

    public CrudServiceTest(String name)

    {

        super(name);

    }

    /**

	 * Establish a testing framework by initializing appropriate objects

	 */

	public void setUp() throws Exception 

	{

		super.setUp();

	}

	/**

	 * Release any objects after tests are complete

	 */

	public void tearDown() 

	{

	    System.out.println(createCount + " docs created in this test session.");

	}

	/**

	 * Create a suite of tests to be run together

	 */

	public static Test suite() 

	{

		TestSuite suite = new TestSuite();

		suite.addTest(new CrudServiceTest("initialize"));

		suite.addTest(new CrudServiceTest("testSingletonAccessor"));

		suite.addTest(new CrudServiceTest("testCreateAndGet"));

		suite.addTest(new CrudServiceTest("testGetSystemMetadata"));

		suite.addTest(new CrudServiceTest("testUpdate"));

		suite.addTest(new CrudServiceTest("testListObjects"));

		suite.addTest(new CrudServiceTest("testAccessControl"));

		//suite.addTest(new CrudServiceTest("testGenerateMissingSystemMetadata"));

		suite.addTest(new CrudServiceTest("testGetLogRecords"));

		suite.addTest(new CrudServiceTest("testChecksumError"));

		suite.addTest(new CrudServiceTest("testPublicAccess"));

		suite.addTest(new CrudServiceTest("testFailedCreate"));

		suite.addTest(new CrudServiceTest("testChecksum"));

		return suite;

	}

	/**

	 * test the checksum creation

	 */

	public void testChecksum()

	{

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        System.out.println("token: " + token.getToken());

	        String doc = getTestDoc();

	        Identifier id = createDoc(token, doc);

	        System.out.println("ID: " + id.getValue());

            makeDocPublic(token, id, true);

	        String doc2 = getDoc(token, id);

	        String checksum = checksum(doc);

            Checksum checksum2 = cs.getChecksum(token, id, "MD5");

            System.out.println("doc: \"" + doc + "\"");

            System.out.println("doc2: \"" + doc2 + "\"");

            System.out.println("checksum1: " + checksum);

            System.out.println("checksum2: " + checksum2.getValue());

            assertTrue(checksum.equals(checksum2.getValue()));

	    }

	    catch(Exception e)

	    {

	        fail("Unexpected error in testChecksum: " + e.getMessage());

	    }

	}

	/**

	 * insert an invalid document and make sure create fails and does not

	 * leave any artifacts behind

	 */

	public void testFailedCreate()

	{

	    try

	    {

	        String invalidDoc = "<xxx></yyy>";

	        System.out.println("trying to insert doc " + invalidDoc);

	        CrudService cs = CrudService.getInstance();

            AuthToken token = getToken();

            //run create

            try

            {

                Identifier id = createDoc(token, invalidDoc);

                fail("Should have thrown an exception.");

            }

            catch(Exception e)

            {

                //e.printStackTrace();

            }

	    }

	    catch(Exception e)

	    {

	        fail("unexpected exception in testFailedCreate: " + e.getMessage());

	    }

	}

	/**

	 * test to make sure that get and listObjects methods are publicly accessible

	 */

	public void testPublicAccess()

	{

	    try

	    {

	        AuthToken token = new AuthToken("public");

	        //AuthToken token = getToken();

	        CrudService cs = CrudService.getInstance();

	        ObjectList ol = cs.listObjects(token, null, null, null);

	        System.out.println("ol: " + ol.sizeObjectInfoList());

	        assertTrue(ol.sizeObjectInfoList() > 0);

	        ObjectInfo oi = ol.getObjectInfo(0);

	        String s = getDoc(token, oi.getIdentifier());

	        assertNotNull(s);

	        try

	        { //try a create with the public auth.  should fail

	            Identifier id = new Identifier();

	            String docid = generateDocumentId();

	            id.setValue(docid);

	            String testDoc = getTestDoc();

	            StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

	            ObjectFormat of1 = ObjectFormat.convert("eml://ecoinformatics.org/eml-2.1.0");

	            SystemMetadata sm = createSystemMetadata(id, testDoc, of1);

	            assertFalse(sm.getChecksum().getValue().startsWith("MD5"));

	            //create the doc

	            Identifier idC = cs.create(token, id, sbis, sm);

	            fail("Should have thrown an auth exception");

	        }

	        catch(Exception e)

	        {

	        }

	        //create a legit doc

	        token = getToken();

	        Identifier id = new Identifier();

            String docid = generateDocumentId();

            id.setValue(docid);

            String testDoc = getTestDoc();

            StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

            ObjectFormat of1 = ObjectFormat.convert("eml://ecoinformatics.org/eml-2.1.0");

            SystemMetadata sm = createSystemMetadata(id, testDoc, of1);

            assertFalse(sm.getChecksum().getValue().startsWith("MD5"));

            //create the doc

            Identifier idC = cs.create(token, id, sbis, sm);

            //make it public, then use a public token to try to get it

            InputStream data = null;

            try

            {

                AuthToken publicToken = new AuthToken("public");

                data = cs.get(publicToken, id);

                data.read();

                fail("should have thrown an exception");

            }

            catch(Exception e)

            {

                /*System.out.println("%%%%%%%%%%%%%%%%%%%%%%%getting result");

                Object o = ((InputStreamFromOutputStream)data).getResult();

                System.out.println("result: " + o);

                System.out.println("exception thrown!");

                System.out.println("exception thrown: " + e.getClass().getName());*/

            }

            makeDocPublic(token, id, true);

            token = new AuthToken("public");

	        data = cs.get(token, id);

	    }

	    catch(Exception e)

	    {

	        fail("Error in testPublicAccess: " + e.getMessage());

	    }

	}

	/**

	 * test for an error where the checksum algorithm gets appended onto the checksum.

	 */

	public void testChecksumError()

	{

	    printTestHeader("testChecksumError");

	    try

	    {

	        Date d1 = new Date(new Date().getTime() - 5000);

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        ObjectFormat of1 = ObjectFormat.convert("eml://ecoinformatics.org/eml-2.1.0");

	        //create docs at different times

	        Identifier id = new Identifier();

	        String docid = "test." + new Date().getTime();

	        id.setValue(docid);

	        //create the system metadata then run the create method

	        String testDoc = getTestDoc();

	        StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

	        SystemMetadata sm = createSystemMetadata(id, testDoc, of1);

	        assertFalse(sm.getChecksum().getValue().startsWith("MD5"));

	        //create the doc

	        Identifier idC = cs.create(token, id, sbis, sm);

	        makeDocPublic(token, id, true);

	        assertTrue(idC.getValue().equals(id.getValue()));

	        SystemMetadata smC = getSystemMetadata(token, idC);

	        assertFalse(smC.getChecksum().getValue().startsWith("MD5"));

	        Date d2 = new Date(new Date().getTime() + 5000);

	        ObjectList ol = cs.listObjects(token, d1, d2, of1);

	        assertTrue(ol.sizeObjectInfoList() > 0);

	        ObjectInfo oi = ol.getObjectInfo(0);

	        //this will fail if the error state exists.

	        assertFalse(oi.getChecksum().getValue().startsWith("MD5"));

	    }

	    catch(Exception e)

	    {

	        fail("Unexpected exception: " + e.getMessage());

	    }

	}

	/**

	 * test CrudService.getLogRecords

	 */

	public void testGetLogRecords()

	{

	    printTestHeader("testGetLogRecords");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        Date fromDate = new Date();

	        Identifier id = createDoc(token, getTestDoc());

	        Date toDate = new Date();

	        Log lrs = cs.getLogRecords(token, fromDate, toDate, null);

	        assertNotNull(lrs);

	        assertTrue(lrs.sizeLogEntryList() == 1);

	        LogEntry lrLogEvent = lrs.getLogEntry(0);

	        assertTrue(lrLogEvent.getEvent().name().equals("CREATE"));

	        assertTrue(lrLogEvent.getIdentifier().getValue().equals(id.getValue()));

	    }

	    catch(Exception e)

	    {

	        e.printStackTrace();

	        fail("testGetLogRecords threw an unexpected exception: " + e.getMessage());

	    }

	}

	/**

	 * test the generation of system metadata for docs that don't already 

	 * have it.  This will be used for migration of existing object stores

	 * to dataone.

	 */

	public void testGenerateMissingSystemMetadata()

	{

	    printTestHeader("testGenerateMissingSystemMetadata");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        //create a document with no system metadata

	        String testDoc = getTestDoc();

	        Identifier id = new Identifier();

	        String docid = generateDocumentId();

	        id.setValue(docid);

	        cs.insertOrUpdateDocument(testDoc, id, cs.getSessionData(token), "insert", false); 

	        //try to get its system metadata, should fail

	        try

	        {

	            getSystemMetadata(token, id);

	            fail("call to getSystemMetadata should have failed.");

	        }

	        catch(Exception e)

	        {}

	        //generate missing system metadata

	        cs.generateMissingSystemMetadata(token);

	        //try to get system metadata again, should succeed

	        getSystemMetadata(token, id);

	    }

	    catch(Exception e)

	    {

	        fail("Unexpected error generating missing system metadata: " + e.getMessage());

	    }

	}

	/**

	 * make sure that only valid sessions can update/delete

	 */

	public void testAccessControl()

	{

	    printTestHeader("testAccessControl");

        try

        {

            CrudService cs = CrudService.getInstance();

            AuthToken token = getToken();

            //create a doc

            Identifier id = createDoc(token, getTestDoc());

            //get the doc and sysmetadata

            String gotDoc = getDoc(token, id);

            SystemMetadata sm = getSystemMetadata(token, id);

            //break the session id

            String sessionid = "somefakesessionid";

            token = new AuthToken(sessionid);

            //update the doc

            gotDoc = gotDoc.replaceAll("XXX", "YYY");

            Identifier newid = new Identifier();

            newid.setValue(generateDocumentId());

            StringBufferInputStream sbis = new StringBufferInputStream(gotDoc);

            SystemMetadata newsm = createSystemMetadata(newid, gotDoc);

            Identifier updatedid = cs.update(token, newid, sbis, id, newsm);

            fail("exception should have been thrown.");

        }

        catch(Exception e)

        {

        }

        try

        {

            CrudService cs = CrudService.getInstance();

            AuthToken token = new AuthToken("somefakesessionid");

            //create a doc

            Identifier id = createDoc(token, getTestDoc());

            fail("exception should have been thrown.");

        }

        catch(Exception e)

        {

        }

	}

	/**

	 * public ObjectList listObjects(AuthToken token, Date startTime, Date endTime, 

     *     ObjectFormat objectFormat, boolean replicaStatus, int start, int count)

     *       throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken

	 */

	public void testListObjects()

	{

	    printTestHeader("testListObjects");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        ObjectFormat of1 = ObjectFormat.convert("eml://ecoinformatics.org/eml-2.1.0");

	        ObjectFormat of2 = ObjectFormat.convert("eml://ecoinformatics.org/eml-2.0.0");

	        //create docs at different times

	        Date d1 = new Date();

	        Identifier id1 = createDoc(token, getTestDoc(), of1);

            SystemMetadata sm1 = getSystemMetadata(token, id1);

            Date d2 = new Date();

            Identifier id2 = createDoc(token, getTestDoc(), of2);

            makeDocPublic(token, id2, true);

            SystemMetadata sm2 = getSystemMetadata(token, id2);

            Date d3 = new Date();

            Identifier id3 = createDoc(token, getTestDoc(), of1);

            makeDocPublic(token, id3, true);

            SystemMetadata sm3 = getSystemMetadata(token, id3);

            Date d4 = new Date();

            Identifier id4 = createDoc(token, getTestDoc(), of2);

            makeDocPublic(token, id4, true);

            SystemMetadata sm4 = getSystemMetadata(token, id4);

            Date d5 = new Date();

            Identifier id5 = createDoc(token, getTestDoc(), of1);

            makeDocPublic(token, id5, true);

            SystemMetadata sm5 = getSystemMetadata(token, id5);  

            Date d6 = new Date();

            //now get the objects for specific time ranges and test that it returns

            //the correct objects

            //ObjectList list = cs.listObjects(token, null, null, null);

            //System.out.println("list size: " + list.sizeObjectInfoList());

            //should return sm1 and sm2

            ObjectList list1 = cs.listObjects(token, d1, d3, null);

            assertTrue(list1.sizeObjectInfoList() == 2);

            assertTrue(idInObjectList(id1, list1));

            assertTrue(idInObjectList(id2, list1));

            ObjectInfo info = list1.getObjectInfo(0);

            //should only return sm1

            ObjectList list2 = cs.listObjects(token, d1, d3, of1);

            assertTrue(list2.sizeObjectInfoList() == 1);

            assertTrue(idInObjectList(id1, list2));

            //should return sm1-sm4

            ObjectList list3 = cs.listObjects(token, d1, d5, null);

            assertTrue(list3.sizeObjectInfoList() == 4);

            ObjectInfo oi4 = list3.getObjectInfo(0);

            assertTrue(idInObjectList(id1, list3));

            assertTrue(idInObjectList(id2, list3));

            assertTrue(idInObjectList(id3, list3));

            assertTrue(idInObjectList(id4, list3));

            //should only return sm2 and sm4

            ObjectList list4 = cs.listObjects(token, d1, d5, of2);

            assertTrue(list4.sizeObjectInfoList() == 2);

            assertTrue(idInObjectList(id2, list4));

            assertTrue(idInObjectList(id4, list4));

            //should return all

            ObjectList list5 = cs.listObjects(token, d1, d6, null);

            assertTrue(list5.sizeObjectInfoList() == 5);

            assertTrue(idInObjectList(id1, list5));

            assertTrue(idInObjectList(id2, list5));

            assertTrue(idInObjectList(id3, list5));

            assertTrue(idInObjectList(id4, list5));

            assertTrue(idInObjectList(id5, list5));

            //should return 1, 3, 5

            ObjectList list6 = cs.listObjects(token, d1, d6, of1);

            assertTrue(list6.sizeObjectInfoList() == 3);

            assertTrue(idInObjectList(id1, list6));

            assertTrue(idInObjectList(id3, list6));

            assertTrue(idInObjectList(id5, list6));

            //should return 4 (id1 is not public)

            token = new AuthToken("public");

            ObjectList list7 = cs.listObjects(token, d1, d6, null);

            //System.out.println("list7 size: " + list7.sizeObjectInfoList());

            assertTrue(list7.sizeObjectInfoList() == 4);

            //test paging

            ObjectList list8 = cs.listObjects(token, d1, d6, null, false, 2, 2);

            assertTrue(list8.getCount() == 2);

            assertTrue(list8.getStart() == 2);

            assertTrue(list8.getTotal() == 4);

	    }

	    catch(Exception e)

	    {

	        //e.printStackTrace();

	        fail("Error in listObjects: " + e.getMessage());

	    }

	}

	private boolean idInObjectList(Identifier id, ObjectList list)

	{

	    for(int i=0; i<list.sizeObjectInfoList(); i++)

	    {

	        ObjectInfo oi = list.getObjectInfo(i);

	        if(id.getValue().equals(oi.getIdentifier().getValue()))

	            return true;

	    }

	    return false;

	}

	/**

	 * public Identifier update(AuthToken token, Identifier guid, 

     *       InputStream object, Identifier obsoletedGuid, SystemMetadata sysmeta) 

     *         throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, 

     *           UnsupportedType, InsufficientResources, NotFound, InvalidSystemMetadata, 

     *           NotImplemented

	 */

	public void testUpdate()

	{

	    printTestHeader("testUpdate");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

            //create a doc

            Identifier id = createDoc(token, getTestDoc());

            //get the doc and sysmetadata

            String gotDoc = getDoc(token, id);

            SystemMetadata sm = getSystemMetadata(token, id);

            //update the doc

            gotDoc = gotDoc.replaceAll("XXX", "YYY");

            Identifier newid = new Identifier();

            newid.setValue(generateDocumentId());

            StringBufferInputStream sbis = new StringBufferInputStream(gotDoc);

            SystemMetadata newsm = createSystemMetadata(newid, gotDoc);

            Identifier updatedid = cs.update(token, newid, sbis, id, newsm);

            //get doc - check that it matches update

            String newdoc = getDoc(token, newid);

            assertTrue(gotDoc.equals(newdoc));

            //get sysmeta - check that ids and other fields are updated

            SystemMetadata newnewsm = getSystemMetadata(token, id);

            assertTrue(newnewsm.getObsoletedBy(0).getValue().equals(newid.getValue()));

            //get the new sysmeta and make sure the obsoletes field is set

            SystemMetadata newnewnewsm = getSystemMetadata(token, newid);

            assertTrue(newnewnewsm.getObsolete(0).getValue().equals(id.getValue()));

        }

        catch(Exception e)

        {

            e.printStackTrace();

            fail("Error in testUpdate: " + e.getMessage());

        }

	}

	/**

	 * public SystemMetadata getSystemMetadata(AuthToken token, Identifier guid)

     *       throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

     *       InvalidRequest, NotImplemented

	 */

	public void testGetSystemMetadata()

	{

	    printTestHeader("testGetSystemMetadata");

	    try

	    {

            CrudService cs = CrudService.getInstance();

            AuthToken token = getToken();

            //run create

            Identifier id = createDoc(token, getTestDoc());

            //get the systemMetadata and make sure it is the correct object for this testDoc

            SystemMetadata sm = getSystemMetadata(token, id);

            assertTrue(sm.getIdentifier().getValue().equals(id.getValue()));

            assertTrue(sm.getChecksum().getValue().equals(checksum(getTestDoc())));

            try

            {

                Identifier fakeid = new Identifier();

                fakeid.setValue("somethingfake.234234");

                getSystemMetadata(token, fakeid);

                fail("getSystemMetadata should have thrown an exception.");

            }

            catch(Exception e)

            {

                assertTrue(true);

            }

        }

        catch(Exception e)

        {

            e.printStackTrace();

            fail("Error testing system metadata: " + e.getMessage());

        }

	}

	/**

	 * create(AuthToken token, Identifier guid, InputStream object, SystemMetadata sysmeta) 

     *   throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, 

     *       InsufficientResources, InvalidSystemMetadata, NotImplemented

     *

     * public InputStream get(AuthToken token, Identifier guid)

     *       throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

     *       NotImplemented

	 */

	public void testCreateAndGet()

	{

	    printTestHeader("testCreateAndGet");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        //run create

	        Identifier id = createDoc(token, getTestDoc());

	        //make these docs public for debugging purposes.

	        makeDocPublic(token, id, true);

            //compare the docs

            String gotDoc = getDoc(token, id);

            assertTrue(gotDoc.trim().equals(getTestDoc().trim()));

            try

            {

                Identifier fakeid = new Identifier();

                fakeid.setValue("somethingfake.234234");

                getDoc(token, fakeid);

                fail("testCreateAndGet should have thrown an exception.");

            }

            catch(Exception e)

            {

                assertTrue(true);

            }

        }

        catch(Exception e)

        {

            e.printStackTrace();

            fail("Error in testCreate: " + e.getMessage());

        }

	}

	/**

	 * getInstance()

	 */

	public void testSingletonAccessor()

	{

	    printTestHeader("testSingletonAccessor");

	    CrudService cs = CrudService.getInstance();

	    assertNotNull(cs);

	}

	/**

	 * Run an initial test that always passes to check that the test harness is

	 * working.

	 */

	public void initialize() 

	{

	    printTestHeader("initialize");

		assertTrue(1 == 1);

	}

	/**

	 * return the systemMetadata object for id

	 */

	private SystemMetadata getSystemMetadata(AuthToken token, Identifier id)

	  throws Exception

	{

	    CrudService cs = CrudService.getInstance();

	    return cs.getSystemMetadata(token, id);

	}

	/**

	 * get a doc from metacat using CrudService.get()

	 */

	private String getDoc(AuthToken token, Identifier id)

	  throws Exception

	{

	    CrudService cs = CrudService.getInstance();

        InputStream gotDocStream = cs.get(token, id);

        return IOUtils.toString(gotDocStream);

	}

	/**

	 * return a test document

	 */

	private String getTestDoc()

	{

	    return "<?xml version=\"1.0\"?><test><somecontent>This is some content XXX</somecontent></test>\n";

	}

	/**

	 * authenticate and return a token

	 */

	private AuthToken getToken()

	  throws Exception

	{

	    CrudService cs = CrudService.getInstance();

        //login and get a sessionid

        MetacatRestClient restClient = new MetacatRestClient(cs.getContextUrl());

        String username = PropertyService.getProperty("test.mcUser");

        String password = PropertyService.getProperty("test.mcPassword");

        String response = restClient.login(username, password);

        String sessionid = restClient.getSessionId();

        SessionService sessionService = SessionService.getInstance();

        sessionService.registerSession(new SessionData(sessionid, username, new String[0], password, "CrudServiceLogin"));

        AuthToken token = new AuthToken(sessionid);

        return token;

	}

	/**

	 * create a doc using CrudService.create() and return its id

	 */

	private Identifier createDoc(AuthToken token, String testDoc) throws Exception

	{

	    return createDoc(token, testDoc, ObjectFormat.convert("eml://ecoinformatics.org/eml-2.1.0"));

	}

	/**

	 * create a doc using CrudService.create() and return its id

	 */

	private Identifier createDoc(AuthToken token, String testDoc, ObjectFormat format) throws Exception

	{

	    Identifier id;

        CrudService cs = CrudService.getInstance();

        id = new Identifier();

        String docid = generateDocumentId();

        id.setValue(docid);

        //create the system metadata then run the create method

        StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

        SystemMetadata sm = createSystemMetadata(id, testDoc, format);

        //create the doc

        cs.create(token, id, sbis, sm);

        createCount++;

        return id;

	}

	/**

	 * create a generic SystemMetadata object for testing

	 */

	private SystemMetadata createSystemMetadata(Identifier id, String testDoc)

	  throws Exception

	{

	    return createSystemMetadata(id, testDoc, 

	            ObjectFormat.convert("eml://ecoinformatics.org/eml-2.1.0"));

	}

	/**

	 * create system metadata with a specified id, doc and format

	 */

	private SystemMetadata createSystemMetadata(Identifier id, String testDoc, ObjectFormat format)

	  throws Exception

	{

	    SystemMetadata sm = new SystemMetadata();

        //set the id

        sm.setIdentifier(id);

        sm.setObjectFormat(format);

        //create the checksum

        String checksumS = checksum(testDoc);

        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");

        Checksum checksum = new Checksum();

        checksum.setValue(checksumS);

        checksum.setAlgorithm(ca);

        sm.setChecksum(checksum);

        //set the size

        sm.setSize(testDoc.getBytes().length);

        //submitter

        Principal p = new Principal();

        p.setValue("joe");

        sm.setSubmitter(p);

        sm.setRightsHolder(p);

        sm.setDateUploaded(new Date());

        sm.setDateSysMetadataModified(new Date());

        NodeReference nr = new NodeReference();

        nr.setValue("metacat");

        sm.setOriginMemberNode(nr);

        sm.setAuthoritativeMemberNode(nr);

        return sm;

	}

	/**

	 *  make a document public in metacat by inserting an access document

	 * @param id

	 */

	private void makeDocPublic(AuthToken token, Identifier id, boolean systemMetadataToo)

	  throws Exception

	{

	    CrudService cs = CrudService.getInstance();

	    cs.setAccess(token, id, "public", "read", "allow", "allowFirst", systemMetadataToo);

	}

	/**

	 * print a header to start each test

	 */

	private void printTestHeader(String testName)

	{

	    System.out.println();

	    System.out.println("*************** " + testName + " ***************");

	}

	/**

	 * produce an md5 checksum for item

	 */

	private String checksum(String item)

	  throws Exception

	{

        StringBufferInputStream fis =  new StringBufferInputStream(item);

        byte[] buffer = new byte[1024];

        MessageDigest complete = MessageDigest.getInstance("MD5");

        int numRead;

        do 

        {

          numRead = fis.read(buffer);

          if (numRead > 0) 

          {

            complete.update(buffer, 0, numRead);

          }

        } while (numRead != -1);

        return getHex(complete.digest());

	}

	/**

	 * convert a byte array to a hex string

	 */

	private static String getHex( byte [] raw ) 

	{

	    final String HEXES = "0123456789ABCDEF";

        if ( raw == null ) {

          return null;

        }

        final StringBuilder hex = new StringBuilder( 2 * raw.length );

        for ( final byte b : raw ) {

          hex.append(HEXES.charAt((b & 0xF0) >> 4))

             .append(HEXES.charAt((b & 0x0F)));

        }

        return hex.toString();

    }

}