Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2010 Regents of the University of California and the

 *              National Center for Ecological Analysis and Synthesis

 *  Purpose: To test the Access Controls in metacat by JUnit

 *

 *   '$Author: berkley $'

 *     '$Date: 2010-05-03 14:26:08 -0700 (Fri, 14 Aug 2009) $'

 * '$Revision: 5027 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.dataone;

import java.io.InputStream;

import java.io.StringBufferInputStream;

import java.security.MessageDigest;

import java.util.Date;

import junit.framework.Test;

import junit.framework.TestSuite;

import org.apache.commons.io.IOUtils;

import org.dataone.client.ObjectFormatCache;

import org.dataone.service.exceptions.BaseException;

import org.dataone.service.types.AuthToken;

import org.dataone.service.types.Checksum;

import org.dataone.service.types.ChecksumAlgorithm;

import org.dataone.service.types.DescribeResponse;

import org.dataone.service.types.Identifier;

import org.dataone.service.types.Log;

import org.dataone.service.types.LogEntry;

import org.dataone.service.types.NodeReference;

import org.dataone.service.types.ObjectFormat;

import org.dataone.service.types.ObjectInfo;

import org.dataone.service.types.ObjectList;

import org.dataone.service.types.Subject;

import org.dataone.service.types.SystemMetadata;

import edu.ucsb.nceas.MCTestCase;

import edu.ucsb.nceas.metacat.MetaCatServlet;

import edu.ucsb.nceas.metacat.client.rest.MetacatRestClient;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.service.SessionService;

import edu.ucsb.nceas.metacat.util.SessionData;

/**

 * A JUnit test for testing the dataone CrudService class

 */

public class CrudServiceTest extends MCTestCase 

{   

    private int createCount = 0;

    /**

    * consstructor for the test

    */

    public CrudServiceTest(String name)

    {

        super(name);

    }

    /**

	 * Establish a testing framework by initializing appropriate objects

	 */

	public void setUp() throws Exception 

	{

		super.setUp();

	}

	/**

	 * Release any objects after tests are complete

	 */

	public void tearDown() 

	{

	    System.out.println(createCount + " docs created in this test session.");

	}

	/**

	 * Create a suite of tests to be run together

	 */

	public static Test suite() 

	{

		TestSuite suite = new TestSuite();

		suite.addTest(new CrudServiceTest("initialize"));

		suite.addTest(new CrudServiceTest("testSingletonAccessor"));

		suite.addTest(new CrudServiceTest("testCreateAndGet"));

		suite.addTest(new CrudServiceTest("testGetSystemMetadata"));

		suite.addTest(new CrudServiceTest("testUpdate"));

		suite.addTest(new CrudServiceTest("testListObjects"));

		suite.addTest(new CrudServiceTest("testAccessControl"));

		suite.addTest(new CrudServiceTest("testGetLogRecords"));

		suite.addTest(new CrudServiceTest("testChecksumError"));

		suite.addTest(new CrudServiceTest("testPublicAccess"));

		suite.addTest(new CrudServiceTest("testFailedCreate"));

		suite.addTest(new CrudServiceTest("testChecksum"));

		suite.addTest(new CrudServiceTest("testDescribe"));

		suite.addTest(new CrudServiceTest("testDelete"));

		suite.addTest(new CrudServiceTest("testSemiColonsInIdentifiers"));

		return suite;

	}

	/**

	 * test for the use of semi colons in identifiers

	 * in response to https://redmine.dataone.org/issues/1143

	 */

	public void testSemiColonsInIdentifiers()

	{

	    try

        {

            CrudService cs = CrudService.getInstance();

            AuthToken token = getToken();

            String testDoc = getTestDoc();

            Identifier id = new Identifier();

            id.setValue("someid;id;with;semi;colons;in;it." + new Date().getTime());

            String docid = generateDocumentId();

            //create the system metadata then run the create method

            StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

            SystemMetadata sm = createSystemMetadata(id, testDoc, 

                    ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0"));

            //create the doc

            Identifier rGuid = cs.create(token, id, sbis, sm);

            System.out.println("ID: " + rGuid.getValue());

            assertTrue(rGuid.toString().equals(id.toString())); 

            System.out.println("created doc with id " + id.getValue());

        }

        catch(Exception e)

        {

            fail("Unexpected error in testDescribe: " + e.getMessage());

        }

	}

	/**

	 * test the delete function

	 */

	public void testDelete()

	{

	    try

        {

            CrudService cs = CrudService.getInstance();

            AuthToken token = getToken();

            String doc = getTestDoc();

            Identifier id = createDoc(token, doc);

            System.out.println("ID: " + id.getValue());

            makeDocPublic(token, id, true);

            cs.get(token, id);

            cs.delete(token, id);

            //there should be a try/catch around the next statement

            //to make sure that the document is no longer readable,

            //but because the data is output in a 2nd thread, the

            //exception does not get caught here.  See 

            //https://redmine.dataone.org/issues/1079

            //cs.get(token, id);

        }

        catch(Exception e)

        {

            fail("Unexpected error in testDescribe: " + e.getMessage());

        }

	}

	/**

	 * test the describe crud function

	 */

	public void testDescribe()

	{

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        String doc = getTestDoc();

	        Identifier id = createDoc(token, doc);

	        System.out.println("ID: " + id.getValue());

	        makeDocPublic(token, id, true);

	        SystemMetadata sm = cs.getSystemMetadata(token, id);

	        DescribeResponse dr = cs.describe(token, id);

	        assertTrue(dr.getDataONE_Checksum().getValue().equals(sm.getChecksum().getValue()));

	        assertTrue(dr.getDataONE_ObjectFormat().toString().equals(sm.getObjectFormat().getFmtid().getValue()));

	        assertTrue(dr.getLast_Modified().getTime() == sm.getDateSysMetadataModified().getTime());

	        assertTrue(dr.getContent_Length() == sm.getSize());

	    }

	    catch(Exception e)

	    {

	        fail("Unexpected error in testDescribe: " + e.getMessage());

	    }

	}

	/**

	 * test the checksum creation

	 */

	public void testChecksum()

	{

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        System.out.println("token: " + token.getToken());

	        String doc = getTestDoc();

	        Identifier id = createDoc(token, doc);

	        System.out.println("ID: " + id.getValue());

            makeDocPublic(token, id, true);

	        String doc2 = getDoc(token, id);

	        String checksum = checksum(doc);

            Checksum checksum2 = cs.getChecksum(token, id, "MD5");

            System.out.println("doc: \"" + doc + "\"");

            System.out.println("doc2: \"" + doc2 + "\"");

            System.out.println("checksum1: " + checksum);

            System.out.println("checksum2: " + checksum2.getValue());

            assertTrue(checksum.equals(checksum2.getValue()));

	    }

	    catch(Exception e)

	    {

	        fail("Unexpected error in testChecksum: " + e.getMessage());

	    }

	}

	/**

	 * insert an invalid document and make sure create fails and does not

	 * leave any artifacts behind

	 */

	public void testFailedCreate()

	{

	    try

	    {

	        String invalidDoc = "<xxx></yyy>";

	        System.out.println("trying to insert doc " + invalidDoc);

	        CrudService cs = CrudService.getInstance();

            AuthToken token = getToken();

            //run create

            try

            {

                Identifier id = createDoc(token, invalidDoc);

                fail("Should have thrown an exception.");

            }

            catch(Exception e)

            {

                //e.printStackTrace();

            }

	    }

	    catch(Exception e)

	    {

	        fail("unexpected exception in testFailedCreate: " + e.getMessage());

	    }

	}

	/**

	 * test to make sure that get and listObjects methods are publicly accessible

	 */

	public void testPublicAccess()

	{

	    try

	    {

	        System.out.println("********************* testPublicAccess *********************");

	        AuthToken token = new AuthToken("public");

	        //AuthToken token = getToken();

	        CrudService cs = CrudService.getInstance();

	        ObjectList ol = cs.listObjects(token, null, null, null);

	        System.out.println("ol: " + ol.sizeObjectInfoList());

	        assertTrue(ol.sizeObjectInfoList() > 0);

	        ObjectInfo oi = ol.getObjectInfo(0);

	        String s = getDoc(token, oi.getIdentifier());

	        assertNotNull(s);

	        try

	        { //try a create with the public auth.  should fail

	            Identifier id = new Identifier();

	            String docid = generateDocumentId();

	            id.setValue(docid);

	            String testDoc = getTestDoc();

	            StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

	            ObjectFormat of1 = ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0");

	            SystemMetadata sm = createSystemMetadata(id, testDoc, of1);

	            assertFalse(sm.getChecksum().getValue().startsWith("MD5"));

	            //create the doc

	            Identifier idC = cs.create(token, id, sbis, sm);

	            fail("Should have thrown an auth exception");

	        }

	        catch(Exception e)

	        {

	        }

	        //create a legit doc

	        token = getToken();

	        Identifier id = new Identifier();

            String docid = generateDocumentId();

            id.setValue(docid);

            String testDoc = getTestDoc();

            StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

            ObjectFormat of1 = ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0");

            SystemMetadata sm = createSystemMetadata(id, testDoc, of1);

            assertFalse(sm.getChecksum().getValue().startsWith("MD5"));

            //create the doc

            Identifier idC = cs.create(token, id, sbis, sm);

            //make it public, then use a public token to try to get it

            InputStream data = null;

            try

            {

                AuthToken publicToken = new AuthToken("public");

                data = cs.get(publicToken, id);

                data.read();

                fail("should have thrown an exception");

            }

            catch(Exception e)

            {

                /*System.out.println("%%%%%%%%%%%%%%%%%%%%%%%getting result");

                Object o = ((InputStreamFromOutputStream)data).getResult();

                System.out.println("result: " + o);

                System.out.println("exception thrown!");

                System.out.println("exception thrown: " + e.getClass().getName());*/

            }

            makeDocPublic(token, id, true);

            token = new AuthToken("public");

	        data = cs.get(token, id);

	    }

	    catch(Exception e)

	    {

	        fail("Error in testPublicAccess: " + e.getMessage());

	    }

	}

	/**

	 * test for an error where the checksum algorithm gets appended onto the checksum.

	 */

	public void testChecksumError()

	{

	    printTestHeader("testChecksumError");

	    try

	    {

	        Date d1 = new Date(new Date().getTime() - 5000);

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        ObjectFormat of1 = ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0");

	        //create docs at different times

	        Identifier id = new Identifier();

	        String docid = "test." + new Date().getTime();

	        id.setValue(docid);

	        //create the system metadata then run the create method

	        String testDoc = getTestDoc();

	        StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

	        SystemMetadata sm = createSystemMetadata(id, testDoc, of1);

	        assertFalse(sm.getChecksum().getValue().startsWith("MD5"));

	        //create the doc

	        Identifier idC = cs.create(token, id, sbis, sm);

	        makeDocPublic(token, id, true);

	        assertTrue(idC.getValue().equals(id.getValue()));

	        SystemMetadata smC = getSystemMetadata(token, idC);

	        assertFalse(smC.getChecksum().getValue().startsWith("MD5"));

	        Date d2 = new Date(new Date().getTime() + 5000);

	        ObjectList ol = cs.listObjects(token, d1, d2, of1);

	        assertTrue(ol.sizeObjectInfoList() > 0);

	        ObjectInfo oi = ol.getObjectInfo(0);

	        //this will fail if the error state exists.

	        assertFalse(oi.getChecksum().getValue().startsWith("MD5"));

	    }

	    catch(Exception e)

	    {

	        fail("Unexpected exception: " + e.getMessage());

	    }

	}

	/**

	 * test CrudService.getLogRecords

	 */

	public void testGetLogRecords()

	{

	    printTestHeader("testGetLogRecords");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        Date fromDate = new Date();

	        Identifier id = createDoc(token, getTestDoc());

	        Date toDate = new Date();

	        Log lrs = cs.getLogRecords(token, fromDate, toDate, null);

	        assertNotNull(lrs);

	        System.out.println("log entry size: " + lrs.sizeLogEntryList());

	        assertTrue(lrs.sizeLogEntryList() == 1);

	        LogEntry lrLogEvent = lrs.getLogEntry(0);

	        assertTrue(lrLogEvent.getEvent().name().equals("CREATE"));

	        assertTrue(lrLogEvent.getIdentifier().getValue().equals(id.getValue()));

	    }

	    catch(Exception e)

	    {

	        e.printStackTrace();

	        fail("testGetLogRecords threw an unexpected exception: " + e.getMessage());

	    }

	}

	/**

	 * make sure that only valid sessions can update/delete

	 */

	public void testAccessControl()

	{

	    printTestHeader("testAccessControl");

        try

        {

            CrudService cs = CrudService.getInstance();

            AuthToken token = getToken();

            //create a doc

            Identifier id = createDoc(token, getTestDoc());

            //get the doc and sysmetadata

            String gotDoc = getDoc(token, id);

            SystemMetadata sm = getSystemMetadata(token, id);

            //break the session id

            String sessionid = "somefakesessionid";

            token = new AuthToken(sessionid);

            //update the doc

            gotDoc = gotDoc.replaceAll("XXX", "YYY");

            Identifier newid = new Identifier();

            newid.setValue(generateDocumentId());

            StringBufferInputStream sbis = new StringBufferInputStream(gotDoc);

            SystemMetadata newsm = createSystemMetadata(newid, gotDoc);

            Identifier updatedid = cs.update(token, newid, sbis, id, newsm);

            fail("exception should have been thrown.");

        }

        catch(Exception e)

        {

        }

        try

        {

            CrudService cs = CrudService.getInstance();

            AuthToken token = new AuthToken("somefakesessionid");

            //create a doc

            Identifier id = createDoc(token, getTestDoc());

            fail("exception should have been thrown.");

        }

        catch(Exception e)

        {

        }

	}

	/**

	 * public ObjectList listObjects(AuthToken token, Date startTime, Date endTime, 

     *     ObjectFormat objectFormat, boolean replicaStatus, int start, int count)

     *       throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken

	 */

	public void testListObjects()

	{

	    printTestHeader("testListObjects");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        ObjectFormat of1 = ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0");

	        ObjectFormat of2 = ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.0.0");

	        //create docs at different times

	        Date d1 = new Date();

	        Identifier id1 = createDoc(token, getTestDoc(EML2_1_0), of1);

            SystemMetadata sm1 = getSystemMetadata(token, id1);

            Date d2 = new Date();

            Identifier id2 = createDoc(token, getTestDoc(EML2_0_0), of2);

            makeDocPublic(token, id2, true);

            SystemMetadata sm2 = getSystemMetadata(token, id2);

            Date d3 = new Date();

            Identifier id3 = createDoc(token, getTestDoc(EML2_1_0), of1);

            makeDocPublic(token, id3, true);

            SystemMetadata sm3 = getSystemMetadata(token, id3);

            Date d4 = new Date();

            Identifier id4 = createDoc(token, getTestDoc(EML2_0_0), of2);

            makeDocPublic(token, id4, true);

            SystemMetadata sm4 = getSystemMetadata(token, id4);

            Date d5 = new Date();

            Identifier id5 = createDoc(token, getTestDoc(EML2_1_0), of1);

            makeDocPublic(token, id5, true);

            SystemMetadata sm5 = getSystemMetadata(token, id5);  

            Date d6 = new Date();

            //now get the objects for specific time ranges and test that it returns

            //the correct objects

            //ObjectList list = cs.listObjects(token, null, null, null);

            //System.out.println("list size: " + list.sizeObjectInfoList());

            //should return sm1 and sm2

            ObjectList list1 = cs.listObjects(token, d1, d3, null);

            assertTrue(list1.sizeObjectInfoList() == 2);

            assertTrue(idInObjectList(id1, list1));

            assertTrue(idInObjectList(id2, list1));

            ObjectInfo info = list1.getObjectInfo(0);

            //should only return sm1

            ObjectList list2 = cs.listObjects(token, d1, d3, of1);

            assertTrue(list2.sizeObjectInfoList() == 1);

            assertTrue(idInObjectList(id1, list2));

            //should return sm1-sm4

            ObjectList list3 = cs.listObjects(token, d1, d5, null);

            assertTrue(list3.sizeObjectInfoList() == 4);

            ObjectInfo oi4 = list3.getObjectInfo(0);

            assertTrue(idInObjectList(id1, list3));

            assertTrue(idInObjectList(id2, list3));

            assertTrue(idInObjectList(id3, list3));

            assertTrue(idInObjectList(id4, list3));

            //should only return sm2 and sm4

            ObjectList list4 = cs.listObjects(token, d1, d5, of2);

            assertTrue(list4.sizeObjectInfoList() == 2);

            assertTrue(idInObjectList(id2, list4));

            assertTrue(idInObjectList(id4, list4));

            //should return all

            ObjectList list5 = cs.listObjects(token, d1, d6, null);

            assertTrue(list5.sizeObjectInfoList() == 5);

            assertTrue(idInObjectList(id1, list5));

            assertTrue(idInObjectList(id2, list5));

            assertTrue(idInObjectList(id3, list5));

            assertTrue(idInObjectList(id4, list5));

            assertTrue(idInObjectList(id5, list5));

            //should return 1, 3, 5

            ObjectList list6 = cs.listObjects(token, d1, d6, of1);

            assertTrue(list6.sizeObjectInfoList() == 3);

            assertTrue(idInObjectList(id1, list6));

            assertTrue(idInObjectList(id3, list6));

            assertTrue(idInObjectList(id5, list6));

            //should return 4 (id1 is not public)

            token = new AuthToken("public");

            ObjectList list7 = cs.listObjects(token, d1, d6, null);

            //System.out.println("list7 size: " + list7.sizeObjectInfoList());

            assertTrue(list7.sizeObjectInfoList() == 5);

            //test paging

            ObjectList list8 = cs.listObjects(token, d1, d6, null, false, 2, 2);

            assertTrue(list8.getCount() == 2);

            assertTrue(list8.getStart() == 2);

            assertTrue(list8.getTotal() == 5);

	    }

	    catch(Exception e)

	    {

	        //e.printStackTrace();

	        fail("Error in listObjects: " + e.getMessage());

	    }

	}

	private boolean idInObjectList(Identifier id, ObjectList list)

	{

	    for(int i=0; i<list.sizeObjectInfoList(); i++)

	    {

	        ObjectInfo oi = list.getObjectInfo(i);

	        if(id.getValue().equals(oi.getIdentifier().getValue()))

	            return true;

	    }

	    return false;

	}

	/**

	 * public Identifier update(AuthToken token, Identifier guid, 

     *       InputStream object, Identifier obsoletedGuid, SystemMetadata sysmeta) 

     *         throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, 

     *           UnsupportedType, InsufficientResources, NotFound, InvalidSystemMetadata, 

     *           NotImplemented

	 */

	public void testUpdate()

	{

	    printTestHeader("testUpdate");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

            //create a doc

            Identifier id = createDoc(token, getTestDoc());

            //get the doc and sysmetadata

            String gotDoc = getDoc(token, id);

            SystemMetadata sm = getSystemMetadata(token, id);

            //update the doc

            gotDoc = gotDoc.replaceAll("XXX", "YYY");

            Identifier newid = new Identifier();

            newid.setValue(generateDocumentId());

            StringBufferInputStream sbis = new StringBufferInputStream(gotDoc);

            SystemMetadata newsm = createSystemMetadata(newid, gotDoc);

            Identifier updatedid = cs.update(token, newid, sbis, id, newsm);

            //get doc - check that it matches update

            String newdoc = getDoc(token, newid);

            assertTrue(gotDoc.equals(newdoc));

            //get sysmeta - check that ids and other fields are updated

            SystemMetadata newnewsm = getSystemMetadata(token, id);

            assertTrue(newnewsm.getObsoletedBy(0).getValue().equals(newid.getValue()));

            //get the new sysmeta and make sure the obsoletes field is set

            SystemMetadata newnewnewsm = getSystemMetadata(token, newid);

            assertTrue(newnewnewsm.getObsolete(0).getValue().equals(id.getValue()));

        }

        catch(Exception e)

        {

            e.printStackTrace();

            fail("Error in testUpdate: " + e.getMessage());

        }

	}

	/**

	 * public SystemMetadata getSystemMetadata(AuthToken token, Identifier guid)

     *       throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

     *       InvalidRequest, NotImplemented

	 */

	public void testGetSystemMetadata()

	{

	    printTestHeader("testGetSystemMetadata");

	    try

	    {

            CrudService cs = CrudService.getInstance();

            AuthToken token = getToken();

            //run create

            Identifier id = createDoc(token, getTestDoc());

            //get the systemMetadata and make sure it is the correct object for this testDoc

            SystemMetadata sm = getSystemMetadata(token, id);

            assertTrue(sm.getIdentifier().getValue().equals(id.getValue()));

            assertTrue(sm.getChecksum().getValue().equals(checksum(getTestDoc())));

            try

            {

                Identifier fakeid = new Identifier();

                fakeid.setValue("somethingfake.234234");

                getSystemMetadata(token, fakeid);

                fail("getSystemMetadata should have thrown an exception.");

            }

            catch(Exception e)

            {

                assertTrue(true);

            }

        }

        catch(Exception e)

        {

            e.printStackTrace();

            fail("Error testing system metadata: " + e.getMessage());

        }

	}

	/**

	 * create(AuthToken token, Identifier guid, InputStream object, SystemMetadata sysmeta) 

   *   throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, 

   *       InsufficientResources, InvalidSystemMetadata, NotImplemented

   *

   * public InputStream get(AuthToken token, Identifier guid)

   *       throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 

   *       NotImplemented

	 */

	public void testCreateAndGet()

	{

	    printTestHeader("testCreateAndGet");

	    try

	    {

	        CrudService cs = CrudService.getInstance();

	        AuthToken token = getToken();

	        //run create

	        Identifier id = createDoc(token, getTestDoc());

	        //make these docs public for debugging purposes.

	        makeDocPublic(token, id, true);

            //compare the docs

            String gotDoc = getDoc(token, id);

            assertTrue(gotDoc.trim().equals(getTestDoc().trim()));

            try

            {

                Identifier fakeid = new Identifier();

                fakeid.setValue("somethingfake.234234");

                getDoc(token, fakeid);

                fail("testCreateAndGet should have thrown an exception.");

            }

            catch(Exception e)

            {

                assertTrue(true);

            }

        }

        catch(Exception e)

        {

            e.printStackTrace();

            fail("Error in testCreate: " + e.getMessage());

        }

	}

	/**

	 * getInstance()

	 */

	public void testSingletonAccessor()

	{

	    printTestHeader("testSingletonAccessor");

	    CrudService cs = CrudService.getInstance();

	    assertNotNull(cs);

	}

	/**

	 * Run an initial test that always passes to check that the test harness is

	 * working.

	 */

	public void initialize() 

	{

	    printTestHeader("initialize");

		assertTrue(1 == 1);

	}

	/**

	 * return the systemMetadata object for id

	 */

	private SystemMetadata getSystemMetadata(AuthToken token, Identifier id)

	  throws BaseException

	{

	    CrudService cs = CrudService.getInstance();

	    return cs.getSystemMetadata(token, id);

	}

	/**

	 * get a doc from metacat using CrudService.get()

	 */

	private String getDoc(AuthToken token, Identifier id)

	  throws Exception

	{

	    CrudService cs = CrudService.getInstance();

        InputStream gotDocStream = cs.get(token, id);

        return IOUtils.toString(gotDocStream);

	}

	public String getTestDoc()

	{

	    return getTestDoc("eml://ecoinformatics.org/eml-2.1.0");

	}

	/**

	 * return a test document.  objectFormat should come from MCTestCase

	 */

	public String getTestDoc(String objectFormat)

	{

	    if(objectFormat == null)

	    {

	        return "<?xml version=\"1.0\"?><test><somecontent>This is some content XXX</somecontent></test>\n";

	    }

	    String accessBlock = getAccessBlock("public", true, true,

                false, false, false);

	    return getTestEmlDoc("Test identifier manager", objectFormat, null,

	                null, "http://fake.example.com/somedata", null,

	                accessBlock, null, null,

	                null, null);

	}

	/**

	 * authenticate and return a token

	 * use the test.mcUser and test.mcPassword username/password combo

	 */

	public AuthToken getToken()

	  throws Exception

	{

        String username = PropertyService.getProperty("test.mcUser");

        String password = PropertyService.getProperty("test.mcPassword");

        return getToken(username, password);

	}

	/**

	 * authenticate and return a token using the given credentials

	 */

	public AuthToken getToken(String username, String password)

	  throws Exception

	{

	    CrudService cs = CrudService.getInstance();

        //login and get a sessionid

        MetacatRestClient restClient = new MetacatRestClient(cs.getContextUrl());

        String response = restClient.login(username, password);

        String sessionid = restClient.getSessionId();

        SessionService sessionService = SessionService.getInstance();

        sessionService.registerSession(new SessionData(sessionid, username, new String[0], password, "CrudServiceLogin"));

        AuthToken token = new AuthToken(sessionid);

        return token;

	}

	/**

	 * create a doc using CrudService.create() and return its id

	 */

	public Identifier createDoc(AuthToken token, String testDoc) throws Exception

	{

	    return createDoc(token, testDoc, ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0"));

	}

	/**

	 * create a doc using CrudService.create() and return its id

	 */

	public Identifier createDoc(AuthToken token, String testDoc, ObjectFormat format) throws Exception

	{

	    Identifier id;

        CrudService cs = CrudService.getInstance();

        id = new Identifier();

        String docid = generateDocumentId();

        id.setValue(docid);

        //create the system metadata then run the create method

        StringBufferInputStream sbis = new StringBufferInputStream(testDoc);

        SystemMetadata sm = createSystemMetadata(id, testDoc, format);

        //create the doc

        cs.create(token, id, sbis, sm);

        createCount++;

        return id;

	}

	/**

	 * create a generic SystemMetadata object for testing

	 */

	private SystemMetadata createSystemMetadata(Identifier id, String testDoc)

	  throws Exception

	{

	    return createSystemMetadata(id, testDoc, 

	            ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0"));

	}

	/**

	 * create system metadata with a specified id, doc and format

	 */

	private SystemMetadata createSystemMetadata(Identifier id, String testDoc, ObjectFormat format)

	  throws Exception

	{

	    SystemMetadata sm = new SystemMetadata();

        //set the id

        sm.setIdentifier(id);

        sm.setObjectFormat(format);

        //create the checksum

        String checksumS = checksum(testDoc);

        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");

        Checksum checksum = new Checksum();

        checksum.setValue(checksumS);

        checksum.setAlgorithm(ca);

        sm.setChecksum(checksum);

        //set the size

        sm.setSize(testDoc.getBytes(MetaCatServlet.DEFAULT_ENCODING).length);

        //submitter

        Subject p = new Subject();

        p.setValue("joe");

        sm.setSubmitter(p);

        sm.setRightsHolder(p);

        sm.setDateUploaded(new Date());

        sm.setDateSysMetadataModified(new Date());

        NodeReference nr = new NodeReference();

        nr.setValue("metacat");

        sm.setOriginMemberNode(nr);

        sm.setAuthoritativeMemberNode(nr);

        return sm;

	}

	/**

	 *  make a document public in metacat by inserting an access document

	 * @param id

	 */

	private void makeDocPublic(AuthToken token, Identifier id, boolean systemMetadataToo)

	  throws Exception

	{

	    CrudService cs = CrudService.getInstance();

	    cs.setAccess(token, id, "public", "read", "allow", "allowFirst", systemMetadataToo);

	}

	/**

	 * print a header to start each test

	 */

	private void printTestHeader(String testName)

	{

	    System.out.println();

	    System.out.println("*************** " + testName + " ***************");

	}

	/**

	 * produce an md5 checksum for item

	 */

	private String checksum(String item)

	  throws Exception

	{

        StringBufferInputStream fis =  new StringBufferInputStream(item);

        byte[] buffer = new byte[1024];

        MessageDigest complete = MessageDigest.getInstance("MD5");

        int numRead;

        do 

        {

          numRead = fis.read(buffer);

          if (numRead > 0) 

          {

            complete.update(buffer, 0, numRead);

          }

        } while (numRead != -1);

        return getHex(complete.digest());

	}

	/**

	 * convert a byte array to a hex string

	 */

	private static String getHex( byte [] raw ) 

	{

	    final String HEXES = "0123456789ABCDEF";

        if ( raw == null ) {

          return null;

        }

        final StringBuilder hex = new StringBuilder( 2 * raw.length );

        for ( final byte b : raw ) {

          hex.append(HEXES.charAt((b & 0xF0) >> 4))

             .append(HEXES.charAt((b & 0x0F)));

        }

        return hex.toString();

  }

}

/**

 *  '$RCSfile$'

 *  Copyright: 2000 Regents of the University of California and the

 *              National Center for Ecological Analysis and Synthesis

 *

 *   '$Author: $'

 *     '$Date: 2009-06-13 15:28:13 +0300  $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.dataone;

import java.io.ByteArrayInputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.security.NoSuchAlgorithmException;

import java.sql.SQLException;

import java.text.DateFormat;

import java.util.Calendar;

import java.util.Date;

import java.util.Enumeration;

import java.util.Hashtable;

import java.util.List;

import java.util.TimeZone;

import java.util.Timer;

import java.util.TimerTask;

import java.util.Vector;

import javax.servlet.http.HttpServletRequest;

import org.apache.commons.io.IOUtils;

import org.apache.log4j.Logger;

import org.dataone.service.exceptions.IdentifierNotUnique;

import org.dataone.service.exceptions.InsufficientResources;

import org.dataone.service.exceptions.InvalidRequest;

import org.dataone.service.exceptions.InvalidSystemMetadata;

import org.dataone.service.exceptions.InvalidToken;

import org.dataone.service.exceptions.NotAuthorized;

import org.dataone.service.exceptions.NotFound;

import org.dataone.service.exceptions.NotImplemented;

import org.dataone.service.exceptions.ServiceFailure;

import org.dataone.service.exceptions.UnsupportedType;

import org.dataone.service.mn.MemberNodeCrud;

import org.dataone.service.types.AuthToken;

import org.dataone.service.types.Checksum;

import org.dataone.service.types.ChecksumAlgorithm;

import org.dataone.service.types.DescribeResponse;

import org.dataone.service.types.Event;

import org.dataone.service.types.Identifier;

import org.dataone.service.types.Log;

import org.dataone.service.types.LogEntry;

import org.dataone.service.types.NodeReference;

import org.dataone.service.types.ObjectFormat;

import org.dataone.service.types.ObjectList;

import org.dataone.service.types.Subject;

import org.dataone.service.types.SystemMetadata;

import org.dataone.service.types.util.ServiceTypeUtil;

import edu.ucsb.nceas.metacat.AccessionNumberException;

import edu.ucsb.nceas.metacat.DocumentImpl;

import edu.ucsb.nceas.metacat.EventLog;

import edu.ucsb.nceas.metacat.IdentifierManager;

import edu.ucsb.nceas.metacat.McdbDocNotFoundException;

import edu.ucsb.nceas.metacat.McdbException;

import edu.ucsb.nceas.metacat.MetacatHandler;

import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;

import edu.ucsb.nceas.metacat.client.rest.MetacatRestClient;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;

import edu.ucsb.nceas.metacat.service.SessionService;

import edu.ucsb.nceas.metacat.util.DocumentUtil;

import edu.ucsb.nceas.metacat.util.SessionData;

import edu.ucsb.nceas.utilities.ParseLSIDException;

import edu.ucsb.nceas.utilities.PropertyNotFoundException;

/**

 * 

 * Implements DataONE MemberNode CRUD API for Metacat. 

 * 

 * @author Matthew Jones

 */

public class CrudService implements MemberNodeCrud

{

    private static CrudService crudService = null;

    private MetacatHandler handler;

    private Hashtable<String, String[]> params;

    private Logger logMetacat = null;

    private Logger logCrud = null;

    private String metacatUrl;

    /**

     * singleton accessor

     */

    public static CrudService getInstance() 

    {

      if(crudService == null)

      {

        crudService = new CrudService();

      }

      return crudService;

    }

    /**

     * Constructor, private for singleton access

     */

    private CrudService() {

        logMetacat = Logger.getLogger(CrudService.class);

        logCrud = Logger.getLogger("DataOneLogger");

        try

        {

            String server = PropertyService.getProperty("server.name");

            String port = PropertyService.getProperty("server.httpPort");

            String context = PropertyService.getProperty("application.context");

            metacatUrl = "http://" + server + ":" + port + "/" + context + "/d1";

            logMetacat.debug("Initializing CrudService with url " + metacatUrl);

        }

        catch(Exception e)

        {

            logMetacat.error("Could not find servlet url in CrudService: " + e.getMessage());

            e.printStackTrace();

            throw new RuntimeException("Error getting servlet url in CrudService: " + e.getMessage());

        }

        params = new Hashtable<String, String[]>();

        handler = new MetacatHandler(new Timer());

    }

    /**

     * return the context url CrudService is using.

     */

    public String getContextUrl()

    {

        return metacatUrl;

    }

    /**

     * Set the context url that this service uses.  It is normally not necessary

     * to call this method unless you are trying to connect to a server other

     * than the one in which this service is installed.  Otherwise, this value is

     * taken from the metacat.properties file (server.name, server.port, application.context).

     */

    public void setContextUrl(String url)

    {

        metacatUrl = url;

    }

    /**

     * set the params for this service from an HttpServletRequest param list

     */

    public void setParamsFromRequest(HttpServletRequest request)

    {

        @SuppressWarnings("unchecked")

        Enumeration<String> paramlist = request.getParameterNames();

        while (paramlist.hasMoreElements()) {

            String name = (String) paramlist.nextElement();

            String[] value = (String[])request.getParameterValues(name);

            params.put(name, value);

        }

    }

    /**

     * Authenticate against metacat and get a token.

     * @param username

     * @param password

     * @return

     * @throws ServiceFailure

     */

    public AuthToken authenticate(String username, String password)

      throws ServiceFailure

    {

        /* TODO:

         * This method is not in the original D1 crud spec.  It is highly

         * metacat centric.  Higher level decisions need to be made on authentication

         * interfaces for D1 nodes.

         */

        try

        {

            MetacatRestClient restClient = new MetacatRestClient(getContextUrl());   

            String response = restClient.login(username, password);

            String sessionid = restClient.getSessionId();

            SessionService sessionService = SessionService.getInstance();

            sessionService.registerSession(new SessionData(sessionid, username, new String[0], password, "CrudServiceLogin"));

            AuthToken token = new AuthToken(sessionid);

            EventLog.getInstance().log(metacatUrl,

                    username, null, "authenticate");

            logCrud.info("authenticate");

            return token;

        }

        catch(Exception e)

        {

            throw new ServiceFailure("1620", "Error authenticating with metacat: " + e.getMessage());

        }

    }

    /**

     * set the parameter values needed for this request

     */

    public void setParameter(String name, String[] value)

    {

        params.put(name, value);

    }

    /**

     * create an object via the crud interface

     */

    public Identifier create(AuthToken token, Identifier guid, 

            InputStream object, SystemMetadata sysmeta) throws InvalidToken, 

            ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, 

            InsufficientResources, InvalidSystemMetadata, NotImplemented {

        logMetacat.debug("Starting CrudService.create()...");

        // authenticate & get user info

        SessionData sessionData = getSessionData(token);

        String username = "public";

        String[] groups = null;

        if(sessionData != null)

        {

            username = sessionData.getUserName();

            groups = sessionData.getGroupNames();

        }

        String localId = null;

        if (username == null || username.equals("public"))

        {

            //TODO: many of the thrown exceptions do not use the correct error codes

            //check these against the docs and correct them

            throw new NotAuthorized("1100", "User " + username + " is not authorized to create content." +

                    "  If you are not logged in, please do so and retry the request.");

        }

        // verify that guid == SystemMetadata.getIdentifier()

        logMetacat.debug("Comparing guid|sysmeta_guid: " + guid.getValue() + "|" + sysmeta.getIdentifier().getValue());

        if (!guid.getValue().equals(sysmeta.getIdentifier().getValue())) {

            throw new InvalidSystemMetadata("1180", 

                "GUID in method call (" + guid.getValue() + ") does not match GUID in system metadata (" +

                sysmeta.getIdentifier().getValue() + ").");

        }

        logMetacat.debug("Checking if identifier exists...");

        // Check that the identifier does not already exist

        IdentifierManager im = IdentifierManager.getInstance();

        if (im.identifierExists(guid.getValue())) {

            throw new IdentifierNotUnique("1120", 

                "GUID is already in use by an existing object.");

        }

        // Check if we are handling metadata or data

        boolean isScienceMetadata = isScienceMetadata(sysmeta);

        if (isScienceMetadata) {

            // CASE METADATA:

            try {

                //logCrud.debug("CrudService: inserting document with guid " + guid.getValue());

                this.insertDocument(object, guid, sessionData);

                localId = im.getLocalId(guid.getValue());

            } catch (IOException e) {

                String msg = "Could not create string from XML stream: " +

                    " " + e.getMessage();

                logMetacat.debug(msg);

                throw new ServiceFailure("1190", msg);

            } catch(Exception e) {

                String msg = "Unexpected error in CrudService.create: " + e.getMessage();

                logMetacat.debug(msg);

                throw new ServiceFailure("1190", msg);

            }

        } else {

            // DEFAULT CASE: DATA (needs to be checked and completed)

            localId = insertDataObject(object, guid, sessionData);

        }

        // For Metadata and Data, insert the system metadata into the object store too

        insertSystemMetadata(sysmeta, sessionData);

        //get the document info.  add any access params for the sysmeta too

        //logCrud.debug("looking for access records to add for system " +

        //    "metadata who's parent doc's  local id is " + localId);

        try

        {

            Hashtable<String, Object> h = im.getDocumentInfo(localId.substring(0, localId.lastIndexOf(".")));

            Vector v = (Vector)h.get("access");

            for(int i=0; i<v.size(); i++)

            {

                @SuppressWarnings("unchecked")

                Hashtable<String, String> ah = (Hashtable<String, String>)v.elementAt(i);

                String principal = (String)ah.get("principal_name");

                String permission = (String)ah.get("permission");

                String permissionType = (String)ah.get("permission_type");

                String permissionOrder = (String)ah.get("permission_order");

                int perm = new Integer(permission).intValue();

                //logCrud.debug("found access record for principal " + principal);

                //logCrud.debug("permission: " + perm + " perm_type: " + permissionType + 

                //    " perm_order: " + permissionOrder);

                this.setAccess(token, guid, principal, perm, permissionType, permissionOrder, true);

            }

        }

        catch(Exception e)

        {

            logMetacat.error("Error setting permissions on System Metadata object: " +

            		e.getMessage());

            //TODO: decide if this error should cancel the entire create or

            //if it should continue with just a logged error.

        }

        logMetacat.debug("Returning from CrudService.create()");

        EventLog.getInstance().log(metacatUrl,

                username, localId, "create");

        logCrud.info("create D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId);

        return guid;

    }

    /**

     * update an existing object with a new object.  Change the system metadata

     * to reflect the changes and update it as well.

     */

    public Identifier update(AuthToken token, Identifier guid, 

            InputStream object, Identifier obsoletedGuid, SystemMetadata sysmeta) 

            throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, 

            UnsupportedType, InsufficientResources, NotFound, InvalidSystemMetadata, 

            NotImplemented {

    	try

    	{

    		SessionData sessionData = getSessionData(token);

            //find the old systemmetadata (sm.old) document id (the one linked to obsoletedGuid)

            SystemMetadata sm = getSystemMetadata(token, obsoletedGuid);

            //change sm.old's obsoletedBy field 

            List<Identifier> l = sm.getObsoletedByList();

            l.add(guid);

            sm.setObsoletedByList(l);

            //update sm.old

            updateSystemMetadata(sm, sessionData);

            //change the obsoletes field of the new systemMetadata (sm.new) to point to the id of the old one

            sysmeta.addObsolete(obsoletedGuid);

            //insert sm.new

            insertSystemMetadata(sysmeta, sessionData);

            String localId;

            boolean isScienceMetadata = isScienceMetadata(sysmeta);

            if(isScienceMetadata)

            {

                //update the doc

                localId = updateDocument(object, obsoletedGuid, guid, sessionData, false);

            }

            else

            {

                //update a data file, not xml

                localId = insertDataObject(object, guid, sessionData);

            }

            IdentifierManager im = IdentifierManager.getInstance();

            String username = "public";

            if(sessionData != null)

            {

                username = sessionData.getUserName();

            }

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(guid.getValue()), "update");

            logCrud.info("update D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId);

            return guid;

        }

    	catch(IOException e) {

            throw new ServiceFailure("1310", "Error updating document in CrudService: " + e.getMessage());

        }

    	catch( McdbDocNotFoundException e) {

    		throw new ServiceFailure("1310", "Error updating document in CrudService: " + e.getMessage());

    	}

    	catch( InvalidRequest e) {

    		throw new ServiceFailure("1310", "Error updating document in CrudService: " + e.getMessage());

    	}

    }

    /**

     * set access permissions on both the science metadata and system metadata

     */

    public void setAccess(AuthToken token, Identifier id, String principal, String permission,

            String permissionType, String permissionOrder)

      throws ServiceFailure

    {

        setAccess(token, id, principal, permission, permissionType, permissionOrder, true);

    }

    /**

     * set access control on the doc

     * @param token

     * @param id

     * @param principal

     * @param permission

     */

    public void setAccess(AuthToken token, Identifier id, String principal, int permission,

      String permissionType, String permissionOrder, boolean setSystemMetadata)

      throws ServiceFailure

    {

        String perm = "";

        if(permission >= 4)

        {

            perm = "read";

        }

        if(permission >= 6)

        {

            perm = "write";

        }

        //logCrud.debug("perm in setAccess: " + perm);

        //logCrud.debug("permission in setAccess: " + permission);

        setAccess(token, id, principal, perm, permissionType, permissionOrder,

                setSystemMetadata);

    }

    /**

     * set the permission on the document

     * @param token

     * @param principal

     * @param permission

     * @param permissionType

     * @param permissionOrder

     * @return

     */

    public void setAccess(AuthToken token, Identifier id, String principal, String permission,

            String permissionType, String permissionOrder, boolean setSystemMetadata)

      throws ServiceFailure

    {

        /* TODO:

         * This is also not part of the D1 Crud spec.  This method is needed for

         * systems such as metacat where access to objects is controlled by

         * and ACL.  Higher level decisions need to be made about how this

         * should work within D1.

         */

        try

        {

            final SessionData sessionData = getSessionData(token);

            if(sessionData == null)

            {

                throw new ServiceFailure("1000", "User must be logged in to set access.");

            }

            IdentifierManager im = IdentifierManager.getInstance();

            String docid = im.getLocalId(id.getValue());

            String permNum = "0";

            if(permission.equals("read"))

            {

                permNum = "4";

            }

            else if(permission.equals("write"))

            {

                permNum = "6";

            }

            logCrud.debug("user " + sessionData.getUserName() + 

                    " is setting access level " + permNum + " for permission " + 

                    permissionType + " on doc with localid " + docid);

            handler.setAccess(metacatUrl, sessionData.getUserName(), docid, 

                    principal, permNum, permissionType, permissionOrder);

            String username = "public";

            if(sessionData != null)

            {

                username = sessionData.getUserName();

            }

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(id.getValue()), "setAccess");

            logCrud.info("setAccess");

        }

        catch(Exception e)

        {

            e.printStackTrace();

            throw new ServiceFailure("1000", "Could not set access on the document with id " + id.getValue());

        }

    }

    /**

     *  Retrieve the list of objects present on the MN that match the calling 

     *  parameters. This method is required to support the process of Member 

     *  Node synchronization. At a minimum, this method should be able to 

     *  return a list of objects that match:

     *  startTime <= SystemMetadata.dateSysMetadataModified

     *  but is expected to also support date range (by also specifying endTime), 

     *  and should also support slicing of the matching set of records by 

     *  indicating the starting index of the response (where 0 is the index 

     *  of the first item) and the count of elements to be returned.

     *  

     *  If startTime or endTime is null, the query is not restricted by that parameter.

     *  

     * @see http://mule1.dataone.org/ArchitectureDocs/mn_api_replication.html#MN_replication.listObjects

     * @param token

     * @param startTime

     * @param endTime

     * @param objectFormat

     * @param replicaStatus

     * @param start

     * @param count

     * @return ObjectList

     * @throws NotAuthorized

     * @throws InvalidRequest

     * @throws NotImplemented

     * @throws ServiceFailure

     * @throws InvalidToken

     */

    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime, 

        ObjectFormat objectFormat, boolean replicaStatus, int start, int count)

      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken

    {

      //ObjectList ol = new ObjectList();

      //final SessionData sessionData = getSessionData(token);

      //int totalAfterQuery = 0;

    	ObjectList objectList = null;

        try {

    	    objectList = IdentifierManager.getInstance().querySystemMetadata(startTime, endTime,

    	              objectFormat.getFmtid(), replicaStatus, start, count);

        } catch (Exception e) {