Metacat

import java.io.ByteArrayOutputStream;

import javax.xml.parsers.ParserConfigurationException;

import javax.xml.xpath.XPathExpressionException;

import org.dataone.eml.DataoneEMLParser;

import org.dataone.eml.EMLDocument;

import org.dataone.eml.EMLDocument.DistributionMetadata;

import org.dataone.service.exceptions.BaseException;

import org.jibx.runtime.BindingDirectory;

import org.jibx.runtime.IBindingFactory;

import org.jibx.runtime.IMarshallingContext;

import org.jibx.runtime.IUnmarshallingContext;

import org.jibx.runtime.JiBXException;

import org.xml.sax.SAXException;

import edu.ucsb.nceas.metacat.AccessionNumber;

    /**

     * Generate SystemMetadata for any object in the object store that does

     * not already have it.  SystemMetadata documents themselves, are, of course,

     * exempt.  This is a utility method for migration of existing object 

     * stores to DataONE where SystemMetadata is required for all objects.  See 

     * https://trac.dataone.org/ticket/591

     * 

     * @param token an authtoken with appropriate permissions to read all 

     * documents in the object store.  To work correctly, this should probably

     * be an adminstrative credential.

     * @throws SQLException 

     * @throws AccessionNumberException 

     * @throws NoSuchAlgorithmException 

     * @throws InvalidRequest 

     * @throws NotImplemented 

     * @throws NotFound 

     * @throws NotAuthorized 

     * @throws InvalidToken 

     * @throws PropertyNotFoundException 

     * @throws McdbDocNotFoundException 

     * @throws ServiceFailure 

     */

    public void generateMissingSystemMetadata(AuthToken token) 

    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized, 

    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 

    {

        IdentifierManager im = IdentifierManager.getInstance();

        //get the list of ids with no SM

        List<String> idList = im.getLocalIdsWithNoSystemMetadata();

        for (String localId : idList) { 

            //for each id, add a system metadata doc

            generateMissingSystemMetadata(token, localId);

        }

        logCrud.info("generateMissingSystemMetadata(token)");

    }

    /**

     * Generate SystemMetadata for a particular object with identifier localId.

     * This is a utility method for migration of existing objects 

     * to DataONE where SystemMetadata is required for all objects.

     * 

     * @param token an authtoken with appropriate permissions to read all 

     *        documents in the object store.  To work correctly, this should

     *        be an adminstrative credential.

     * @param localId the identifier of the object to be processed

     * @throws ServiceFailure 

     * @throws SQLException 

     * @throws AccessionNumberException 

     * @throws NoSuchAlgorithmException 

     * @throws InvalidRequest 

     * @throws NotImplemented 

     * @throws NotFound 

     * @throws NotAuthorized 

     * @throws InvalidToken 

     * @throws PropertyNotFoundException 

     * @throws McdbDocNotFoundException 

     */

    public void generateMissingSystemMetadata(AuthToken token, String localId) 

    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized,

    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 

    {

    	logCrud.debug("CrudService.generateMissingSystemMetadata() called.");

        logCrud.debug("Creating SystemMetadata for localId " + localId);

        SystemMetadata sm = null;

        //generate required system metadata fields from the document

        try {

        	sm = createSystemMetadata(localId, token);

        } catch (IOException e1) {

        	e1.printStackTrace();

        	ServiceFailure sf = new ServiceFailure("00","IOException in generateMissingSystemMetadata: " +

        			e1.getMessage());

        	sf.setStackTrace(e1.getStackTrace());

        	throw sf;

        }

        //insert the systemmetadata object

        SessionData sessionData = getSessionData(token);

        String smlocalid = insertSystemMetadata(sm, sessionData);

        logCrud.debug("setting access on SM doc with localid " + smlocalid);

        try {

        	handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid, "public", "4", "allow", "allowFirst");

        } catch (Exception e) {

        	logCrud.debug("Unspecified exception thrown by MetacatHandler.setAccess(): " + e.getMessage());

        	logMetacat.error("Could not generate missing systemMetadata: " + e.getMessage());

        	logMetacat.error(e.getStackTrace());

        	ServiceFailure sf = new ServiceFailure("0","Unspecified exception thrown by MetacatHandler.setAccess(): " + e.getMessage());

        	sf.setStackTrace(e.getStackTrace());

        	throw sf;

        }

        String username = "public";

        if (sessionData != null) {

        	username = sessionData.getUserName();

        }

        EventLog.getInstance().log(metacatUrl, username, localId, "generateMissingSystemMetadata");

//        catch (Exception e) { // TODO: Please don't catch Exception -- it masks bad things

//            e.printStackTrace();

//            logCrud.debug("Exception generating missing system metadata: " + e.getMessage());

//            logMetacat.error("Could not generate missing system metadata: " + e.getMessage());

//        }

        logCrud.info("generateMissingSystemMetadata(token, localId)");

    }

        String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);

            logMetacat.error("Error setting permissions on System Metadata object " + 

                    " with id " + sysMetaLocalId + ": " + e.getMessage());

        logCrud.info("create D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId + 

                ":D1SYSMETADATA:"+ sysMetaLocalId + ":");

            String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);

            logCrud.info("update D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId + 

                    ":D1SYSMETADATA:"+ sysMetaLocalId + ":");

            InputStream docStream = get(token, guid);

public SystemMetadata getSystemMetadata(AuthToken token, Identifier guid)

        logMetacat.debug("CrudService.getSystemMetadata - for guid: " + guid.getValue());

        // Retrieve the session information from the AuthToken

        // If the session is expired, then the user is 'public'

        final SessionData sessionData = getSessionData(token);

        try {

            IdentifierManager im = IdentifierManager.getInstance();

            final String localId = im.getSystemMetadataLocalId(guid.getValue());

            InputStream objectStream;

            try {

                String username = "public";

                String[] groupnames = null;

                if(sessionData != null)

                {

                    username = sessionData.getUserName();

                    groupnames = sessionData.getGroupNames();

                }

                objectStream = readFromMetacat(localId, username, groupnames);

            } catch (PropertyNotFoundException e) {

                e.printStackTrace();

                throw new ServiceFailure("1090", "Property not found while reading system metadata from metacat: " + e.getMessage());

            } catch (ClassNotFoundException e) {

                e.printStackTrace();

                throw new ServiceFailure("1090", "Class not found while reading system metadata from metacat: " + e.getMessage());

            } catch (IOException e) {

                e.printStackTrace();

                throw new ServiceFailure("1090", "IOException while reading system metadata from metacat: " + e.getMessage());

            } catch (SQLException e) {

                e.printStackTrace();

                throw new ServiceFailure("1090", "SQLException while reading system metadata from metacat: " + e.getMessage());

            } catch (McdbException e) {

                e.printStackTrace();

                throw new ServiceFailure("1090", "Metacat DB Exception while reading system metadata from metacat: " + e.getMessage());

            } catch (ParseLSIDException e) {

                e.printStackTrace();

                throw new NotFound("1060", "Error parsing LSID while reading system metadata from metacat: " + e.getMessage());

            } catch (InsufficientKarmaException e) {

                e.printStackTrace();

                throw new NotAuthorized("1040", "User not authorized for get() on system metadata: " + e.getMessage());

            }

            // Deserialize the xml to create a SystemMetadata object

            SystemMetadata sysmeta = deserializeSystemMetadata(objectStream);

            String username = "public";

            if(sessionData != null)

            {

                username = sessionData.getUserName();

            }

            EventLog.getInstance().log(metacatUrl,

                    username, im.getLocalId(guid.getValue()), "read");

            logCrud.info("getsystemmetadata D1GUID:" + guid.getValue()  + 

                    ":D1SYSMETADATA:"+ localId + ":");

            return sysmeta;

        } catch (McdbDocNotFoundException e) {

            //e.printStackTrace();

            throw new NotFound("1040", e.getMessage());

        }                

    private String insertSystemMetadata(SystemMetadata sysmeta, SessionData sessionData) 

        String xml = new String(serializeSystemMetadata(sysmeta).toByteArray());

        logCrud.debug("sysmeta: " + xml);

        String localId = insertDocument(xml, sysMetaGuid, sessionData, true);

        logCrud.debug("sysmeta inserted with localId " + localId);

        return localId;

            String smId = IdentifierManager.getInstance().getSystemMetadataLocalId(sm.getIdentifier().getValue());

            String xml = new String(serializeSystemMetadata(sm).toByteArray());

            String localId = updateDocument(xml, sm.getIdentifier(), null, sessionData, true);

            IdentifierManager.getInstance().updateSystemMetadataMapping(sm.getIdentifier().getValue(), localId);

            IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(

              sm.getDateUploaded().getTime(), 

              sm.getRightsHolder().getValue(),

              sm.getChecksum().getValue(), 

              /*sm.getChecksum().getAlgorithm().toString()*/sm.getChecksum().getAlgorithm().name(), 

              sm.getOriginMemberNode().getValue(), 

              sm.getAuthoritativeMemberNode().getValue(), 

              sm.getDateSysMetadataModified().getTime(), 

              sm.getSubmitter().getValue(), 

              sm.getIdentifier().getValue(), 

              sm.getObjectFormat().toString(), 

              sm.getSize());

    public static ByteArrayOutputStream serializeSystemMetadata(SystemMetadata sysmeta) 

        throws ServiceFailure {

        IBindingFactory bfact;

        ByteArrayOutputStream sysmetaOut = null;

        try {

            bfact = BindingDirectory.getFactory(SystemMetadata.class);

            IMarshallingContext mctx = bfact.createMarshallingContext();

            sysmetaOut = new ByteArrayOutputStream();

            mctx.marshalDocument(sysmeta, "UTF-8", null, sysmetaOut);

        } catch (JiBXException e) {

            e.printStackTrace();

            throw new ServiceFailure("1190", "Failed to serialize and insert SystemMetadata: " + e.getMessage());

        }

        return sysmetaOut;

    }

    public static SystemMetadata deserializeSystemMetadata(InputStream xml) 

        throws ServiceFailure {

        try {

            IBindingFactory bfact = BindingDirectory.getFactory(SystemMetadata.class);

            IUnmarshallingContext uctx = bfact.createUnmarshallingContext();

            SystemMetadata sysmeta = (SystemMetadata) uctx.unmarshalDocument(xml, null);

            return sysmeta;

        } catch (JiBXException e) {

            e.printStackTrace();

            throw new ServiceFailure("1190", "Failed to deserialize and insert SystemMetadata: " + e.getMessage());

        }    

    }

    /**

     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into

     * a proper date object

     * @param date

     * @return

     */

    private Date parseMetacatDate(String date)

    {

        String year = date.substring(0, 4);

        String month = date.substring(5, 7);

        String day = date.substring(8, 10);

        Calendar c = Calendar.getInstance(TimeZone.getDefault());

        c.set(new Integer(year).intValue(), 

              new Integer(month).intValue(), 

              new Integer(day).intValue());

        logCrud.debug("time in parseMetacatDate: " + c.getTime());

        return c.getTime();

    }

    /**

     * create system metadata with a specified id, doc and format

     * @throws McdbDocNotFoundException 

     * @throws SQLException

     * @throws AccessionNumberException 

     * @throws NumberFormatException 

     * @throws IOException 

     * @throws NoSuchAlgorithmException 

     * @throws PropertyNotFoundException 

     * @throws NotImplemented 

     * @throws NotFound 

     * @throws NotAuthorized 

     * @throws InvalidToken 

     * @throws InvalidRequest 

     * @throws NoSuchAlgorithmException 

     */

    private SystemMetadata createSystemMetadata(String localId, AuthToken token) 

    throws McdbDocNotFoundException, PropertyNotFoundException, AccessionNumberException, SQLException,

    ServiceFailure, InvalidToken, NotAuthorized, NotFound, NotImplemented, InvalidRequest,

    IOException, NoSuchAlgorithmException

    {     

    	logCrud.debug("CrudService.createSystemMetadata() called.");

    	IdentifierManager im = IdentifierManager.getInstance();

    	Hashtable<String, Object> docInfo = im.getDocumentInfo(localId);

    	//get the document text

    	int rev = im.getLatestRevForLocalId(localId);

    	Identifier identifier = new Identifier();

    	try {

    		identifier.setValue(im.getGUID(localId, rev));

    	} catch (McdbDocNotFoundException mcdbe) { 

    		//we're creating a new SM doc for a doc that is not in the identifier table                                       

    		//so we need to add it to

    		logCrud.debug("No guid in the identifier table.  adding it for " + localId);

    		im.createMapping(localId, localId);

    		logCrud.debug("Mapping created for " + localId);

    		AccessionNumber accNum = new AccessionNumber(localId, "NONE");

    		identifier.setValue(im.getGUID(accNum.getDocid(), rev));

    	}

    	logCrud.debug("Creating system metadata for guid " + identifier.getValue());

    	InputStream is = this.get(token, identifier);

    	SystemMetadata sm = new SystemMetadata();

    	//set the id

    	sm.setIdentifier(identifier);

    	//set the default object format

    	String doctype = (String) docInfo.get("doctype");

    	ObjectFormat format = ObjectFormat.convert(doctype);

    	if (format == null) {

    		if (doctype.trim().equals("BIN")) {

    			format = ObjectFormat.OCTET_STREAM;

    		} else {

    			format = ObjectFormat.convert("text/plain");

    		}

    	}

    	sm.setObjectFormat(format);

    	logCrud.debug("The ObjectFormat for " + localId + " is " + format.toString());

    	// further parse EML documents to get data object format,

    	// describes and describedBy information

    	if ( format == ObjectFormat.EML_2_0_0 ||

    			format == ObjectFormat.EML_2_0_1 ||

    			format == ObjectFormat.EML_2_1_0 ) {

    		try {

    			DataoneEMLParser emlParser = DataoneEMLParser.getInstance();

    			EMLDocument emlDocument = emlParser.parseDocument(is);

    			// iterate through the data objects in the EML doc and add sysmeta

    			logCrud.debug("The number of data entities is: " +

    					emlDocument.distributionMetadata.size());

    			for( int j = 0; j < emlDocument.distributionMetadata.size(); j++ ) {

    				DistributionMetadata distMetadata = 

    					emlDocument.distributionMetadata.elementAt(j);

    				String dataDocUrl = distMetadata.url;

    				String dataDocMimeType = distMetadata.mimeType;

    				String dataDocLocalId = "";

    				logCrud.debug("\tData local ID: " + dataDocLocalId);

    				logCrud.debug("\tData URL: " + dataDocUrl);

    				logCrud.debug("\tData mime: " + dataDocMimeType);

    				//we only handle ecogrid urls right now

    				if ( dataDocUrl.trim().startsWith("ecogrid://knb/") ) {

    					dataDocLocalId = 

    						dataDocUrl.substring(dataDocUrl.indexOf("ecogrid://knb/") + 

    								"ecogrid://knb/".length(), dataDocUrl.length());

    					//set the id

    					Identifier dataDocId = new Identifier();

    					dataDocId.setValue(dataDocLocalId);

    					// add describes into EML system metadata

    					sm.addDescribe(dataDocId);

    					SystemMetadata dataSysMeta = new SystemMetadata();

    					// check if data system metadata exists

    					try {

    						logCrud.debug("Checking for existing system metadata for " + dataDocId.getValue());

    						dataSysMeta = this.getSystemMetadata(token, dataDocId);

    						// add describedBy sysmeta

    						logCrud.debug("Setting describedBy for " + dataDocId.getValue() +

    								" to " + identifier.getValue());

    						dataSysMeta.addDescribedBy(identifier);

    						dataSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));

    						this.updateSystemMetadata(dataSysMeta, getSessionData(token));

    					} catch ( NotFound nf ) {

    						// System metadata for data doesn't exist

    						logCrud.debug("There was not an existing system metadata " + "document for " + dataDocId.getValue());

    						logCrud.debug("Creating a system metadata " + "document for " + dataDocId.getValue());

    						dataSysMeta = this.createSystemMetadata(dataDocLocalId, token);

    						logCrud.debug("Setting describedBy for " + dataDocId.getValue() + " to " + identifier.getValue());

    						dataSysMeta.addDescribedBy(identifier);

    						logCrud.debug("Setting mimeType for " + dataDocId.getValue() + " to " + dataDocMimeType);

    						dataSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));

    						logCrud.debug("Updating system metadata for " + dataDocId.getValue() + " to " + dataDocMimeType);

    						this.updateSystemMetadata(dataSysMeta, getSessionData(token));

    					}

    				} // end if()

    			} // end for()

    		} catch ( ParserConfigurationException pce ) {

    			logCrud.debug("There was a problem parsing the EML document. " +

    					"The error message was: " + pce.getMessage());

    		} catch ( SAXException saxe ) {

    			logCrud.debug("There was a problem traversing the EML document. " +

    					"The error message was: " + saxe.getMessage());

    		} catch ( XPathExpressionException xpee ) {

    			logCrud.debug("There was a problem searching the EML document. " +

    					"The error message was: " + xpee.getMessage());

    		} // end try

    	} // end if()

    	//create the checksum

    	is = this.get(token, identifier);

    	String checksumS = checksum(is);

    	ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");

    	Checksum checksum = new Checksum();

    	checksum.setValue(checksumS);

    	checksum.setAlgorithm(ca);

    	sm.setChecksum(checksum);

    	//set the size

    	is = this.get(token, identifier);

    	sm.setSize(sizeOfStream(is));

    	//submitter

    	Subject p = new Subject();

    	p.setValue((String) docInfo.get("user_owner"));

    	sm.setSubmitter(p);

    	sm.setRightsHolder(p);

    	try {

    		Date dateCreated = parseMetacatDate((String) docInfo.get("date_created"));

    		sm.setDateUploaded(dateCreated);

    		Date dateUpdated = parseMetacatDate((String) docInfo.get("date_updated"));

    		sm.setDateSysMetadataModified(dateUpdated);

    	} catch (Exception e) {

    		logCrud.debug("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());

    		Date dateCreated = new Date();

    		sm.setDateUploaded(dateCreated);

    		Date dateUpdated = new Date();

    		sm.setDateSysMetadataModified(dateUpdated);

    	}

    	NodeReference nr = new NodeReference();

    	nr.setValue(PropertyService.getProperty("dataone.memberNodeId"));

    	sm.setOriginMemberNode(nr);

    	sm.setAuthoritativeMemberNode(nr);

    	// TODO: Need to set describes/describedBy

    	return sm;

    }