Project

General

Profile

« Previous | Next » 

Revision 5953

Added by Chris Jones over 13 years ago

To support the generatemissingsystemmetadata REST call, modified CrudService.createSystemMetadata() to use DataoneEMLParser and further determine object formats from EML metadata. Formats currently supported are text/plain, text/csv, image/[jpg|jp2|bmp|tiff|png], and only for EML documents with 'ecogrid://' defined entity urls.
Todo: transition to using the EML Data Manager library's DataPackage parsers to handle a broader set of metadata types instead of the DataoneEMLParser.

View differences:

src/edu/ucsb/nceas/metacat/dataone/CrudService.java
30 30
import java.io.IOException;
31 31
import java.io.InputStream;
32 32
import java.io.OutputStream;
33

  
33 34
import java.security.NoSuchAlgorithmException;
35

  
34 36
import java.sql.SQLException;
37

  
35 38
import java.text.DateFormat;
39

  
36 40
import java.util.Calendar;
37 41
import java.util.Date;
38 42
import java.util.Enumeration;
......
45 49

  
46 50
import javax.servlet.http.HttpServletRequest;
47 51

  
52
import javax.xml.parsers.ParserConfigurationException;
53
import javax.xml.xpath.XPathExpressionException;
54

  
48 55
import org.apache.commons.io.IOUtils;
49 56
import org.apache.log4j.Logger;
57

  
58
import org.dataone.eml.DataoneEMLParser;
59
import org.dataone.eml.EMLDocument;
60
import org.dataone.eml.EMLDocument.DistributionMetadata;
61

  
50 62
import org.dataone.service.exceptions.BaseException;
51 63
import org.dataone.service.exceptions.IdentifierNotUnique;
52 64
import org.dataone.service.exceptions.InsufficientResources;
......
58 70
import org.dataone.service.exceptions.NotImplemented;
59 71
import org.dataone.service.exceptions.ServiceFailure;
60 72
import org.dataone.service.exceptions.UnsupportedType;
73

  
61 74
import org.dataone.service.mn.MemberNodeCrud;
75

  
62 76
import org.dataone.service.types.AuthToken;
63 77
import org.dataone.service.types.Checksum;
64 78
import org.dataone.service.types.ChecksumAlgorithm;
......
73 87
import org.dataone.service.types.Principal;
74 88
import org.dataone.service.types.SystemMetadata;
75 89
import org.dataone.service.types.util.ServiceTypeUtil;
90

  
76 91
import org.jibx.runtime.BindingDirectory;
77 92
import org.jibx.runtime.IBindingFactory;
78 93
import org.jibx.runtime.IMarshallingContext;
79 94
import org.jibx.runtime.IUnmarshallingContext;
80 95
import org.jibx.runtime.JiBXException;
81 96

  
97
import org.xml.sax.SAXException;
98

  
82 99
import edu.ucsb.nceas.metacat.AccessionNumber;
83 100
import edu.ucsb.nceas.metacat.AccessionNumberException;
84 101
import edu.ucsb.nceas.metacat.DocumentImpl;
......
94 111
import edu.ucsb.nceas.metacat.service.SessionService;
95 112
import edu.ucsb.nceas.metacat.util.DocumentUtil;
96 113
import edu.ucsb.nceas.metacat.util.SessionData;
114

  
97 115
import edu.ucsb.nceas.utilities.ParseLSIDException;
98 116
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
99 117

  
......
260 278
     * @param localId the identifier of the object to be processed
261 279
     */
262 280
    public void generateMissingSystemMetadata(AuthToken token, String localId) {
263
        System.out.println("Creating SystemMetadata for localId " + localId);
281
        
282
        logCrud.debug("CrudService.generateMissingSystemMetadata() called.");
283
        
284
        logCrud.debug("Creating SystemMetadata for localId " + localId);
264 285
        try {
265 286
            //generate required system metadata fields from the document
266 287
            SystemMetadata sm = createSystemMetadata(localId, token);
......
268 289
            //insert the systemmetadata object
269 290
            SessionData sessionData = getSessionData(token);
270 291
            String smlocalid = insertSystemMetadata(sm, sessionData);
271
            System.out.println("setting access on SM doc with localid " + smlocalid);
292
            logCrud.debug("setting access on SM doc with localid " + smlocalid);
272 293
            handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid, "public", "4", "allow", "allowFirst");
273 294

  
274 295
            String username = "public";
......
278 299
            EventLog.getInstance().log(metacatUrl, username, localId, "generateMissingSystemMetadata");
279 300
        } catch (Exception e) { // TODO: Please don't catch Exception -- it masks bad things
280 301
            e.printStackTrace();
281
            System.out.println("Exception generating missing system metadata: " + e.getMessage());
302
            logCrud.debug("Exception generating missing system metadata: " + e.getMessage());
282 303
            logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
283 304
        }
284 305
        logCrud.info("generateMissingSystemMetadata(token, localId)");
......
334 355
        if (isScienceMetadata) {
335 356
            // CASE METADATA:
336 357
            try {
337
                //System.out.println("CrudService: inserting document with guid " + guid.getValue());
358
                //logCrud.debug("CrudService: inserting document with guid " + guid.getValue());
338 359
                this.insertDocument(object, guid, sessionData);
339 360
                localId = im.getLocalId(guid.getValue());
340 361
            } catch (IOException e) {
......
358 379
        // For Metadata and Data, insert the system metadata into the object store too
359 380
        String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);
360 381
        //get the document info.  add any access params for the sysmeta too
361
        //System.out.println("looking for access records to add for system " +
382
        //logCrud.debug("looking for access records to add for system " +
362 383
        //    "metadata who's parent doc's  local id is " + localId);
363 384
        try
364 385
        {
......
373 394
                String permissionType = (String)ah.get("permission_type");
374 395
                String permissionOrder = (String)ah.get("permission_order");
375 396
                int perm = new Integer(permission).intValue();
376
                //System.out.println("found access record for principal " + principal);
377
                //System.out.println("permission: " + perm + " perm_type: " + permissionType + 
397
                //logCrud.debug("found access record for principal " + principal);
398
                //logCrud.debug("permission: " + perm + " perm_type: " + permissionType + 
378 399
                //    " perm_order: " + permissionOrder);
379 400
                this.setAccess(token, guid, principal, perm, permissionType, permissionOrder, true);
380 401
            }
......
484 505
        {
485 506
            perm = "write";
486 507
        }
487
        //System.out.println("perm in setAccess: " + perm);
488
        //System.out.println("permission in setAccess: " + permission);
508
        //logCrud.debug("perm in setAccess: " + perm);
509
        //logCrud.debug("permission in setAccess: " + permission);
489 510
        setAccess(token, id, principal, perm, permissionType, permissionOrder,
490 511
                setSystemMetadata);
491 512
       
......
529 550
            {
530 551
                permNum = "6";
531 552
            }
532
            System.out.println("user " + sessionData.getUserName() + 
553
            logCrud.debug("user " + sessionData.getUserName() + 
533 554
                    " is setting access level " + permNum + " for permission " + 
534 555
                    permissionType + " on doc with localid " + docid);
535 556
            handler.setAccess(metacatUrl, sessionData.getUserName(), docid, 
......
538 559
            {
539 560
                //set the same perms on the system metadata doc
540 561
                String smlocalid = im.getSystemMetadataLocalId(id.getValue());
541
                System.out.println("setting access on SM doc with localid " + smlocalid);
562
                logCrud.debug("setting access on SM doc with localid " + smlocalid);
542 563
                //cs.setAccess(token, smid, principal, permission, permissionType, permissionOrder);
543 564
                handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid,
544 565
                        principal, permNum, permissionType, permissionOrder);
......
1572 1593
        Identifier sysMetaGuid = new Identifier();
1573 1594
        sysMetaGuid.setValue(DocumentUtil.generateDocumentId(1));
1574 1595
        sysmeta.setDateSysMetadataModified(new Date());
1575
        System.out.println("****inserting new system metadata with modified date " + 
1596
        logCrud.debug("****inserting new system metadata with modified date " + 
1576 1597
                sysmeta.getDateSysMetadataModified());
1577 1598

  
1578 1599
        String xml = new String(serializeSystemMetadata(sysmeta).toByteArray());
1579
        System.out.println("sysmeta: " + xml);
1600
        logCrud.debug("sysmeta: " + xml);
1580 1601
        String localId = insertDocument(xml, sysMetaGuid, sessionData, true);
1581
        System.out.println("sysmeta inserted with localId " + localId);
1602
        logCrud.debug("sysmeta inserted with localId " + localId);
1582 1603
        //insert the system metadata doc id into the systemmetadata table to 
1583 1604
        //link it to the data or metadata document
1584 1605
        IdentifierManager.getInstance().createSystemMetadataMapping(
......
1592 1613
    private void updateSystemMetadata(SystemMetadata sm, SessionData sessionData)
1593 1614
      throws ServiceFailure
1594 1615
    {
1616
        
1617
        logCrud.debug("CrudService.updateSystemMetadata() called.");
1595 1618
        try
1596 1619
        {
1597 1620
            String smId = IdentifierManager.getInstance().getSystemMetadataLocalId(sm.getIdentifier().getValue());
1598
            System.out.println("setting date modified to " + new Date());
1621
            logCrud.debug("Setting date modified to " + new Date());
1599 1622
            sm.setDateSysMetadataModified(new Date());
1600 1623
            String xml = new String(serializeSystemMetadata(sm).toByteArray());
1601 1624
            String localId = updateDocument(xml, sm.getIdentifier(), null, sessionData, true);
1602 1625
            IdentifierManager.getInstance().updateSystemMetadataMapping(sm.getIdentifier().getValue(), localId);
1626
            IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(
1627
              sm.getDateUploaded().getTime(), 
1628
              sm.getRightsHolder().getValue(),
1629
              sm.getChecksum().getValue(), 
1630
              sm.getChecksum().getAlgorithm().toString(), 
1631
              sm.getOriginMemberNode().getValue(), 
1632
              sm.getAuthoritativeMemberNode().getValue(), 
1633
              sm.getDateSysMetadataModified().getTime(), 
1634
              sm.getSubmitter().getValue(), 
1635
              sm.getIdentifier().getValue(), 
1636
              sm.getObjectFormat().toString(), 
1637
              sm.getSize());
1638
                        
1603 1639
        }
1604 1640
        catch(Exception e)
1605 1641
        {
......
1686 1722
            //localid should already exist in the identifier table, so just find it
1687 1723
            try
1688 1724
            {
1689
                System.out.println("updating guid " + guid.getValue());
1725
                logCrud.debug("updating guid " + guid.getValue());
1690 1726
                if(!isSystemMetadata)
1691 1727
                {
1692
                    System.out.println("looking in identifier table for guid " + guid.getValue());
1728
                    logCrud.debug("looking in identifier table for guid " + guid.getValue());
1693 1729
                    localId = im.getLocalId(guid.getValue());
1694 1730
                }
1695 1731
                else
1696 1732
                {
1697
                    System.out.println("looking in systemmetadata table for guid " + guid.getValue());
1733
                    logCrud.debug("looking in systemmetadata table for guid " + guid.getValue());
1698 1734
                    localId = im.getSystemMetadataLocalId(guid.getValue());
1699 1735
                }
1700
                System.out.println("localId: " + localId);
1736
                logCrud.debug("localId: " + localId);
1701 1737
                //increment the revision
1702 1738
                String docid = localId.substring(0, localId.lastIndexOf("."));
1703 1739
                String revS = localId.substring(localId.lastIndexOf(".") + 1, localId.length());
......
1705 1741
                rev++;
1706 1742
                docid = docid + "." + rev;
1707 1743
                localId = docid;
1708
                System.out.println("incremented localId: " + localId);
1744
                logCrud.debug("incremented localId: " + localId);
1709 1745
            }
1710 1746
            catch(McdbDocNotFoundException e)
1711 1747
            {
......
1896 1932
        c.set(new Integer(year).intValue(), 
1897 1933
              new Integer(month).intValue(), 
1898 1934
              new Integer(day).intValue());
1899
        System.out.println("time in parseMetacatDate: " + c.getTime());
1935
        logCrud.debug("time in parseMetacatDate: " + c.getTime());
1900 1936
        return c.getTime();
1901 1937
    }
1902 1938
    
......
1934 1970
        throws McdbDocNotFoundException, NumberFormatException, AccessionNumberException, 
1935 1971
        SQLException, NoSuchAlgorithmException, IOException, PropertyNotFoundException, BaseException {
1936 1972
        
1973
        logCrud.debug("CrudService.createSystemMetadata() called.");
1974
        
1937 1975
        IdentifierManager im = IdentifierManager.getInstance();
1938 1976
        Hashtable<String, Object> docInfo = im.getDocumentInfo(localId);
1939 1977

  
......
1942 1980
        Identifier identifier = new Identifier();
1943 1981
        try {
1944 1982
            identifier.setValue(im.getGUID(localId, rev));
1983
            
1945 1984
        } catch (McdbDocNotFoundException mcdbe) { 
1946 1985
            //we're creating a new SM doc for a doc that is not in the identifier table                                       
1947 1986
            //so we need to add it to
1948
            System.out.println("No guid in the identifier table.  adding it for " + localId);
1987
            logCrud.debug("No guid in the identifier table.  adding it for " + localId);
1949 1988
            im.createMapping(localId, localId);
1950
            System.out.println("mapping created for " + localId);
1989
            logCrud.debug("Mapping created for " + localId);
1951 1990
            AccessionNumber accNum = new AccessionNumber(localId, "NONE");
1952 1991
            identifier.setValue(im.getGUID(accNum.getDocid(), rev));
1953 1992
        }
1954 1993

  
1955
        System.out.println("creating system metadata for guid " + identifier.getValue());
1994
        logCrud.debug("Creating system metadata for guid " + identifier.getValue());
1956 1995
        InputStream is = this.get(token, identifier);
1957 1996
        SystemMetadata sm = new SystemMetadata();
1958 1997
        
1959 1998
        //set the id
1960 1999
        sm.setIdentifier(identifier);
1961 2000

  
1962
        //set the object format
2001
        //set the default object format
1963 2002
        String doctype = (String) docInfo.get("doctype");
1964
        ObjectFormat format = ObjectFormat.convert((String) docInfo.get("doctype"));
2003
        ObjectFormat format = ObjectFormat.convert(doctype);
1965 2004
        if (format == null) {
1966 2005
            if (doctype.trim().equals("BIN")) {
1967 2006
                format = ObjectFormat.OCTET_STREAM;
......
1970 2009
            }
1971 2010
        }
1972 2011
        sm.setObjectFormat(format);
1973

  
2012
        logCrud.debug("The ObjectFormat for " + localId + " is " + format.toString());
2013
        
2014
        // further parse EML documents to get data object format,
2015
        // describes and describedBy information
2016
        if ( format == ObjectFormat.EML_2_0_0 ||
2017
             format == ObjectFormat.EML_2_0_1 ||
2018
             format == ObjectFormat.EML_2_1_0 ) {
2019
          
2020
          try {
2021
            DataoneEMLParser emlParser = DataoneEMLParser.getInstance();
2022
            EMLDocument emlDocument = emlParser.parseDocument(is);
2023
            
2024
            // iterate through the data objects in the EML doc and add sysmeta
2025
           logCrud.debug("The number of data entities is: " +
2026
                         emlDocument.distributionMetadata.size());
2027
                            
2028
           for( int j = 0; j < emlDocument.distributionMetadata.size(); j++ ) {
2029
             
2030
             DistributionMetadata distMetadata = 
2031
               emlDocument.distributionMetadata.elementAt(j);
2032
             String dataDocUrl = distMetadata.url;
2033
             String dataDocMimeType = distMetadata.mimeType;
2034
             String dataDocLocalId = "";
2035
             logCrud.debug("\tData local ID: " + dataDocLocalId);
2036
             logCrud.debug("\tData URL: " + dataDocUrl);
2037
             logCrud.debug("\tData mime: " + dataDocMimeType);
2038
             
2039
             //we only handle ecogrid urls right now
2040
             if ( dataDocUrl.trim().startsWith("ecogrid://knb/") ) {
2041
               dataDocLocalId = 
2042
                 dataDocUrl.substring(dataDocUrl.indexOf("ecogrid://knb/") + 
2043
                 "ecogrid://knb/".length(), dataDocUrl.length());
2044
               
2045
               //set the id
2046
               Identifier dataDocId = new Identifier();
2047
               dataDocId.setValue(dataDocLocalId);
2048
               
2049
               // add describes into EML system metadata
2050
               sm.addDescribe(dataDocId);
2051
               
2052
               SystemMetadata dataSysMeta = new SystemMetadata();
2053
               // check if data system metadata exists
2054
               try {
2055
                 logCrud.debug("Checking for existing system metadata for " + dataDocId.getValue());
2056
                 dataSysMeta = this.getSystemMetadata(token, dataDocId);
2057
                 // add describedBy sysmeta
2058
                 logCrud.debug("Setting describedBy for " + dataDocId.getValue() +
2059
                                  " to " + identifier.getValue());
2060
                 dataSysMeta.addDescribedBy(identifier);
2061
                 dataSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));
2062
                 this.updateSystemMetadata(dataSysMeta, getSessionData(token));
2063
                 
2064
               } catch ( NotFound nf ) {
2065
                 // System metadata for data doesn't exist
2066
                 logCrud.debug("There was not an existing system metadata " + "document for " + dataDocId.getValue());
2067
                 try {
2068
                   logCrud.debug("Creating a system metadata " + "document for " + dataDocId.getValue());
2069
                   dataSysMeta = this.createSystemMetadata(dataDocLocalId, token);
2070
                   
2071
                   logCrud.debug("Setting describedBy for " + dataDocId.getValue() + " to " + identifier.getValue());
2072
                   dataSysMeta.addDescribedBy(identifier);
2073
                   
2074
                   logCrud.debug("Setting mimeType for " + dataDocId.getValue() + " to " + dataDocMimeType);
2075
                   dataSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));
2076
                   
2077
                   logCrud.debug("Updating system metadata for " + dataDocId.getValue() + " to " + dataDocMimeType);
2078
                   this.updateSystemMetadata(dataSysMeta, getSessionData(token));
2079
                   
2080
                 } catch ( McdbDocNotFoundException mdnf) {
2081
                   mdnf.printStackTrace();
2082
                   throw mdnf;
2083
                 } catch ( NumberFormatException nfe) {
2084
                   nfe.printStackTrace();
2085
                   throw nfe;
2086
                 } catch ( AccessionNumberException ane) {
2087
                   ane.printStackTrace();
2088
                   throw ane;
2089
                 } catch ( SQLException sqle) {
2090
                   sqle.printStackTrace();
2091
                   throw sqle;
2092
                 } catch ( NoSuchAlgorithmException nsae) {
2093
                   nsae.printStackTrace();
2094
                   throw nsae;
2095
                 } catch ( IOException ioe) {
2096
                   ioe.printStackTrace();
2097
                   throw ioe;
2098
                 } catch ( PropertyNotFoundException pnfe) {
2099
                   pnfe.printStackTrace();
2100
                   throw pnfe;
2101
                 } catch ( BaseException be) {
2102
                   be.printStackTrace();
2103
                   throw be;
2104
                   
2105
                 }
2106
                 
2107
               }
2108
               
2109
             } // end if()
2110
             
2111
           } // end for()
2112
           
2113
          } catch ( ParserConfigurationException pce ) {
2114
            logCrud.debug("There was a problem parsing the EML document. " +
2115
                             "The error message was: " + pce.getMessage());
2116
          
2117
          } catch ( SAXException saxe ) {
2118
            logCrud.debug("There was a problem traversing the EML document. " +
2119
                             "The error message was: " + saxe.getMessage());
2120
          
2121
          } catch ( XPathExpressionException xpee ) {
2122
            logCrud.debug("There was a problem searching the EML document. " +
2123
                             "The error message was: " + xpee.getMessage());
2124
          } // end try
2125
           
2126
        } // end if()
2127
        
1974 2128
        //create the checksum
2129
        is = this.get(token, identifier);
1975 2130
        String checksumS = checksum(is);
1976 2131
        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");
1977 2132
        Checksum checksum = new Checksum();
......
1994 2149
            Date dateUpdated = parseMetacatDate((String) docInfo.get("date_updated"));
1995 2150
            sm.setDateSysMetadataModified(dateUpdated);
1996 2151
        } catch (Exception e) {
1997
            System.out.println("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
2152
            logCrud.debug("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
1998 2153
            Date dateCreated = new Date();
1999 2154
            sm.setDateUploaded(dateCreated);
2000 2155
            Date dateUpdated = new Date();

Also available in: Unified diff