Project

General

Profile

« Previous | Next » 

Revision 5953

To support the generatemissingsystemmetadata REST call, modified CrudService.createSystemMetadata() to use DataoneEMLParser and further determine object formats from EML metadata. Formats currently supported are text/plain, text/csv, image/[jpg|jp2|bmp|tiff|png], and only for EML documents with 'ecogrid://' defined entity urls.
Todo: transition to using the EML Data Manager library's DataPackage parsers to handle a broader set of metadata types instead of the DataoneEMLParser.

View differences:

src/edu/ucsb/nceas/metacat/dataone/CrudService.java
30 30
import java.io.IOException;
31 31
import java.io.InputStream;
32 32
import java.io.OutputStream;
33

  
33 34
import java.security.NoSuchAlgorithmException;
35

  
34 36
import java.sql.SQLException;
37

  
35 38
import java.text.DateFormat;
39

  
36 40
import java.util.Calendar;
37 41
import java.util.Date;
38 42
import java.util.Enumeration;
......
45 49

  
46 50
import javax.servlet.http.HttpServletRequest;
47 51

  
52
import javax.xml.parsers.ParserConfigurationException;
53
import javax.xml.xpath.XPathExpressionException;
54

  
48 55
import org.apache.commons.io.IOUtils;
49 56
import org.apache.log4j.Logger;
57

  
58
import org.dataone.eml.DataoneEMLParser;
59
import org.dataone.eml.EMLDocument;
60
import org.dataone.eml.EMLDocument.DistributionMetadata;
61

  
50 62
import org.dataone.service.exceptions.BaseException;
51 63
import org.dataone.service.exceptions.IdentifierNotUnique;
52 64
import org.dataone.service.exceptions.InsufficientResources;
......
58 70
import org.dataone.service.exceptions.NotImplemented;
59 71
import org.dataone.service.exceptions.ServiceFailure;
60 72
import org.dataone.service.exceptions.UnsupportedType;
73

  
61 74
import org.dataone.service.mn.MemberNodeCrud;
75

  
62 76
import org.dataone.service.types.AuthToken;
63 77
import org.dataone.service.types.Checksum;
64 78
import org.dataone.service.types.ChecksumAlgorithm;
......
73 87
import org.dataone.service.types.Principal;
74 88
import org.dataone.service.types.SystemMetadata;
75 89
import org.dataone.service.types.util.ServiceTypeUtil;
90

  
76 91
import org.jibx.runtime.BindingDirectory;
77 92
import org.jibx.runtime.IBindingFactory;
78 93
import org.jibx.runtime.IMarshallingContext;
79 94
import org.jibx.runtime.IUnmarshallingContext;
80 95
import org.jibx.runtime.JiBXException;
81 96

  
97
import org.xml.sax.SAXException;
98

  
82 99
import edu.ucsb.nceas.metacat.AccessionNumber;
83 100
import edu.ucsb.nceas.metacat.AccessionNumberException;
84 101
import edu.ucsb.nceas.metacat.DocumentImpl;
......
94 111
import edu.ucsb.nceas.metacat.service.SessionService;
95 112
import edu.ucsb.nceas.metacat.util.DocumentUtil;
96 113
import edu.ucsb.nceas.metacat.util.SessionData;
114

  
97 115
import edu.ucsb.nceas.utilities.ParseLSIDException;
98 116
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
99 117

  
......
260 278
     * @param localId the identifier of the object to be processed
261 279
     */
262 280
    public void generateMissingSystemMetadata(AuthToken token, String localId) {
263
        System.out.println("Creating SystemMetadata for localId " + localId);
281
        
282
        logCrud.debug("CrudService.generateMissingSystemMetadata() called.");
283
        
284
        logCrud.debug("Creating SystemMetadata for localId " + localId);
264 285
        try {
265 286
            //generate required system metadata fields from the document
266 287
            SystemMetadata sm = createSystemMetadata(localId, token);
......
268 289
            //insert the systemmetadata object
269 290
            SessionData sessionData = getSessionData(token);
270 291
            String smlocalid = insertSystemMetadata(sm, sessionData);
271
            System.out.println("setting access on SM doc with localid " + smlocalid);
292
            logCrud.debug("setting access on SM doc with localid " + smlocalid);
272 293
            handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid, "public", "4", "allow", "allowFirst");
273 294

  
274 295
            String username = "public";
......
278 299
            EventLog.getInstance().log(metacatUrl, username, localId, "generateMissingSystemMetadata");
279 300
        } catch (Exception e) { // TODO: Please don't catch Exception -- it masks bad things
280 301
            e.printStackTrace();
281
            System.out.println("Exception generating missing system metadata: " + e.getMessage());
302
            logCrud.debug("Exception generating missing system metadata: " + e.getMessage());
282 303
            logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
283 304
        }
284 305
        logCrud.info("generateMissingSystemMetadata(token, localId)");
......
334 355
        if (isScienceMetadata) {
335 356
            // CASE METADATA:
336 357
            try {
337
                //System.out.println("CrudService: inserting document with guid " + guid.getValue());
358
                //logCrud.debug("CrudService: inserting document with guid " + guid.getValue());
338 359
                this.insertDocument(object, guid, sessionData);
339 360
                localId = im.getLocalId(guid.getValue());
340 361
            } catch (IOException e) {
......
358 379
        // For Metadata and Data, insert the system metadata into the object store too
359 380
        String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);
360 381
        //get the document info.  add any access params for the sysmeta too
361
        //System.out.println("looking for access records to add for system " +
382
        //logCrud.debug("looking for access records to add for system " +
362 383
        //    "metadata who's parent doc's  local id is " + localId);
363 384
        try
364 385
        {
......
373 394
                String permissionType = (String)ah.get("permission_type");
374 395
                String permissionOrder = (String)ah.get("permission_order");
375 396
                int perm = new Integer(permission).intValue();
376
                //System.out.println("found access record for principal " + principal);
377
                //System.out.println("permission: " + perm + " perm_type: " + permissionType + 
397
                //logCrud.debug("found access record for principal " + principal);
398
                //logCrud.debug("permission: " + perm + " perm_type: " + permissionType + 
378 399
                //    " perm_order: " + permissionOrder);
379 400
                this.setAccess(token, guid, principal, perm, permissionType, permissionOrder, true);
380 401
            }
......
484 505
        {
485 506
            perm = "write";
486 507
        }
487
        //System.out.println("perm in setAccess: " + perm);
488
        //System.out.println("permission in setAccess: " + permission);
508
        //logCrud.debug("perm in setAccess: " + perm);
509
        //logCrud.debug("permission in setAccess: " + permission);
489 510
        setAccess(token, id, principal, perm, permissionType, permissionOrder,
490 511
                setSystemMetadata);
491 512
       
......
529 550
            {
530 551
                permNum = "6";
531 552
            }
532
            System.out.println("user " + sessionData.getUserName() + 
553
            logCrud.debug("user " + sessionData.getUserName() + 
533 554
                    " is setting access level " + permNum + " for permission " + 
534 555
                    permissionType + " on doc with localid " + docid);
535 556
            handler.setAccess(metacatUrl, sessionData.getUserName(), docid, 
......
538 559
            {
539 560
                //set the same perms on the system metadata doc
540 561
                String smlocalid = im.getSystemMetadataLocalId(id.getValue());
541
                System.out.println("setting access on SM doc with localid " + smlocalid);
562
                logCrud.debug("setting access on SM doc with localid " + smlocalid);
542 563
                //cs.setAccess(token, smid, principal, permission, permissionType, permissionOrder);
543 564
                handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid,
544 565
                        principal, permNum, permissionType, permissionOrder);
......
1572 1593
        Identifier sysMetaGuid = new Identifier();
1573 1594
        sysMetaGuid.setValue(DocumentUtil.generateDocumentId(1));
1574 1595
        sysmeta.setDateSysMetadataModified(new Date());
1575
        System.out.println("****inserting new system metadata with modified date " + 
1596
        logCrud.debug("****inserting new system metadata with modified date " + 
1576 1597
                sysmeta.getDateSysMetadataModified());
1577 1598

  
1578 1599
        String xml = new String(serializeSystemMetadata(sysmeta).toByteArray());
1579
        System.out.println("sysmeta: " + xml);
1600
        logCrud.debug("sysmeta: " + xml);
1580 1601
        String localId = insertDocument(xml, sysMetaGuid, sessionData, true);
1581
        System.out.println("sysmeta inserted with localId " + localId);
1602
        logCrud.debug("sysmeta inserted with localId " + localId);
1582 1603
        //insert the system metadata doc id into the systemmetadata table to 
1583 1604
        //link it to the data or metadata document
1584 1605
        IdentifierManager.getInstance().createSystemMetadataMapping(
......
1592 1613
    private void updateSystemMetadata(SystemMetadata sm, SessionData sessionData)
1593 1614
      throws ServiceFailure
1594 1615
    {
1616
        
1617
        logCrud.debug("CrudService.updateSystemMetadata() called.");
1595 1618
        try
1596 1619
        {
1597 1620
            String smId = IdentifierManager.getInstance().getSystemMetadataLocalId(sm.getIdentifier().getValue());
1598
            System.out.println("setting date modified to " + new Date());
1621
            logCrud.debug("Setting date modified to " + new Date());
1599 1622
            sm.setDateSysMetadataModified(new Date());
1600 1623
            String xml = new String(serializeSystemMetadata(sm).toByteArray());
1601 1624
            String localId = updateDocument(xml, sm.getIdentifier(), null, sessionData, true);
1602 1625
            IdentifierManager.getInstance().updateSystemMetadataMapping(sm.getIdentifier().getValue(), localId);
1626
            IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(
1627
              sm.getDateUploaded().getTime(), 
1628
              sm.getRightsHolder().getValue(),
1629
              sm.getChecksum().getValue(), 
1630
              sm.getChecksum().getAlgorithm().toString(), 
1631
              sm.getOriginMemberNode().getValue(), 
1632
              sm.getAuthoritativeMemberNode().getValue(), 
1633
              sm.getDateSysMetadataModified().getTime(), 
1634
              sm.getSubmitter().getValue(), 
1635
              sm.getIdentifier().getValue(), 
1636
              sm.getObjectFormat().toString(), 
1637
              sm.getSize());
1638
                        
1603 1639
        }
1604 1640
        catch(Exception e)
1605 1641
        {
......
1686 1722
            //localid should already exist in the identifier table, so just find it
1687 1723
            try
1688 1724
            {
1689
                System.out.println("updating guid " + guid.getValue());
1725
                logCrud.debug("updating guid " + guid.getValue());
1690 1726
                if(!isSystemMetadata)
1691 1727
                {
1692
                    System.out.println("looking in identifier table for guid " + guid.getValue());
1728
                    logCrud.debug("looking in identifier table for guid " + guid.getValue());
1693 1729
                    localId = im.getLocalId(guid.getValue());
1694 1730
                }
1695 1731
                else
1696 1732
                {
1697
                    System.out.println("looking in systemmetadata table for guid " + guid.getValue());
1733
                    logCrud.debug("looking in systemmetadata table for guid " + guid.getValue());
1698 1734
                    localId = im.getSystemMetadataLocalId(guid.getValue());
1699 1735
                }
1700
                System.out.println("localId: " + localId);
1736
                logCrud.debug("localId: " + localId);
1701 1737
                //increment the revision
1702 1738
                String docid = localId.substring(0, localId.lastIndexOf("."));
1703 1739
                String revS = localId.substring(localId.lastIndexOf(".") + 1, localId.length());
......
1705 1741
                rev++;
1706 1742
                docid = docid + "." + rev;
1707 1743
                localId = docid;
1708
                System.out.println("incremented localId: " + localId);
1744
                logCrud.debug("incremented localId: " + localId);
1709 1745
            }
1710 1746
            catch(McdbDocNotFoundException e)
1711 1747
            {
......
1896 1932
        c.set(new Integer(year).intValue(), 
1897 1933
              new Integer(month).intValue(), 
1898 1934
              new Integer(day).intValue());
1899
        System.out.println("time in parseMetacatDate: " + c.getTime());
1935
        logCrud.debug("time in parseMetacatDate: " + c.getTime());
1900 1936
        return c.getTime();
1901 1937
    }
1902 1938
    
......
1934 1970
        throws McdbDocNotFoundException, NumberFormatException, AccessionNumberException, 
1935 1971
        SQLException, NoSuchAlgorithmException, IOException, PropertyNotFoundException, BaseException {
1936 1972
        
1973
        logCrud.debug("CrudService.createSystemMetadata() called.");
1974
        
1937 1975
        IdentifierManager im = IdentifierManager.getInstance();
1938 1976
        Hashtable<String, Object> docInfo = im.getDocumentInfo(localId);
1939 1977

  
......
1942 1980
        Identifier identifier = new Identifier();
1943 1981
        try {
1944 1982
            identifier.setValue(im.getGUID(localId, rev));
1983
            
1945 1984
        } catch (McdbDocNotFoundException mcdbe) { 
1946 1985
            //we're creating a new SM doc for a doc that is not in the identifier table                                       
1947 1986
            //so we need to add it to
1948
            System.out.println("No guid in the identifier table.  adding it for " + localId);
1987
            logCrud.debug("No guid in the identifier table.  adding it for " + localId);
1949 1988
            im.createMapping(localId, localId);
1950
            System.out.println("mapping created for " + localId);
1989
            logCrud.debug("Mapping created for " + localId);
1951 1990
            AccessionNumber accNum = new AccessionNumber(localId, "NONE");
1952 1991
            identifier.setValue(im.getGUID(accNum.getDocid(), rev));
1953 1992
        }
1954 1993

  
1955
        System.out.println("creating system metadata for guid " + identifier.getValue());
1994
        logCrud.debug("Creating system metadata for guid " + identifier.getValue());
1956 1995
        InputStream is = this.get(token, identifier);
1957 1996
        SystemMetadata sm = new SystemMetadata();
1958 1997
        
1959 1998
        //set the id
1960 1999
        sm.setIdentifier(identifier);
1961 2000

  
1962
        //set the object format
2001
        //set the default object format
1963 2002
        String doctype = (String) docInfo.get("doctype");
1964
        ObjectFormat format = ObjectFormat.convert((String) docInfo.get("doctype"));
2003
        ObjectFormat format = ObjectFormat.convert(doctype);
1965 2004
        if (format == null) {
1966 2005
            if (doctype.trim().equals("BIN")) {
1967 2006
                format = ObjectFormat.OCTET_STREAM;
......
1970 2009
            }
1971 2010
        }
1972 2011
        sm.setObjectFormat(format);
1973

  
2012
        logCrud.debug("The ObjectFormat for " + localId + " is " + format.toString());
2013
        
2014
        // further parse EML documents to get data object format,
2015
        // describes and describedBy information
2016
        if ( format == ObjectFormat.EML_2_0_0 ||
2017
             format == ObjectFormat.EML_2_0_1 ||
2018
             format == ObjectFormat.EML_2_1_0 ) {
2019
          
2020
          try {
2021
            DataoneEMLParser emlParser = DataoneEMLParser.getInstance();
2022
            EMLDocument emlDocument = emlParser.parseDocument(is);
2023
            
2024
            // iterate through the data objects in the EML doc and add sysmeta
2025
           logCrud.debug("The number of data entities is: " +
2026
                         emlDocument.distributionMetadata.size());
2027
                            
2028
           for( int j = 0; j < emlDocument.distributionMetadata.size(); j++ ) {
2029
             
2030
             DistributionMetadata distMetadata = 
2031
               emlDocument.distributionMetadata.elementAt(j);
2032
             String dataDocUrl = distMetadata.url;
2033
             String dataDocMimeType = distMetadata.mimeType;
2034
             String dataDocLocalId = "";
2035
             logCrud.debug("\tData local ID: " + dataDocLocalId);
2036
             logCrud.debug("\tData URL: " + dataDocUrl);
2037
             logCrud.debug("\tData mime: " + dataDocMimeType);
2038
             
2039
             //we only handle ecogrid urls right now
2040
             if ( dataDocUrl.trim().startsWith("ecogrid://knb/") ) {
2041
               dataDocLocalId = 
2042
                 dataDocUrl.substring(dataDocUrl.indexOf("ecogrid://knb/") + 
2043
                 "ecogrid://knb/".length(), dataDocUrl.length());
2044
               
2045
               //set the id
2046
               Identifier dataDocId = new Identifier();
2047
               dataDocId.setValue(dataDocLocalId);
2048
               
2049
               // add describes into EML system metadata
2050
               sm.addDescribe(dataDocId);
2051
               
2052
               SystemMetadata dataSysMeta = new SystemMetadata();
2053
               // check if data system metadata exists
2054
               try {
2055
                 logCrud.debug("Checking for existing system metadata for " + dataDocId.getValue());
2056
                 dataSysMeta = this.getSystemMetadata(token, dataDocId);
2057
                 // add describedBy sysmeta
2058
                 logCrud.debug("Setting describedBy for " + dataDocId.getValue() +
2059
                                  " to " + identifier.getValue());
2060
                 dataSysMeta.addDescribedBy(identifier);
2061
                 dataSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));
2062
                 this.updateSystemMetadata(dataSysMeta, getSessionData(token));
2063
                 
2064
               } catch ( NotFound nf ) {
2065
                 // System metadata for data doesn't exist
2066
                 logCrud.debug("There was not an existing system metadata " + "document for " + dataDocId.getValue());
2067
                 try {
2068
                   logCrud.debug("Creating a system metadata " + "document for " + dataDocId.getValue());
2069
                   dataSysMeta = this.createSystemMetadata(dataDocLocalId, token);
2070
                   
2071
                   logCrud.debug("Setting describedBy for " + dataDocId.getValue() + " to " + identifier.getValue());
2072
                   dataSysMeta.addDescribedBy(identifier);
2073
                   
2074
                   logCrud.debug("Setting mimeType for " + dataDocId.getValue() + " to " + dataDocMimeType);
2075
                   dataSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));
2076
                   
2077
                   logCrud.debug("Updating system metadata for " + dataDocId.getValue() + " to " + dataDocMimeType);
2078
                   this.updateSystemMetadata(dataSysMeta, getSessionData(token));
2079
                   
2080
                 } catch ( McdbDocNotFoundException mdnf) {
2081
                   mdnf.printStackTrace();
2082
                   throw mdnf;
2083
                 } catch ( NumberFormatException nfe) {
2084
                   nfe.printStackTrace();
2085
                   throw nfe;
2086
                 } catch ( AccessionNumberException ane) {
2087
                   ane.printStackTrace();
2088
                   throw ane;
2089
                 } catch ( SQLException sqle) {
2090
                   sqle.printStackTrace();
2091
                   throw sqle;
2092
                 } catch ( NoSuchAlgorithmException nsae) {
2093
                   nsae.printStackTrace();
2094
                   throw nsae;
2095
                 } catch ( IOException ioe) {
2096
                   ioe.printStackTrace();
2097
                   throw ioe;
2098
                 } catch ( PropertyNotFoundException pnfe) {
2099
                   pnfe.printStackTrace();
2100
                   throw pnfe;
2101
                 } catch ( BaseException be) {
2102
                   be.printStackTrace();
2103
                   throw be;
2104
                   
2105
                 }
2106
                 
2107
               }
2108
               
2109
             } // end if()
2110
             
2111
           } // end for()
2112
           
2113
          } catch ( ParserConfigurationException pce ) {
2114
            logCrud.debug("There was a problem parsing the EML document. " +
2115
                             "The error message was: " + pce.getMessage());
2116
          
2117
          } catch ( SAXException saxe ) {
2118
            logCrud.debug("There was a problem traversing the EML document. " +
2119
                             "The error message was: " + saxe.getMessage());
2120
          
2121
          } catch ( XPathExpressionException xpee ) {
2122
            logCrud.debug("There was a problem searching the EML document. " +
2123
                             "The error message was: " + xpee.getMessage());
2124
          } // end try
2125
           
2126
        } // end if()
2127
        
1974 2128
        //create the checksum
2129
        is = this.get(token, identifier);
1975 2130
        String checksumS = checksum(is);
1976 2131
        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");
1977 2132
        Checksum checksum = new Checksum();
......
1994 2149
            Date dateUpdated = parseMetacatDate((String) docInfo.get("date_updated"));
1995 2150
            sm.setDateSysMetadataModified(dateUpdated);
1996 2151
        } catch (Exception e) {
1997
            System.out.println("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
2152
            logCrud.debug("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
1998 2153
            Date dateCreated = new Date();
1999 2154
            sm.setDateUploaded(dateCreated);
2000 2155
            Date dateUpdated = new Date();

Also available in: Unified diff