Project

General

Profile

« Previous | Next » 

Revision 6097

do not use XML files for storing SystemMetadata - use DB tables only.

View differences:

test/edu/ucsb/nceas/metacat/dataone/CrudServiceTest.java
109 109
		suite.addTest(new CrudServiceTest("testDescribe"));
110 110
		suite.addTest(new CrudServiceTest("testDelete"));
111 111
		suite.addTest(new CrudServiceTest("testSemiColonsInIdentifiers"));
112
		suite.addTest(new CrudServiceTest("testGenerateMissingSystemMetadata"));
113 112
		return suite;
114 113
	}
115 114
	
......
422 421
	}
423 422
	
424 423
	/**
425
	 * test the generation of system metadata for docs that don't already 
426
	 * have it.  This will be used for migration of existing object stores
427
	 * to dataone.
428
	 */
429
	public void testGenerateMissingSystemMetadata()
430
	{
431
	    printTestHeader("testGenerateMissingSystemMetadata");
432
	    try
433
	    {
434
	        
435
	        CrudService cs = CrudService.getInstance();
436
	        AuthToken token = getToken();
437
	        //create a document with no system metadata
438
	        String testDoc = getTestDoc();
439
	        Identifier id = new Identifier();
440
	        String docid = generateDocumentId();
441
	        id.setValue(docid);
442
	        
443
	        cs.insertOrUpdateDocument(testDoc, id, cs.getSessionData(token), "insert", false); 
444
	        //try to get its system metadata, should fail
445
	        try
446
	        {
447
	            getSystemMetadata(token, id);
448
	            fail("call to getSystemMetadata should have failed.");
449
	        }
450
	        catch(org.dataone.service.exceptions.NotFound e)
451
	        {
452
	        	// expect an exception of this type
453
	        }
454
	        
455
	        //generate missing system metadata
456
	        cs.generateMissingSystemMetadata(token);
457
	        //try to get system metadata again, should succeed
458
	        SystemMetadata smd = getSystemMetadata(token, id);
459
	        assertTrue("SystemMetadata should not be null", smd != null);
460
	    }
461
	    catch(Exception e)
462
	    {
463
	    	System.out.println("testGenerateSystemMetadata  ***  exception trace:");
464
	    	e.printStackTrace();
465
	    	fail("Unexpected error generating missing system metadata: " + e.getClass() + ": " + e.getMessage());
466
	    }
467
	}
468
	
469
	/**
470 424
	 * make sure that only valid sessions can update/delete
471 425
	 */
472 426
	public void testAccessControl()
src/edu/ucsb/nceas/metacat/restservice/ResourceHandler.java
326 326
                    System.out.println("Using resource 'meta'");
327 327
                    if(params != null && params.get(FUNCTION_KEYWORD) != null &&
328 328
                            params.get(FUNCTION_KEYWORD)[0].equals(FUNCTION_NAME_GENERATE_MISSING_SYSTEM_METADATA))
329
                    { //generate system metadata for any object that is
330
                        //a) not system metadata itself
331
                        //b) does not already have a system metadata id in the systemmetadata table
332
                        //c) not a BIN object (data)
333
                        //TODO: check if we need this anymore.  Might be superceded
334
                        //by MetacatPopulator
335
                        generateMissingSystemMetadata();
329
                    { 
336 330
                        status = true;
337 331
                    }
338 332
                    else
......
1026 1020
            throw new Exception("Could not load the session data: " + e.getClass() + ": " + e.getMessage());
1027 1021
        }
1028 1022
    }
1029
    
1030
    /**
1031
     * generate missing system metadata for any science metadata objects
1032
     * that don't already have it. https://trac.dataone.org/ticket/591
1033
     * 
1034
     * called with POST meta/?op=generatemissingsystemmetadata
1035
     * @throws SQLException 
1036
     * @throws AccessionNumberException 
1037
     * @throws NoSuchAlgorithmException 
1038
     * @throws InvalidRequest 
1039
     * @throws NotImplemented 
1040
     * @throws NotFound 
1041
     * @throws NotAuthorized 
1042
     * @throws InvalidToken 
1043
     * @throws PropertyNotFoundException 
1044
     * @throws McdbDocNotFoundException 
1045
     * @throws ServiceFailure 
1046
     */
1047
    private void generateMissingSystemMetadata() throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException,
1048
    InvalidToken, NotAuthorized, NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException,
1049
    AccessionNumberException, SQLException
1050
    {
1051
        AuthToken token = new AuthToken(sessionId);
1052
        CrudService.getInstance().generateMissingSystemMetadata(token);
1053
    }
1054 1023

  
1055 1024
    /**
1056 1025
     *  Earthgrid API > Identifier Service > isRegistered Function : 
src/edu/ucsb/nceas/metacat/MetacatHandler.java
123 123
import org.jibx.runtime.BindingDirectory;
124 124
import org.jibx.runtime.IBindingFactory;
125 125
import org.jibx.runtime.IMarshallingContext;
126
import org.jibx.runtime.IUnmarshallingContext;
127 126
import org.jibx.runtime.JiBXException;
128 127

  
129 128
import org.xml.sax.SAXException;
......
133 132
import com.oreilly.servlet.multipart.ParamPart;
134 133
import com.oreilly.servlet.multipart.Part;
135 134

  
136
import org.dataone.service.types.AuthToken;
137 135
import org.dataone.service.types.Checksum;
138 136
import org.dataone.service.types.ChecksumAlgorithm;
139 137
import org.dataone.service.types.Identifier;
......
1867 1865
                try {
1868 1866
                  // handle updates
1869 1867
                  identifier.setValue(im.getGUID(newdocid, rev));
1870
                  sysMeta = getSystemMetadata(identifier, user, groups);
1871
                  sysMetaLocalId = insertOrUpdateSystemMetadata(
1872
                    sysMeta, "update", user, groups);
1868
                  sysMeta = getSystemMetadata(identifier);
1869
                  IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(sysMeta);
1873 1870
                  
1874
                } catch ( PropertyNotFoundException pnfe) {
1875
                  logMetacat.error("There was a problem getting the server " +
1876
                    "url properties: " + pnfe.getMessage());
1877
                  
1878 1871
                } catch ( McdbDocNotFoundException mnfe) {
1879 1872
                  
1880 1873
                  // handle inserts
1881 1874
                  try {
1882 1875
                    sysMeta = createSystemMetadata(newdocid, user, groups);
1883
                    sysMetaLocalId = insertOrUpdateSystemMetadata(
1884
                      sysMeta, "insert", user, groups);
1876
                    IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(sysMeta);
1885 1877
                    
1886 1878
                  } catch ( McdbDocNotFoundException dnfe ) {
1887 1879
                    logMetacat.debug(
......
3516 3508
    }
3517 3509

  
3518 3510
    /**
3519
     * Serialize a system metadata doc. Note: This needs to refactored out
3520
     * of MetacatHandler and into a utitlity when stream i/o in Metacat is
3521
     * evaluated.
3522
     *
3523
     * @param sysmeta  The system metadata object to be serialized
3524
     * @return sysMetaOut  The system metadata XML document as a ByteArrayOutputStream
3525
     * @throws JiBXException
3526
     */
3527
    private static ByteArrayOutputStream serializeSystemMetadata(SystemMetadata sysmeta) 
3528
      throws JiBXException {
3529
      
3530
      IBindingFactory bfact;
3531
      ByteArrayOutputStream sysmetaOut = null;
3532
      
3533
      try {
3534
          bfact = BindingDirectory.getFactory(SystemMetadata.class);
3535
          IMarshallingContext mctx = bfact.createMarshallingContext();
3536
          sysmetaOut = new ByteArrayOutputStream();
3537
          mctx.marshalDocument(sysmeta, "UTF-8", null, sysmetaOut);
3538
      } catch (JiBXException jxe) {
3539
        throw jxe;
3540
        
3541
      }
3542
      
3543
      return sysmetaOut;
3544
    }
3545
    
3546
    /**
3547 3511
     * Creates a system metadata object for insertion into metacat
3548 3512
     *
3549 3513
     * @param localId  The local document identifier
......
3570 3534
      SystemMetadata sysMeta = new SystemMetadata();
3571 3535
      InputStream inputStream;
3572 3536
      int rev = im.getLatestRevForLocalId(localId);
3537
      AccessionNumber accNum = new AccessionNumber(localId, "NONE");
3573 3538
      Identifier identifier = new Identifier();
3574 3539
      
3575 3540
      try {
3576 3541
        // get the identifier if it exists
3577
        identifier.setValue(im.getGUID(localId, rev));
3542
        identifier.setValue(im.getGUID(accNum.getDocid(), rev));
3578 3543
        
3579 3544
      } catch ( McdbDocNotFoundException dnfe ) {
3580 3545
        // otherwise create the mapping
......
3583 3548
        logMetacat.debug("No guid in the identifier table.  adding it for " + localId);
3584 3549
        im.createMapping(localId, localId);
3585 3550
        logMetacat.debug("Mapping created for " + localId);
3586
        AccessionNumber accNum = new AccessionNumber(localId, "NONE");
3587
	logMetacat.debug("accessionNumber: " + accNum);
3551
        logMetacat.debug("accessionNumber: " + accNum);
3588 3552
        identifier.setValue(im.getGUID(accNum.getDocid(), rev));
3589 3553
        
3590 3554
      }
......
3692 3656
              try {
3693 3657
                logMetacat.debug("Checking for existing system metadata for " + 
3694 3658
                  dataDocId.getValue());
3695
                dataSysMeta = this.getSystemMetadata(dataDocId, username, groups);
3659
                dataSysMeta = this.getSystemMetadata(dataDocId);
3696 3660
                // add describedBy sysmeta
3697 3661
                logMetacat.debug("Setting describedBy for " + dataDocId.getValue() +
3698 3662
                                 " to " + identifier.getValue());
......
3702 3666
                    f = ObjectFormat.OCTET_STREAM;
3703 3667
                }
3704 3668
                dataSysMeta.setObjectFormat(f);
3705
                dataSysMetaId = insertOrUpdateSystemMetadata(
3706
                  dataSysMeta, "update", username, groups);
3669
                IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(dataSysMeta);
3707 3670
                
3708 3671
              } catch ( McdbDocNotFoundException nf ) {
3709 3672
                // System metadata for data doesn't exist
......
3729 3692
                  
3730 3693
                  logMetacat.debug("Updating system metadata for " + 
3731 3694
                    dataDocId.getValue() + " to " + dataDocMimeType);
3732
                  dataSysMetaId = insertOrUpdateSystemMetadata(
3733
                    dataSysMeta, "insert", username, groups);
3695
                  IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(dataSysMeta);
3734 3696
                  
3735 3697
                } catch ( McdbDocNotFoundException mdnf) {
3736 3698
                  mdnf.printStackTrace();
......
3759 3721
                  
3760 3722
                }
3761 3723

  
3762
              } catch ( InsufficientKarmaException ike ) {
3763
                logMetacat.debug("There was an access problem reading " +
3764
                  localId + ". The error message was: " + ike.getMessage());
3765
                throw ike;
3766

  
3767 3724
              } catch ( McdbException me ) {
3768 3725
                logMetacat.debug("There was a Metacat problem. " +
3769 3726
                  "The error message was: " + me.getMessage());
3770 3727
                throw me;
3771 3728

  
3772
              } catch ( JiBXException jxe ) {
3773
                logMetacat.debug("There was a problem deserializing the system " +
3774
                  "metadata XML.  The error message was: " + jxe.getMessage());
3775
                throw jxe;
3776

  
3777 3729
              } // end try()
3778 3730
              
3779 3731
            } // end if()
......
3838 3790
      return sysMeta;
3839 3791
    }
3840 3792
    
3841
    /*
3842
     * Inserts or updates system metadata documents in Metacat. Note: This
3843
     * needs to be refactored out of MetacatHandler and into a utitlity when 
3844
     * stream i/o in Metacat is evaluated.
3845
     *
3846
     * @param sysMeta The system metadata object to process
3847
     * @param action  The action to perform, insert or update
3848
     * @param user    The username doing the document insert or update
3849
     * @param groups  The groups the user belongs in
3850
     *
3851
     * @return localId  The new docid of the inserted or updated document
3852
     *
3853
     * @throws SQLException
3854
     */
3855
    private String insertOrUpdateSystemMetadata(SystemMetadata sysMeta,
3856
      String action, String user, String[] groups) 
3857
      throws SQLException, InsufficientKarmaException, JiBXException,
3858
      PropertyNotFoundException, McdbException, McdbDocNotFoundException {
3859
      
3860
      logMetacat.debug("MetacatHandler.insertOrUpdateSystemMetadata() called.");
3861
      
3862
      String metacatUrl = "";
3863
      action = action.toUpperCase();
3864
      
3865
      try {
3866
        String server  = PropertyService.getProperty("server.name");
3867
        String port    = PropertyService.getProperty("server.httpPort");
3868
        String context = PropertyService.getProperty("application.context");
3869
        metacatUrl     = "http://" + server + ":" + port + "/" + context;
3870
          
3871
      } catch ( PropertyNotFoundException pnfe ) {
3872
        logMetacat.error("Couldn't get the server url properties: " +
3873
          pnfe.getMessage());
3874
          throw pnfe;
3875
          
3876
      }
3877
      
3878
      DocumentImplWrapper docImpl = new DocumentImplWrapper("", false);
3879
      DBConnection dbConn         = null;
3880
      int serialNumber            = -1;
3881
      String qformat              = "xml";
3882
      String xml                  = null;
3883
      String doc                  = null;
3884
      String localId              = null;
3885
      String newAccNumber         = null;
3886
      String pub                  = null;
3887
      StringReader dtd            = null;
3888
      
3889
      IdentifierManager im = IdentifierManager.getInstance();
3890
      Identifier sysMetaGuid = new Identifier();
3891
      sysMetaGuid.setValue(DocumentUtil.generateDocumentId(1));
3892
      sysMeta.setDateSysMetadataModified(new Date());
3893
      
3894
      // either generate or get a system metadata localId
3895
      try {
3896
        if ( action.equals("INSERT") ) {
3897
          localId = im.generateLocalId(sysMetaGuid.getValue(), 1, true);
3898
        
3899
        } else if ( action.equals("UPDATE") ) {
3900
          localId = im.getSystemMetadataLocalId(sysMetaGuid.getValue());
3901
          String docid = localId.substring(0, localId.lastIndexOf("."));
3902
          String revString = 
3903
            localId.substring(localId.lastIndexOf(".") + 1, localId.length());
3904
          int rev = new Integer(revString).intValue();
3905
          rev++;
3906
          docid = docid + "." + rev;
3907
          localId = docid;
3908
          logMetacat.debug("Incremented system metadata localId: " + localId);
3909
          
3910
        }
3911
        
3912
      } catch ( McdbDocNotFoundException mnfe ) {
3913
        throw mnfe;
3914
        
3915
      }
3916
      
3917
      // get the xml from the sytem metadata object
3918
      try {
3919
          xml = new String(serializeSystemMetadata(sysMeta).toByteArray());
3920
        
3921
      } catch ( JiBXException jxe ) {
3922
        throw jxe;
3923
        
3924
      }
3925
      
3926
      logMetacat.debug("SystemMetadata to insert or update: " + xml);
3927
      
3928
      // ensure user can insert or update      
3929
      try {
3930
        
3931
        if ( !(AuthUtil.canInsertOrUpdate(user, groups)) ) {
3932
          throw new InsufficientKarmaException("The user " + user +
3933
            "does not have permission to insert or update system metadata.");
3934
            
3935
        }
3936
        
3937
      } catch (MetacatUtilException ue) {
3938
        throw new InsufficientKarmaException("Couldn't determine if user " + user +
3939
          "has permission to insert or update system metadata.");
3940
         
3941
      }
3942
      
3943
      try {
3944
        
3945
        // get a database connection from the pool
3946
        dbConn = 
3947
        DBConnectionPool.getDBConnection(
3948
          "MetacatHandler.insertOrUpdateSystemMetadata");
3949
        serialNumber = dbConn.getCheckOutSerialNumber();
3950
        
3951
        // write the document to the database and disk
3952
        logMetacat.debug("MetacatHandler.insertOrUpdateSystemMetadata(): " +
3953
          "Begin writing XML to Metacat for " + action + 
3954
          " operation.");
3955
        
3956
        // write the system metadata document
3957
        newAccNumber = docImpl.write(dbConn, xml, pub, dtd, 
3958
                                     action, localId, user, groups);
3959
                                     
3960
        logMetacat.debug("MetacatHandler.insertOrUpdateSystemMetadata(): " +
3961
          "Wrote XML to Metacat for " + action + 
3962
          " operation.");
3963
        
3964
        if ( action.equals("INSERT") ) {
3965
          IdentifierManager.getInstance().createSystemMetadataMapping(
3966
                  sysMeta, sysMetaGuid.getValue());
3967
          setAccess(metacatUrl, user, newAccNumber, 
3968
                    "public", "4", "allow", "allowFirst");
3969
          
3970
        } else if ( action.equals("UPDATE") ) {
3971
          IdentifierManager.getInstance().updateSystemMetadataMapping(
3972
            sysMeta.getIdentifier().getValue(), localId);
3973
          IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(
3974
            sysMeta.getDateUploaded().getTime(), 
3975
            sysMeta.getRightsHolder().getValue(),
3976
            sysMeta.getChecksum().getValue(), 
3977
            sysMeta.getChecksum().getAlgorithm().name(), 
3978
            sysMeta.getOriginMemberNode().getValue(), 
3979
            sysMeta.getAuthoritativeMemberNode().getValue(), 
3980
            sysMeta.getDateSysMetadataModified().getTime(), 
3981
            sysMeta.getSubmitter().getValue(), 
3982
            sysMeta.getIdentifier().getValue(), 
3983
            sysMeta.getObjectFormat().toString(), 
3984
            sysMeta.getSize());
3985
          
3986
        }
3987
      // unfortunately DocumentImplWrapper only raises a general exception  
3988
      } catch (Exception e ) {
3989
          throw new McdbException(e.getMessage());
3990
          
3991
      } finally {
3992
        // Return db connection
3993
        DBConnectionPool.returnDBConnection(dbConn, serialNumber);
3994
        
3995
      }
3996
      
3997
      return newAccNumber;
3998
    }
3999
    
4000 3793
    /**
4001 3794
     * Get the system metadata for a document with a specified guid
4002 3795
     *
......
4004 3797
     *
4005 3798
     * @return sysMeta  The desired SystemMetadata object
4006 3799
     */
4007
    private SystemMetadata getSystemMetadata(Identifier guid, 
4008
      String user, String[] groups) 
4009
      throws McdbDocNotFoundException, InsufficientKarmaException, 
4010
      JiBXException, ParseLSIDException, PropertyNotFoundException,
4011
      McdbException, SQLException, IOException, ClassNotFoundException {
3800
    private SystemMetadata getSystemMetadata(Identifier guid) 
3801
      throws McdbDocNotFoundException {
4012 3802
      
4013 3803
      logMetacat.debug("MetacatHandler.getSystemMetadata() called.");
4014 3804
      
4015
      SystemMetadata sysMeta = new SystemMetadata();
4016
      InputStream inputStream;
3805
      SystemMetadata sysMeta = null;
4017 3806
      
4018 3807
      try {
4019
        IdentifierManager im = IdentifierManager.getInstance();
4020
        String localId = im.getSystemMetadataLocalId(guid.getValue());
4021
        
4022
        try {
4023
          inputStream = this.read(localId);
4024
          
4025
        } catch ( McdbDocNotFoundException dnfe ) {
4026
          throw dnfe;
4027

  
4028
        } catch ( ParseLSIDException ple ) {
4029
          throw ple;
4030

  
4031
        } catch ( PropertyNotFoundException pnfe ) {
4032
          throw pnfe;
4033

  
4034
        } catch ( McdbException me ) {
4035
          throw me;
4036

  
4037
        } catch ( SQLException sqle ) {
4038
          throw sqle;
4039

  
4040
        } catch ( ClassNotFoundException cnfe ) {
4041
          throw cnfe;
4042

  
4043
        } catch ( IOException ioe ) {
4044
          throw ioe;
4045

  
4046
        } // end try()
4047
        
4048
        try {
4049
          sysMeta = deserializeSystemMetadata(inputStream);
4050
          return sysMeta;
4051
          
4052
        } catch ( JiBXException jxe ) {
4053
          logMetacat.debug("There was a problem deserializing the system " +
4054
            "metadata XML.  The error message was: " + jxe.getMessage());
4055
          throw jxe;
4056
          
4057
        }
3808
        String localId = IdentifierManager.getInstance().getSystemMetadataLocalId(guid.getValue());        
3809
        sysMeta = IdentifierManager.getInstance().getSystemMetadata(localId);
4058 3810
      } catch ( McdbDocNotFoundException dnfe ) {
4059 3811
        logMetacat.debug("There was a problem getting the system " +
4060 3812
        "metadata local id. The error was: " + dnfe.getMessage());
4061 3813
        throw dnfe;
4062
        
4063 3814
      }
3815
      return sysMeta;
3816

  
4064 3817
    }
4065
    
4066
    /**
4067
     * deserialize a system metadata doc. Note: This needs to refactored out
4068
     * of MetacatHandler and into a utitlity when stream i/o in Metacat is
4069
     * evaluated.
4070
     * @param xml  The xml to be deserialized
4071
     * @return sysmeta  The SystemMetadata object to be returned
4072
     * @throws JiBXException
4073
     */
4074
    private static SystemMetadata deserializeSystemMetadata(InputStream xml) 
4075
      throws JiBXException {
4076
      
4077
      try {
4078
          IBindingFactory bfact = 
4079
            BindingDirectory.getFactory(SystemMetadata.class);
4080
          IUnmarshallingContext uctx = bfact.createUnmarshallingContext();
4081
          SystemMetadata sysmeta = 
4082
            (SystemMetadata) uctx.unmarshalDocument(xml, null);
4083
          return sysmeta;
4084
          
4085
      } catch (JiBXException jxe) {
4086
        throw jxe;
4087
        
4088
      }
4089 3818
       
4090
    }
4091
    
4092 3819
    /*
4093 3820
     * Find the size (in bytes) of a stream. Note: This needs to refactored out
4094 3821
     * of MetacatHandler and into a utility when stream i/o in Metacat is
src/edu/ucsb/nceas/metacat/dataone/CrudService.java
22 22
 */
23 23
package edu.ucsb.nceas.metacat.dataone;
24 24

  
25
import java.io.ByteArrayOutputStream;
25
import java.io.ByteArrayInputStream;
26 26
import java.io.File;
27 27
import java.io.FileInputStream;
28 28
import java.io.FileNotFoundException;
......
30 30
import java.io.IOException;
31 31
import java.io.InputStream;
32 32
import java.io.OutputStream;
33

  
34 33
import java.security.NoSuchAlgorithmException;
35

  
36 34
import java.sql.SQLException;
37

  
38 35
import java.text.DateFormat;
39

  
40 36
import java.util.Calendar;
41 37
import java.util.Date;
42 38
import java.util.Enumeration;
......
49 45

  
50 46
import javax.servlet.http.HttpServletRequest;
51 47

  
52
import javax.xml.parsers.ParserConfigurationException;
53
import javax.xml.xpath.XPathExpressionException;
54

  
55 48
import org.apache.commons.io.IOUtils;
56 49
import org.apache.log4j.Logger;
57

  
58
import org.dataone.eml.DataoneEMLParser;
59
import org.dataone.eml.EMLDocument;
60
import org.dataone.eml.EMLDocument.DistributionMetadata;
61

  
62
import org.dataone.service.exceptions.BaseException;
63 50
import org.dataone.service.exceptions.IdentifierNotUnique;
64 51
import org.dataone.service.exceptions.InsufficientResources;
65 52
import org.dataone.service.exceptions.InvalidRequest;
......
70 57
import org.dataone.service.exceptions.NotImplemented;
71 58
import org.dataone.service.exceptions.ServiceFailure;
72 59
import org.dataone.service.exceptions.UnsupportedType;
73

  
74 60
import org.dataone.service.mn.MemberNodeCrud;
75

  
76 61
import org.dataone.service.types.AuthToken;
77 62
import org.dataone.service.types.Checksum;
78 63
import org.dataone.service.types.ChecksumAlgorithm;
......
88 73
import org.dataone.service.types.SystemMetadata;
89 74
import org.dataone.service.types.util.ServiceTypeUtil;
90 75

  
91
import org.jibx.runtime.BindingDirectory;
92
import org.jibx.runtime.IBindingFactory;
93
import org.jibx.runtime.IMarshallingContext;
94
import org.jibx.runtime.IUnmarshallingContext;
95
import org.jibx.runtime.JiBXException;
96

  
97
import org.xml.sax.SAXException;
98

  
99
import edu.ucsb.nceas.metacat.AccessionNumber;
100 76
import edu.ucsb.nceas.metacat.AccessionNumberException;
101 77
import edu.ucsb.nceas.metacat.DocumentImpl;
102 78
import edu.ucsb.nceas.metacat.EventLog;
......
111 87
import edu.ucsb.nceas.metacat.service.SessionService;
112 88
import edu.ucsb.nceas.metacat.util.DocumentUtil;
113 89
import edu.ucsb.nceas.metacat.util.SessionData;
114

  
115 90
import edu.ucsb.nceas.utilities.ParseLSIDException;
116 91
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
117 92

  
......
245 220
        params.put(name, value);
246 221
    }
247 222
    
248
    /**
249
     * Generate SystemMetadata for any object in the object store that does
250
     * not already have it.  SystemMetadata documents themselves, are, of course,
251
     * exempt.  This is a utility method for migration of existing object 
252
     * stores to DataONE where SystemMetadata is required for all objects.  See 
253
     * https://trac.dataone.org/ticket/591
254
     * 
255
     * @param token an authtoken with appropriate permissions to read all 
256
     * documents in the object store.  To work correctly, this should probably
257
     * be an adminstrative credential.
258
     * @throws SQLException 
259
     * @throws AccessionNumberException 
260
     * @throws NoSuchAlgorithmException 
261
     * @throws InvalidRequest 
262
     * @throws NotImplemented 
263
     * @throws NotFound 
264
     * @throws NotAuthorized 
265
     * @throws InvalidToken 
266
     * @throws PropertyNotFoundException 
267
     * @throws McdbDocNotFoundException 
268
     * @throws ServiceFailure 
269
     */
270
    public void generateMissingSystemMetadata(AuthToken token) 
271
    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized, 
272
    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
273
    {
274
        IdentifierManager im = IdentifierManager.getInstance();
275
        //get the list of ids with no SM
276
        List<String> idList = im.getLocalIdsWithNoSystemMetadata();
277
        for (String localId : idList) { 
278
            //for each id, add a system metadata doc
279
            generateMissingSystemMetadata(token, localId);
280
        }
281
        logCrud.info("generateMissingSystemMetadata(token)");
282
    }
283
    
284
    /**
285
     * Generate SystemMetadata for a particular object with identifier localId.
286
     * This is a utility method for migration of existing objects 
287
     * to DataONE where SystemMetadata is required for all objects.
288
     * 
289
     * @param token an authtoken with appropriate permissions to read all 
290
     *        documents in the object store.  To work correctly, this should
291
     *        be an adminstrative credential.
292
     * @param localId the identifier of the object to be processed
293
     * @throws ServiceFailure 
294
     * @throws SQLException 
295
     * @throws AccessionNumberException 
296
     * @throws NoSuchAlgorithmException 
297
     * @throws InvalidRequest 
298
     * @throws NotImplemented 
299
     * @throws NotFound 
300
     * @throws NotAuthorized 
301
     * @throws InvalidToken 
302
     * @throws PropertyNotFoundException 
303
     * @throws McdbDocNotFoundException 
304
     */
305
    public void generateMissingSystemMetadata(AuthToken token, String localId) 
306
    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized,
307
    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
308
    {
309
    	logCrud.debug("CrudService.generateMissingSystemMetadata() called.");
310
        logCrud.debug("Creating SystemMetadata for localId " + localId);
311
        SystemMetadata sm = null;
312 223

  
313
        //generate required system metadata fields from the document
314
        try {
315
        	sm = createSystemMetadata(localId, token);
316
        } catch (IOException e1) {
317
        	e1.printStackTrace();
318
        	ServiceFailure sf = new ServiceFailure("00","IOException in generateMissingSystemMetadata: " +
319
        			e1.getMessage());
320
        	sf.setStackTrace(e1.getStackTrace());
321
        	throw sf;
322
        }
323
        
324
        //insert the systemmetadata object
325
        SessionData sessionData = getSessionData(token);
326
        String smlocalid = insertSystemMetadata(sm, sessionData);
327
        logCrud.debug("setting access on SM doc with localid " + smlocalid);
328
        try {
329
        	handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid, "public", "4", "allow", "allowFirst");
330
        } catch (Exception e) {
331
        	logCrud.debug("Unspecified exception thrown by MetacatHandler.setAccess(): " + e.getMessage());
332
        	logMetacat.error("Could not generate missing systemMetadata: " + e.getMessage());
333
        	logMetacat.error(e.getStackTrace());
334
        	ServiceFailure sf = new ServiceFailure("0","Unspecified exception thrown by MetacatHandler.setAccess(): " + e.getMessage());
335
        	sf.setStackTrace(e.getStackTrace());
336
        	throw sf;
337
        }
338

  
339
        String username = "public";
340
        if (sessionData != null) {
341
        	username = sessionData.getUserName();
342
        }
343
        EventLog.getInstance().log(metacatUrl, username, localId, "generateMissingSystemMetadata");
344

  
345
//        catch (Exception e) { // TODO: Please don't catch Exception -- it masks bad things
346
//            e.printStackTrace();
347
//            logCrud.debug("Exception generating missing system metadata: " + e.getMessage());
348
//            logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
349
//        }
350
        logCrud.info("generateMissingSystemMetadata(token, localId)");
351
    }
352 224
    
225
    
226
    
353 227
    /**
354 228
     * create an object via the crud interface
355 229
     */
......
422 296
        }
423 297

  
424 298
        // For Metadata and Data, insert the system metadata into the object store too
425
        String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);
299
        insertSystemMetadata(sysmeta, sessionData);
426 300
        //get the document info.  add any access params for the sysmeta too
427 301
        //logCrud.debug("looking for access records to add for system " +
428 302
        //    "metadata who's parent doc's  local id is " + localId);
......
447 321
        }
448 322
        catch(Exception e)
449 323
        {
450
            logMetacat.error("Error setting permissions on System Metadata object " + 
451
                    " with id " + sysMetaLocalId + ": " + e.getMessage());
324
            logMetacat.error("Error setting permissions on System Metadata object: " +
325
            		e.getMessage());
452 326
            //TODO: decide if this error should cancel the entire create or
453 327
            //if it should continue with just a logged error.
454 328
        }
......
457 331
        logMetacat.debug("Returning from CrudService.create()");
458 332
        EventLog.getInstance().log(metacatUrl,
459 333
                username, localId, "create");
460
        logCrud.info("create D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId + 
461
                ":D1SYSMETADATA:"+ sysMetaLocalId + ":");
334
        logCrud.info("create D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId);
462 335
        return guid;
463 336
    }
464 337
    
......
487 360
            //change the obsoletes field of the new systemMetadata (sm.new) to point to the id of the old one
488 361
            sysmeta.addObsolete(obsoletedGuid);
489 362
            //insert sm.new
490
            String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);
363
            insertSystemMetadata(sysmeta, sessionData);
491 364
            String localId;
492 365
            
493 366
            boolean isScienceMetadata = isScienceMetadata(sysmeta);
......
510 383
            }
511 384
            EventLog.getInstance().log(metacatUrl,
512 385
                    username, im.getLocalId(guid.getValue()), "update");
513
            logCrud.info("update D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId + 
514
                    ":D1SYSMETADATA:"+ sysMetaLocalId + ":");
386
            logCrud.info("update D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId);
515 387
            return guid;
516 388
        }
517 389

  
......
1170 1042
            {
1171 1043
                checksumAlgorithm = "MD5";
1172 1044
            }
1173
            InputStream docStream = get(token, guid);
1045
            
1174 1046
            String checksum;
1175 1047
            try
1176 1048
            {
1049
            	//InputStream docStream = get(token, guid);
1050
                String localId = IdentifierManager.getInstance().getLocalId(guid.getValue());
1051
                DocumentImpl doc = new DocumentImpl(localId);
1052
                InputStream docStream = new ByteArrayInputStream(doc.getBytes());
1053
                
1177 1054
                checksum = checksum(docStream, checksumAlgorithm);
1178 1055
            }
1179 1056
            catch(Exception e)
......
1333 1210
    /**
1334 1211
     * get the system metadata for a document with a specified guid.
1335 1212
     */
1336
public SystemMetadata getSystemMetadata(AuthToken token, Identifier guid)
1213
    public SystemMetadata getSystemMetadata(AuthToken token, Identifier guid)
1337 1214
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 
1338 1215
            InvalidRequest, NotImplemented {
1339 1216
        
1340
        logMetacat.debug("CrudService.getSystemMetadata - for guid: " + guid.getValue());
1217
    	logMetacat.debug("CrudService.getSystemMetadata - for guid: " + guid.getValue());
1341 1218
        
1342
        // Retrieve the session information from the AuthToken
1343
        // If the session is expired, then the user is 'public'
1344
        final SessionData sessionData = getSessionData(token);
1345
                
1346
        try {
1347
            IdentifierManager im = IdentifierManager.getInstance();
1348
            final String localId = im.getSystemMetadataLocalId(guid.getValue());
1349
            InputStream objectStream;
1350
            
1351
            try {
1352
                String username = "public";
1353
                String[] groupnames = null;
1354
                if(sessionData != null)
1355
                {
1356
                    username = sessionData.getUserName();
1357
                    groupnames = sessionData.getGroupNames();
1358
                }
1359
                
1360
                objectStream = readFromMetacat(localId, username, groupnames);
1361
                
1362
            } catch (PropertyNotFoundException e) {
1363
                e.printStackTrace();
1364
                throw new ServiceFailure("1090", "Property not found while reading system metadata from metacat: " + e.getMessage());
1365
            } catch (ClassNotFoundException e) {
1366
                e.printStackTrace();
1367
                throw new ServiceFailure("1090", "Class not found while reading system metadata from metacat: " + e.getMessage());
1368
            } catch (IOException e) {
1369
                e.printStackTrace();
1370
                throw new ServiceFailure("1090", "IOException while reading system metadata from metacat: " + e.getMessage());
1371
            } catch (SQLException e) {
1372
                e.printStackTrace();
1373
                throw new ServiceFailure("1090", "SQLException while reading system metadata from metacat: " + e.getMessage());
1374
            } catch (McdbException e) {
1375
                e.printStackTrace();
1376
                throw new ServiceFailure("1090", "Metacat DB Exception while reading system metadata from metacat: " + e.getMessage());
1377
            } catch (ParseLSIDException e) {
1378
                e.printStackTrace();
1379
                throw new NotFound("1060", "Error parsing LSID while reading system metadata from metacat: " + e.getMessage());
1380
            } catch (InsufficientKarmaException e) {
1381
                e.printStackTrace();
1382
                throw new NotAuthorized("1040", "User not authorized for get() on system metadata: " + e.getMessage());
1383
            }
1384
                        
1385
            // Deserialize the xml to create a SystemMetadata object
1386
            SystemMetadata sysmeta = deserializeSystemMetadata(objectStream);
1387
            String username = "public";
1388
            if(sessionData != null)
1389
            {
1390
                username = sessionData.getUserName();
1391
            }
1392
            EventLog.getInstance().log(metacatUrl,
1393
                    username, im.getLocalId(guid.getValue()), "read");
1394
            logCrud.info("getsystemmetadata D1GUID:" + guid.getValue()  + 
1395
                    ":D1SYSMETADATA:"+ localId + ":");
1396
            return sysmeta;
1397
            
1398
        } catch (McdbDocNotFoundException e) {
1399
            //e.printStackTrace();
1400
            throw new NotFound("1040", e.getMessage());
1401
        }                
1219
        IdentifierManager im = IdentifierManager.getInstance();
1220
        String localId = null;
1221
        SystemMetadata sysmeta = null;
1222
		try {
1223
			localId = im.getSystemMetadataLocalId(guid.getValue());
1224
	        // look up the SysMeta
1225
	        sysmeta = im.getSystemMetadata(localId);
1226
		} catch (McdbDocNotFoundException e) {
1227
			e.printStackTrace();
1228
			throw new NotFound(null, "Could not locate SystemMetadata for localId: " + localId);
1229
		}        
1230

  
1231
		logCrud.info("getsystemmetadata D1GUID:" + guid.getValue()  + ":D1SYSMETADATA:"+ localId + ":");
1232
        
1233
        return sysmeta;
1234
                   
1402 1235
    }
1403 1236
    
1404 1237
    /**
......
1646 1479
    /**
1647 1480
     * insert a systemMetadata doc, return the localId of the sysmeta
1648 1481
     */
1649
    private String insertSystemMetadata(SystemMetadata sysmeta, SessionData sessionData) 
1482
    private void insertSystemMetadata(SystemMetadata sysmeta, SessionData sessionData) 
1650 1483
        throws ServiceFailure 
1651 1484
    {
1652 1485
        logMetacat.debug("Starting to insert SystemMetadata...");
......
1658 1491
        logCrud.debug("****inserting new system metadata with modified date " + 
1659 1492
                sysmeta.getDateSysMetadataModified());
1660 1493

  
1661
        String xml = new String(serializeSystemMetadata(sysmeta).toByteArray());
1662
        logCrud.debug("sysmeta: " + xml);
1663
        String localId = insertDocument(xml, sysMetaGuid, sessionData, true);
1664
        logCrud.debug("sysmeta inserted with localId " + localId);
1665 1494
        //insert the system metadata doc id into the systemmetadata table to 
1666 1495
        //link it to the data or metadata document
1667 1496
        IdentifierManager.getInstance().createSystemMetadataMapping(
1668 1497
                sysmeta, sysMetaGuid.getValue());
1669
        return localId;
1498
        
1499
        IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(sysmeta);
1500
        
1670 1501
    }
1671 1502
    
1672 1503
    /**
......
1679 1510
        logCrud.debug("CrudService.updateSystemMetadata() called.");
1680 1511
        try
1681 1512
        {
1682
            String smId = IdentifierManager.getInstance().getSystemMetadataLocalId(sm.getIdentifier().getValue());
1683 1513
            logCrud.debug("Setting date modified to " + new Date());
1684 1514
            sm.setDateSysMetadataModified(new Date());
1685
            String xml = new String(serializeSystemMetadata(sm).toByteArray());
1686
            String localId = updateDocument(xml, sm.getIdentifier(), null, sessionData, true);
1687
            IdentifierManager.getInstance().updateSystemMetadataMapping(sm.getIdentifier().getValue(), localId);
1688
            IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(
1689
              sm.getDateUploaded().getTime(), 
1690
              sm.getRightsHolder().getValue(),
1691
              sm.getChecksum().getValue(), 
1692
              /*sm.getChecksum().getAlgorithm().toString()*/sm.getChecksum().getAlgorithm().name(), 
1693
              sm.getOriginMemberNode().getValue(), 
1694
              sm.getAuthoritativeMemberNode().getValue(), 
1695
              sm.getDateSysMetadataModified().getTime(), 
1696
              sm.getSubmitter().getValue(), 
1697
              sm.getIdentifier().getValue(), 
1698
              sm.getObjectFormat().toString(), 
1699
              sm.getSize());
1515

  
1516
            IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(sm);
1700 1517
                        
1701 1518
        }
1702 1519
        catch(Exception e)
......
1866 1683
     * @return
1867 1684
     * @throws ServiceFailure
1868 1685
     */
1869
    public static ByteArrayOutputStream serializeSystemMetadata(SystemMetadata sysmeta) 
1870
        throws ServiceFailure {
1871
        IBindingFactory bfact;
1872
        ByteArrayOutputStream sysmetaOut = null;
1873
        try {
1874
            bfact = BindingDirectory.getFactory(SystemMetadata.class);
1875
            IMarshallingContext mctx = bfact.createMarshallingContext();
1876
            sysmetaOut = new ByteArrayOutputStream();
1877
            mctx.marshalDocument(sysmeta, "UTF-8", null, sysmetaOut);
1878
        } catch (JiBXException e) {
1879
            e.printStackTrace();
1880
            throw new ServiceFailure("1190", "Failed to serialize and insert SystemMetadata: " + e.getMessage());
1881
        }
1882
        
1883
        return sysmetaOut;
1884
    }
1885
    
1686
//    public static ByteArrayOutputStream serializeSystemMetadata(SystemMetadata sysmeta) 
1687
//        throws ServiceFailure {
1688
//        IBindingFactory bfact;
1689
//        ByteArrayOutputStream sysmetaOut = null;
1690
//        try {
1691
//            bfact = BindingDirectory.getFactory(SystemMetadata.class);
1692
//            IMarshallingContext mctx = bfact.createMarshallingContext();
1693
//            sysmetaOut = new ByteArrayOutputStream();
1694
//            mctx.marshalDocument(sysmeta, "UTF-8", null, sysmetaOut);
1695
//        } catch (JiBXException e) {
1696
//            e.printStackTrace();
1697
//            throw new ServiceFailure("1190", "Failed to serialize and insert SystemMetadata: " + e.getMessage());
1698
//        }
1699
//        
1700
//        return sysmetaOut;
1701
//    }
1702
//    
1886 1703
    /**
1887 1704
     * deserialize a system metadata doc
1888 1705
     * @param xml
1889 1706
     * @return
1890 1707
     * @throws ServiceFailure
1891 1708
     */
1892
    public static SystemMetadata deserializeSystemMetadata(InputStream xml) 
1893
        throws ServiceFailure {
1894
        try {
1895
            IBindingFactory bfact = BindingDirectory.getFactory(SystemMetadata.class);
1896
            IUnmarshallingContext uctx = bfact.createUnmarshallingContext();
1897
            SystemMetadata sysmeta = (SystemMetadata) uctx.unmarshalDocument(xml, null);
1898
            return sysmeta;
1899
        } catch (JiBXException e) {
1900
            e.printStackTrace();
1901
            throw new ServiceFailure("1190", "Failed to deserialize and insert SystemMetadata: " + e.getMessage());
1902
        }    
1903
    }
1709
//    public static SystemMetadata deserializeSystemMetadata(InputStream xml) 
1710
//        throws ServiceFailure {
1711
//        try {
1712
//            IBindingFactory bfact = BindingDirectory.getFactory(SystemMetadata.class);
1713
//            IUnmarshallingContext uctx = bfact.createUnmarshallingContext();
1714
//            SystemMetadata sysmeta = (SystemMetadata) uctx.unmarshalDocument(xml, null);
1715
//            return sysmeta;
1716
//        } catch (JiBXException e) {
1717
//            e.printStackTrace();
1718
//            throw new ServiceFailure("1190", "Failed to deserialize and insert SystemMetadata: " + e.getMessage());
1719
//        }    
1720
//    }
1904 1721
    
1905 1722
    /**
1906 1723
     * read a document from metacat and return the InputStream
......
1979 1796
        return ServiceTypeUtil.checksum(is, ChecksumAlgorithm.convert(algorithm)).getValue();
1980 1797
    }
1981 1798
    
1982
    /**
1983
     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into
1984
     * a proper date object
1985
     * @param date
1986
     * @return
1987
     */
1988
    private Date parseMetacatDate(String date)
1989
    {
1990
        String year = date.substring(0, 4);
1991
        String month = date.substring(5, 7);
1992
        String day = date.substring(8, 10);
1993
        Calendar c = Calendar.getInstance(TimeZone.getDefault());
1994
        c.set(new Integer(year).intValue(), 
1995
              new Integer(month).intValue(), 
1996
              new Integer(day).intValue());
1997
        logCrud.debug("time in parseMetacatDate: " + c.getTime());
1998
        return c.getTime();
1999
    }
2000 1799
    
1800
    
2001 1801
    /**
2002 1802
     * find the size (in bytes) of a stream
2003 1803
     * @param is
......
2018 1818
        return size;
2019 1819
    }
2020 1820
    
2021
    /**
2022
     * create system metadata with a specified id, doc and format
2023
     * @throws McdbDocNotFoundException 
2024
     * @throws SQLException
2025
     * @throws AccessionNumberException 
2026
     * @throws NumberFormatException 
2027
     * @throws IOException 
2028
     * @throws NoSuchAlgorithmException 
2029
     * @throws PropertyNotFoundException 
2030
     * @throws NotImplemented 
2031
     * @throws NotFound 
2032
     * @throws NotAuthorized 
2033
     * @throws InvalidToken 
2034
     * @throws InvalidRequest 
2035
     * @throws NoSuchAlgorithmException 
2036
     */
2037
    private SystemMetadata createSystemMetadata(String localId, AuthToken token) 
2038
    throws McdbDocNotFoundException, PropertyNotFoundException, AccessionNumberException, SQLException,
2039
    ServiceFailure, InvalidToken, NotAuthorized, NotFound, NotImplemented, InvalidRequest,
2040
    IOException, NoSuchAlgorithmException
2041
    {     
2042
    	logCrud.debug("CrudService.createSystemMetadata() called.");
2043

  
2044
    	IdentifierManager im = IdentifierManager.getInstance();
2045
    	Hashtable<String, Object> docInfo = im.getDocumentInfo(localId);
2046

  
2047
    	//get the document text
2048
    	int rev = im.getLatestRevForLocalId(localId);
2049
    	Identifier identifier = new Identifier();
2050
    	try {
2051
    		identifier.setValue(im.getGUID(localId, rev));
2052

  
2053
    	} catch (McdbDocNotFoundException mcdbe) { 
2054
    		//we're creating a new SM doc for a doc that is not in the identifier table                                       
2055
    		//so we need to add it to
2056
    		logCrud.debug("No guid in the identifier table.  adding it for " + localId);
2057
    		im.createMapping(localId, localId);
2058
    		logCrud.debug("Mapping created for " + localId);
2059
    		AccessionNumber accNum = new AccessionNumber(localId, "NONE");
2060
    		identifier.setValue(im.getGUID(accNum.getDocid(), rev));
2061
    	}
2062

  
2063
    	logCrud.debug("Creating system metadata for guid " + identifier.getValue());
2064
    	InputStream is = this.get(token, identifier);
2065
    	SystemMetadata sm = new SystemMetadata();
2066

  
2067
    	//set the id
2068
    	sm.setIdentifier(identifier);
2069

  
2070
    	//set the default object format
2071
    	String doctype = (String) docInfo.get("doctype");
2072
    	ObjectFormat format = ObjectFormat.convert(doctype);
2073
    	if (format == null) {
2074
    		if (doctype.trim().equals("BIN")) {
2075
    			format = ObjectFormat.OCTET_STREAM;
2076
    		} else {
2077
    			format = ObjectFormat.convert("text/plain");
2078
    		}
2079
    	}
2080
    	sm.setObjectFormat(format);
2081
    	logCrud.debug("The ObjectFormat for " + localId + " is " + format.toString());
2082

  
2083
    	// further parse EML documents to get data object format,
2084
    	// describes and describedBy information
2085
    	if ( format == ObjectFormat.EML_2_0_0 ||
2086
    			format == ObjectFormat.EML_2_0_1 ||
2087
    			format == ObjectFormat.EML_2_1_0 ) {
2088

  
2089
    		try {
2090
    			DataoneEMLParser emlParser = DataoneEMLParser.getInstance();
2091
    			EMLDocument emlDocument = emlParser.parseDocument(is);
2092

  
2093
    			// iterate through the data objects in the EML doc and add sysmeta
2094
    			logCrud.debug("The number of data entities is: " +
2095
    					emlDocument.distributionMetadata.size());
2096

  
2097
    			for( int j = 0; j < emlDocument.distributionMetadata.size(); j++ ) {
2098

  
2099
    				DistributionMetadata distMetadata = 
2100
    					emlDocument.distributionMetadata.elementAt(j);
2101
    				String dataDocUrl = distMetadata.url;
2102
    				String dataDocMimeType = distMetadata.mimeType;
2103
    				String dataDocLocalId = "";
2104
    				logCrud.debug("\tData local ID: " + dataDocLocalId);
2105
    				logCrud.debug("\tData URL: " + dataDocUrl);
2106
    				logCrud.debug("\tData mime: " + dataDocMimeType);
2107

  
2108
    				//we only handle ecogrid urls right now
2109
    				if ( dataDocUrl.trim().startsWith("ecogrid://knb/") ) {
2110
    					dataDocLocalId = 
2111
    						dataDocUrl.substring(dataDocUrl.indexOf("ecogrid://knb/") + 
2112
    								"ecogrid://knb/".length(), dataDocUrl.length());
2113

  
2114
    					//set the id
2115
    					Identifier dataDocId = new Identifier();
2116
    					dataDocId.setValue(dataDocLocalId);
2117

  
2118
    					// add describes into EML system metadata
2119
    					sm.addDescribe(dataDocId);
2120

  
2121
    					SystemMetadata dataSysMeta = new SystemMetadata();
2122
    					// check if data system metadata exists
2123
    					try {
2124
    						logCrud.debug("Checking for existing system metadata for " + dataDocId.getValue());
2125
    						dataSysMeta = this.getSystemMetadata(token, dataDocId);
2126
    						// add describedBy sysmeta
2127
    						logCrud.debug("Setting describedBy for " + dataDocId.getValue() +
2128
    								" to " + identifier.getValue());
2129
    						dataSysMeta.addDescribedBy(identifier);
2130
    						dataSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));
2131
    						this.updateSystemMetadata(dataSysMeta, getSessionData(token));
2132

  
2133
    					} catch ( NotFound nf ) {
2134
    						// System metadata for data doesn't exist
2135
    						logCrud.debug("There was not an existing system metadata " + "document for " + dataDocId.getValue());
2136
    						logCrud.debug("Creating a system metadata " + "document for " + dataDocId.getValue());
2137
    						dataSysMeta = this.createSystemMetadata(dataDocLocalId, token);
2138

  
2139
    						logCrud.debug("Setting describedBy for " + dataDocId.getValue() + " to " + identifier.getValue());
2140
    						dataSysMeta.addDescribedBy(identifier);
2141

  
2142
    						logCrud.debug("Setting mimeType for " + dataDocId.getValue() + " to " + dataDocMimeType);
2143
    						dataSysMeta.setObjectFormat(ObjectFormat.convert(dataDocMimeType));
2144

  
2145
    						logCrud.debug("Updating system metadata for " + dataDocId.getValue() + " to " + dataDocMimeType);
2146
    						this.updateSystemMetadata(dataSysMeta, getSessionData(token));
2147
    					}
2148
    				} // end if()
2149
    			} // end for()
2150

  
2151
    		} catch ( ParserConfigurationException pce ) {
2152
    			logCrud.debug("There was a problem parsing the EML document. " +
2153
    					"The error message was: " + pce.getMessage());
2154

  
2155
    		} catch ( SAXException saxe ) {
2156
    			logCrud.debug("There was a problem traversing the EML document. " +
2157
    					"The error message was: " + saxe.getMessage());
2158

  
2159
    		} catch ( XPathExpressionException xpee ) {
2160
    			logCrud.debug("There was a problem searching the EML document. " +
2161
    					"The error message was: " + xpee.getMessage());
2162
    		} // end try
2163

  
2164
    	} // end if()
2165

  
2166
    	//create the checksum
2167
    	is = this.get(token, identifier);
2168
    	String checksumS = checksum(is);
2169
    	ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");
2170
    	Checksum checksum = new Checksum();
2171
    	checksum.setValue(checksumS);
2172
    	checksum.setAlgorithm(ca);
2173
    	sm.setChecksum(checksum);
2174

  
2175
    	//set the size
2176
    	is = this.get(token, identifier);
2177
    	sm.setSize(sizeOfStream(is));
2178

  
2179
    	//submitter
2180
    	Subject p = new Subject();
2181
    	p.setValue((String) docInfo.get("user_owner"));
2182
    	sm.setSubmitter(p);
2183
    	sm.setRightsHolder(p);
2184
    	try {
2185
    		Date dateCreated = parseMetacatDate((String) docInfo.get("date_created"));
2186
    		sm.setDateUploaded(dateCreated);
2187
    		Date dateUpdated = parseMetacatDate((String) docInfo.get("date_updated"));
2188
    		sm.setDateSysMetadataModified(dateUpdated);
2189
    	} catch (Exception e) {
2190
    		logCrud.debug("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
2191
    		Date dateCreated = new Date();
2192
    		sm.setDateUploaded(dateCreated);
2193
    		Date dateUpdated = new Date();
2194
    		sm.setDateSysMetadataModified(dateUpdated);
2195
    	}
2196
    	NodeReference nr = new NodeReference();
2197
    	nr.setValue(PropertyService.getProperty("dataone.memberNodeId"));
2198
    	sm.setOriginMemberNode(nr);
2199
    	sm.setAuthoritativeMemberNode(nr);
2200

  
2201
    	// TODO: Need to set describes/describedBy
2202

  
2203
    	return sm;
2204
    }
2205 1821
    
1822
    
2206 1823
    /**
2207 1824
     * create the listObjects pathQuery document
2208 1825
     */
src/edu/ucsb/nceas/metacat/admin/upgrade/GenerateSystemMetadata.java
1
package edu.ucsb.nceas.metacat.admin.upgrade;
2
/**
3
 *  '$RCSfile$'
4
 *    Purpose: A Class for upgrading the database to version 1.5
5
 *  Copyright: 2000 Regents of the University of California and the
6
 *             National Center for Ecological Analysis and Synthesis
7
 *    Authors: Saurabh Garg
8
 *
9
 *   '$Author: leinfelder $'
10
 *     '$Date: 2011-03-29 18:23:38 +0000 (Tue, 29 Mar 2011) $'
11
 * '$Revision: 6025 $'
12
 *
13
 * This program is free software; you can redistribute it and/or modify
14
 * it under the terms of the GNU General Public License as published by
15
 * the Free Software Foundation; either version 2 of the License, or
16
 * (at your option) any later version.
17
 *
18
 * This program is distributed in the hope that it will be useful,
19
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21
 * GNU General Public License for more details.
22
 *
23
 * You should have received a copy of the GNU General Public License
24
 * along with this program; if not, write to the Free Software
25
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26
 */
27

  
28

  
29
import java.io.ByteArrayInputStream;
30
import java.io.IOException;
31
import java.io.InputStream;
32
import java.security.NoSuchAlgorithmException;
33
import java.sql.SQLException;
34
import java.util.Calendar;
35
import java.util.Date;
36
import java.util.Hashtable;
37
import java.util.List;
38
import java.util.TimeZone;
39
import java.util.Timer;
40

  
41
import javax.xml.parsers.ParserConfigurationException;
42
import javax.xml.xpath.XPathExpressionException;
43

  
44
import org.apache.commons.logging.Log;
45
import org.apache.commons.logging.LogFactory;
46
import org.dataone.eml.DataoneEMLParser;
47
import org.dataone.eml.EMLDocument;
48
import org.dataone.eml.EMLDocument.DistributionMetadata;
49
import org.dataone.service.exceptions.InvalidRequest;
50
import org.dataone.service.exceptions.InvalidToken;
51
import org.dataone.service.exceptions.NotAuthorized;
52
import org.dataone.service.exceptions.NotFound;
53
import org.dataone.service.exceptions.NotImplemented;
54
import org.dataone.service.exceptions.ServiceFailure;
55
import org.dataone.service.types.AuthToken;
56
import org.dataone.service.types.Checksum;
57
import org.dataone.service.types.ChecksumAlgorithm;
58
import org.dataone.service.types.Identifier;
59
import org.dataone.service.types.NodeReference;
60
import org.dataone.service.types.ObjectFormat;
61
import org.dataone.service.types.Subject;
62
import org.dataone.service.types.SystemMetadata;
63
import org.xml.sax.SAXException;
64

  
65
import edu.ucsb.nceas.metacat.AccessionNumber;
66
import edu.ucsb.nceas.metacat.AccessionNumberException;
67
import edu.ucsb.nceas.metacat.DocumentImpl;
68
import edu.ucsb.nceas.metacat.IdentifierManager;
69
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
70
import edu.ucsb.nceas.metacat.McdbException;
71
import edu.ucsb.nceas.metacat.MetacatHandler;
72
import edu.ucsb.nceas.metacat.admin.AdminException;
73
import edu.ucsb.nceas.metacat.dataone.CrudService;
74
import edu.ucsb.nceas.metacat.properties.PropertyService;
75
import edu.ucsb.nceas.metacat.util.SessionData;
76
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
77
import edu.ucsb.nceas.utilities.SortedProperties;
78

  
79
public class GenerateSystemMetadata implements UpgradeUtilityInterface {
80

  
81
	private static Log log = LogFactory.getLog(GenerateSystemMetadata.class);
82
	
83
	private MetacatHandler handler = new MetacatHandler(new Timer());
84
	
85
    public boolean upgrade() throws AdminException {
86
        boolean success = true;
87
        
88
        try {
89
			generateMissingSystemMetadata(null);
90
		} catch (Exception e) {
91
			String msg = "Problem generating missing system metadata: " + e.getMessage();
92
			log.error(msg, e);
93
			success = false;
94
			throw new AdminException(msg);
95
		}
96
    	return success;
97
    }
98
    
99
    /**
100
     * Generate SystemMetadata for any object in the object store that does
101
     * not already have it.  SystemMetadata documents themselves, are, of course,
102
     * exempt.  This is a utility method for migration of existing object 
103
     * stores to DataONE where SystemMetadata is required for all objects.  See 
104
     * https://trac.dataone.org/ticket/591
105
     * 
106
     * @param token an authtoken with appropriate permissions to read all 
107
     * documents in the object store.  To work correctly, this should probably
108
     * be an adminstrative credential.
109
     * @throws SQLException 
110
     * @throws AccessionNumberException 
111
     * @throws NoSuchAlgorithmException 
112
     * @throws InvalidRequest 
113
     * @throws NotImplemented 
114
     * @throws NotFound 
115
     * @throws NotAuthorized 
116
     * @throws InvalidToken 
117
     * @throws PropertyNotFoundException 
118
     * @throws McdbDocNotFoundException 
119
     * @throws ServiceFailure 
120
     */
121
    public void generateMissingSystemMetadata(AuthToken token) 
122
    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized, 
123
    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
124
    {
125
        IdentifierManager im = IdentifierManager.getInstance();
126
        //get the list of ids with no SM
127
        List<String> idList = im.getLocalIdsWithNoSystemMetadata();
128
        for (String localId : idList) { 
129
            //for each id, add a system metadata doc
130
        	try {
131
        		generateMissingSystemMetadata(token, localId);
132
        	} catch (Exception e) {
133
				log.error("Error generating system metadata for: " + localId, e);
134
			}
135
        }
136
        log.info("generateMissingSystemMetadata(token)");
137
    }
138
    
139
    /**
140
     * Generate SystemMetadata for a particular object with identifier localId.
141
     * This is a utility method for migration of existing objects 
142
     * to DataONE where SystemMetadata is required for all objects.
143
     * 
144
     * @param token an authtoken with appropriate permissions to read all 
145
     *        documents in the object store.  To work correctly, this should
146
     *        be an adminstrative credential.
147
     * @param localId the identifier of the object to be processed
148
     * @throws ServiceFailure 
149
     * @throws SQLException 
150
     * @throws AccessionNumberException 
151
     * @throws NoSuchAlgorithmException 
152
     * @throws InvalidRequest 
153
     * @throws NotImplemented 
154
     * @throws NotFound 
155
     * @throws NotAuthorized 
156
     * @throws InvalidToken 
157
     * @throws PropertyNotFoundException 
158
     * @throws McdbDocNotFoundException 
159
     */
160
    public void generateMissingSystemMetadata(AuthToken token, String localId) 
161
    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized,
162
    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
163
    {
164
    	log.debug("CrudService.generateMissingSystemMetadata() called.");
165
    	log.debug("Creating SystemMetadata for localId " + localId);
166
        SystemMetadata sm = null;
167

  
168
        //generate required system metadata fields from the document
169
        try {
170
        	sm = handler.createSystemMetadata(localId, null, null);
171
        } catch (Exception e1) {
172
        	e1.printStackTrace();
173
        	ServiceFailure sf = new ServiceFailure("00","Exception in generateMissingSystemMetadata: " +
174
        			e1.getMessage());
175
        	sf.setStackTrace(e1.getStackTrace());
176
        	throw sf;
177
        }
178
        
179
        //insert the systemmetadata object
180
        String guid = null;
181
        try {
182
        	IdentifierManager.getInstance().getLocalId(sm.getIdentifier().getValue());
183
        } catch (Exception e) {
184
			guid = null;
185
		}
186
        if (guid == null) {
187
        	IdentifierManager.getInstance().createSystemMetadataMapping(sm, localId);
188
        }
189
        IdentifierManager.getInstance().insertAdditionalSystemMetadataFields(sm);
190

  
191
        log.info("generateMissingSystemMetadata(token, localId)");
192
    }
193
    
194
    
195
    public static void main(String [] ags){
196

  
197
        try {
198
        	// set up the properties based on the test/deployed configuration of the workspace
199
        	SortedProperties testProperties = 
200
				new SortedProperties("test/test.properties");
201
			testProperties.load();
202
			String metacatContextDir = testProperties.getProperty("metacat.contextDir");
203
			PropertyService.getInstance(metacatContextDir + "/WEB-INF");
204
			// now run it
205
            GenerateSystemMetadata upgrader = new GenerateSystemMetadata();
206
	        upgrader.upgrade();
207
	        
208
        } catch (Exception ex) {
209
            System.out.println("Exception:" + ex.getMessage());
210
            ex.printStackTrace();
211
        }
212
    }
213
}
src/edu/ucsb/nceas/metacat/admin/upgrade/UpgradeNodeDataDatetime.java
1
package edu.ucsb.nceas.metacat.admin.upgrade;
2
/**
3
 *  '$RCSfile$'
4
 *    Purpose: A Class for upgrading the database to version 1.5
5
 *  Copyright: 2000 Regents of the University of California and the
6
 *             National Center for Ecological Analysis and Synthesis
7
 *    Authors: Saurabh Garg
8
 *
9
 *   '$Author: leinfelder $'
10
 *     '$Date: 2011-03-29 18:23:38 +0000 (Tue, 29 Mar 2011) $'
11
 * '$Revision: 6025 $'
12
 *
13
 * This program is free software; you can redistribute it and/or modify
14
 * it under the terms of the GNU General Public License as published by
15
 * the Free Software Foundation; either version 2 of the License, or
16
 * (at your option) any later version.
17
 *
18
 * This program is distributed in the hope that it will be useful,
19
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21
 * GNU General Public License for more details.
22
 *
23
 * You should have received a copy of the GNU General Public License
24
 * along with this program; if not, write to the Free Software
25
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26
 */
27

  
28

  
29
import java.sql.Connection;
30
import java.sql.Driver;
31
import java.sql.ResultSet;
32
import java.sql.PreparedStatement;
33
import java.sql.SQLException;
34
import java.sql.Statement;
35
import java.sql.DriverManager;
36
import java.sql.Timestamp;
37
import java.util.Calendar;
38

  
39
import javax.xml.bind.DatatypeConverter;
40

  
41
import edu.ucsb.nceas.metacat.admin.AdminException;
42
import edu.ucsb.nceas.metacat.properties.PropertyService;
43
import edu.ucsb.nceas.utilities.SortedProperties;
44

  
45
public class UpgradeNodeDataDatetime implements UpgradeUtilityInterface {
46

  
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff