Project

General

Profile

« Previous | Next » 

Revision 5920

Added by Matt Jones over 13 years ago

Updated to most recent DataONE libraries. Updated CrudService to set the correct origin MN and auth MN in system metadata. Refactored exception passing. More work to come in generating SystemMetadata.

View differences:

src/edu/ucsb/nceas/metacat/dataone/CrudService.java
30 30
import java.io.IOException;
31 31
import java.io.InputStream;
32 32
import java.io.OutputStream;
33
import java.io.PrintWriter;
34
import java.io.StringBufferInputStream;
35
import java.security.MessageDigest;
33
import java.security.NoSuchAlgorithmException;
36 34
import java.sql.SQLException;
37
import java.util.*;
38 35
import java.text.DateFormat;
39 36
import java.text.SimpleDateFormat;
37
import java.util.Calendar;
38
import java.util.Date;
39
import java.util.Enumeration;
40
import java.util.Hashtable;
41
import java.util.List;
42
import java.util.TimeZone;
43
import java.util.Timer;
44
import java.util.TimerTask;
45
import java.util.Vector;
40 46

  
41
import javax.servlet.ServletContext;
42 47
import javax.servlet.http.HttpServletRequest;
43
import javax.servlet.http.HttpServletResponse;
44 48

  
45 49
import org.apache.commons.io.IOUtils;
46 50
import org.apache.log4j.Logger;
51
import org.dataone.service.exceptions.BaseException;
47 52
import org.dataone.service.exceptions.IdentifierNotUnique;
48 53
import org.dataone.service.exceptions.InsufficientResources;
49 54
import org.dataone.service.exceptions.InvalidRequest;
......
55 60
import org.dataone.service.exceptions.ServiceFailure;
56 61
import org.dataone.service.exceptions.UnsupportedType;
57 62
import org.dataone.service.mn.MemberNodeCrud;
58
import org.dataone.service.types.*;
63
import org.dataone.service.types.AuthToken;
64
import org.dataone.service.types.Checksum;
65
import org.dataone.service.types.ChecksumAlgorithm;
66
import org.dataone.service.types.DescribeResponse;
67
import org.dataone.service.types.Event;
68
import org.dataone.service.types.Identifier;
69
import org.dataone.service.types.Log;
70
import org.dataone.service.types.LogEntry;
71
import org.dataone.service.types.NodeReference;
72
import org.dataone.service.types.ObjectFormat;
73
import org.dataone.service.types.ObjectList;
74
import org.dataone.service.types.Principal;
75
import org.dataone.service.types.SystemMetadata;
76
import org.dataone.service.types.util.ServiceTypeUtil;
59 77
import org.jibx.runtime.BindingDirectory;
60 78
import org.jibx.runtime.IBindingFactory;
61 79
import org.jibx.runtime.IMarshallingContext;
62 80
import org.jibx.runtime.IUnmarshallingContext;
63 81
import org.jibx.runtime.JiBXException;
64 82

  
65
import org.dataone.service.types.Identifier;
66
import org.dataone.service.types.util.ServiceTypeUtil;
67

  
68
import com.gc.iotools.stream.is.InputStreamFromOutputStream;
69

  
70 83
import edu.ucsb.nceas.metacat.AccessionNumber;
71 84
import edu.ucsb.nceas.metacat.AccessionNumberException;
72
import edu.ucsb.nceas.metacat.MetacatResultSet;
73
import edu.ucsb.nceas.metacat.MetacatResultSet.Document;
74
import edu.ucsb.nceas.metacat.DBQuery;
75 85
import edu.ucsb.nceas.metacat.DocumentImpl;
76 86
import edu.ucsb.nceas.metacat.EventLog;
77 87
import edu.ucsb.nceas.metacat.IdentifierManager;
......
230 240
     * documents in the object store.  To work correctly, this should probably
231 241
     * be an adminstrative credential.
232 242
     */
233
    public void generateMissingSystemMetadata(AuthToken token)
234
    {
243
    public void generateMissingSystemMetadata(AuthToken token) {
235 244
        IdentifierManager im = IdentifierManager.getInstance();
236 245
        //get the list of ids with no SM
237
        List<String> l = im.getLocalIdsWithNoSystemMetadata();
238
        for(int i=0; i<l.size(); i++)
239
        { //for each id, add a system metadata doc
240
            String localId = l.get(i);
241
            System.out.println("Creating SystemMetadata for localId " + localId);
242
            //get the document
243
            try
244
            {
245
                //generate required system metadata fields from the document
246
                SystemMetadata sm = createSystemMetadata(localId, token);
247
                //insert the systemmetadata object
248
                SessionData sessionData = getSessionData(token);
249
                String smlocalid = insertSystemMetadata(sm, sessionData);
250
                System.out.println("setting access on SM doc with localid " + smlocalid);
251
                handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid,
252
                        "public", "4", "allow", "allowFirst");
253
                
254
                String username = "public";
255
                if(sessionData != null)
256
                {
257
                    username = sessionData.getUserName();
258
                }
259
                EventLog.getInstance().log(metacatUrl,
260
                        username, localId, "generateMissingSystemMetadata");
246
        List<String> idList = im.getLocalIdsWithNoSystemMetadata();
247
        for (String localId : idList) { 
248
            //for each id, add a system metadata doc
249
            generateMissingSystemMetadata(token, localId);
250
        }
251
        logCrud.info("generateMissingSystemMetadata(token)");
252
    }
253
    
254
    /**
255
     * Generate SystemMetadata for a particular object with identifier localId.
256
     * This is a utility method for migration of existing objects 
257
     * to DataONE where SystemMetadata is required for all objects.
258
     * 
259
     * @param token an authtoken with appropriate permissions to read all 
260
     *        documents in the object store.  To work correctly, this should
261
     *        be an adminstrative credential.
262
     * @param localId the identifier of the object to be processed
263
     */
264
    public void generateMissingSystemMetadata(AuthToken token, String localId) {
265
        System.out.println("Creating SystemMetadata for localId " + localId);
266
        try {
267
            //generate required system metadata fields from the document
268
            SystemMetadata sm = createSystemMetadata(localId, token);
269
            
270
            //insert the systemmetadata object
271
            SessionData sessionData = getSessionData(token);
272
            String smlocalid = insertSystemMetadata(sm, sessionData);
273
            System.out.println("setting access on SM doc with localid " + smlocalid);
274
            handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid, "public", "4", "allow", "allowFirst");
275

  
276
            String username = "public";
277
            if (sessionData != null) {
278
                username = sessionData.getUserName();
261 279
            }
262
            catch(Exception e)
263
            {
264
                e.printStackTrace();
265
                System.out.println("Exception generating missing system metadata: " + e.getMessage());
266
                logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
267
            }
280
            EventLog.getInstance().log(metacatUrl, username, localId, "generateMissingSystemMetadata");
281
        } catch (Exception e) { // TODO: Please don't catch Exception -- it masks bad things
282
            e.printStackTrace();
283
            System.out.println("Exception generating missing system metadata: " + e.getMessage());
284
            logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
268 285
        }
269
        logCrud.info("generateMissingSystemMetadata");
286
        logCrud.info("generateMissingSystemMetadata(token, localId)");
270 287
    }
271 288
    
272 289
    /**
......
1847 1864
     * return an MD5 checksum for the stream
1848 1865
     * @param is
1849 1866
     * @return
1867
     * @throws IOException 
1868
     * @throws NoSuchAlgorithmException 
1850 1869
     */
1851
    public static String checksum(InputStream is)
1852
        throws Exception
1870
    public static String checksum(InputStream is) throws NoSuchAlgorithmException, IOException
1853 1871
    {
1854 1872
        return checksum(is, "MD5");
1855 1873
    }
1856 1874
    
1857 1875
    /**
1858 1876
     * produce a checksum for item using the given algorithm
1877
     * @throws IOException 
1878
     * @throws NoSuchAlgorithmException 
1859 1879
     */
1860
    public static String checksum(InputStream is, String algorithm)
1861
      throws Exception
1880
    public static String checksum(InputStream is, String algorithm) throws NoSuchAlgorithmException, IOException
1862 1881
    {        
1863 1882
        return ServiceTypeUtil.checksum(is, ChecksumAlgorithm.convert(algorithm)).getValue();
1864 1883
    }
......
1904 1923
    
1905 1924
    /**
1906 1925
     * create system metadata with a specified id, doc and format
1926
     * @throws McdbDocNotFoundException 
1927
     * @throws SQLException
1928
     * @throws AccessionNumberException 
1929
     * @throws NumberFormatException 
1930
     * @throws IOException 
1931
     * @throws NoSuchAlgorithmException 
1932
     * @throws PropertyNotFoundException 
1907 1933
     */
1908
    private SystemMetadata createSystemMetadata(String localId, AuthToken token)
1909
      throws Exception
1910
    {
1934
    private SystemMetadata createSystemMetadata(String localId, AuthToken token) 
1935
        throws McdbDocNotFoundException, NumberFormatException, AccessionNumberException, 
1936
        SQLException, NoSuchAlgorithmException, IOException, PropertyNotFoundException, BaseException {
1937
        
1911 1938
        IdentifierManager im = IdentifierManager.getInstance();
1912 1939
        Hashtable<String, Object> docInfo = im.getDocumentInfo(localId);
1913
        
1940

  
1914 1941
        //get the document text
1915 1942
        int rev = im.getLatestRevForLocalId(localId);
1916 1943
        Identifier identifier = new Identifier();
1917
        try
1918
        {
1944
        try {
1919 1945
            identifier.setValue(im.getGUID(localId, rev));
1920
        }
1921
        catch(McdbDocNotFoundException mcdbe)
1922
        { //we're creating a new SM doc for a doc that is not in the identifier table
1923
          //so we need to add it to
1946
        } catch (McdbDocNotFoundException mcdbe) { 
1947
            //we're creating a new SM doc for a doc that is not in the identifier table                                       
1948
            //so we need to add it to
1924 1949
            System.out.println("No guid in the identifier table.  adding it for " + localId);
1925 1950
            im.createMapping(localId, localId);
1926 1951
            System.out.println("mapping created for " + localId);
1927 1952
            AccessionNumber accNum = new AccessionNumber(localId, "NONE");
1928 1953
            identifier.setValue(im.getGUID(accNum.getDocid(), rev));
1929 1954
        }
1930
        
1955

  
1931 1956
        System.out.println("creating system metadata for guid " + identifier.getValue());
1932 1957
        InputStream is = this.get(token, identifier);
1958
        SystemMetadata sm = new SystemMetadata();
1933 1959
        
1934
        SystemMetadata sm = new SystemMetadata();
1935 1960
        //set the id
1936 1961
        sm.setIdentifier(identifier);
1937
        
1962

  
1938 1963
        //set the object format
1939
        String doctype = (String)docInfo.get("doctype");
1940
        ObjectFormat format = ObjectFormat.convert((String)docInfo.get("doctype"));
1941
        if(format == null)
1942
        {
1943
            if(doctype.trim().equals("BIN"))
1944
            {
1964
        String doctype = (String) docInfo.get("doctype");
1965
        ObjectFormat format = ObjectFormat.convert((String) docInfo.get("doctype"));
1966
        if (format == null) {
1967
            if (doctype.trim().equals("BIN")) {
1945 1968
                format = ObjectFormat.OCTET_STREAM;
1946
            }
1947
            else
1948
            {
1969
            } else {
1949 1970
                format = ObjectFormat.convert("text/plain");
1950 1971
            }
1951 1972
        }
1952 1973
        sm.setObjectFormat(format);
1953
        
1974

  
1954 1975
        //create the checksum
1955 1976
        String checksumS = checksum(is);
1956 1977
        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");
......
1958 1979
        checksum.setValue(checksumS);
1959 1980
        checksum.setAlgorithm(ca);
1960 1981
        sm.setChecksum(checksum);
1961
        
1982

  
1962 1983
        //set the size
1963 1984
        is = this.get(token, identifier);
1964 1985
        sm.setSize(sizeOfStream(is));
1965
        
1986

  
1966 1987
        //submitter
1967 1988
        Principal p = new Principal();
1968
        p.setValue((String)docInfo.get("user_owner"));
1989
        p.setValue((String) docInfo.get("user_owner"));
1969 1990
        sm.setSubmitter(p);
1970 1991
        sm.setRightsHolder(p);
1971
        try
1972
        {
1973
            Date dateCreated = parseMetacatDate((String)docInfo.get("date_created"));
1992
        try {
1993
            Date dateCreated = parseMetacatDate((String) docInfo.get("date_created"));
1974 1994
            sm.setDateUploaded(dateCreated);
1975
            Date dateUpdated = parseMetacatDate((String)docInfo.get("date_updated"));
1995
            Date dateUpdated = parseMetacatDate((String) docInfo.get("date_updated"));
1976 1996
            sm.setDateSysMetadataModified(dateUpdated);
1977
        }
1978
        catch(Exception e)
1979
        {
1997
        } catch (Exception e) {
1980 1998
            System.out.println("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
1981 1999
            Date dateCreated = new Date();
1982 2000
            sm.setDateUploaded(dateCreated);
......
1984 2002
            sm.setDateSysMetadataModified(dateUpdated);
1985 2003
        }
1986 2004
        NodeReference nr = new NodeReference();
1987
        //TODO: this should be set to be something more meaningful once the registry is up
1988
        nr.setValue("metacat");
2005
        nr.setValue(PropertyService.getProperty("dataone.memberNodeId"));
1989 2006
        sm.setOriginMemberNode(nr);
1990 2007
        sm.setAuthoritativeMemberNode(nr);
2008
        
2009
        // TODO: Need to set describes/describedBy
2010
        
1991 2011
        return sm;
1992 2012
    }
1993 2013
    

Also available in: Unified diff