Project

General

Profile

« Previous | Next » 

Revision 6852

download remote data and save locally when it is referenced by an EML package, then include it in the ORE map.
http://bugzilla.ecoinformatics.org/show_bug.cgi?id=5522

View differences:

src/edu/ucsb/nceas/metacat/MetacatHandler.java
1849 1849
                  // handle inserts
1850 1850
                  try {
1851 1851
                	// create the system metadata  
1852
                    sysMeta = SystemMetadataFactory.createSystemMetadata(newdocid, false);
1852
                    sysMeta = SystemMetadataFactory.createSystemMetadata(newdocid, false, false);
1853 1853
                    
1854 1854
                    // save it to the map
1855 1855
                    HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);
......
2992 2992
                                    username, groupnames);
2993 2993
                            
2994 2994
                            // generate system metadata about the doc
2995
                            SystemMetadata sm = SystemMetadataFactory.createSystemMetadata(docid, false);
2995
                            SystemMetadata sm = SystemMetadataFactory.createSystemMetadata(docid, false, false);
2996 2996
							
2997 2997
					        // manage it in the store
2998 2998
                            HazelcastService.getInstance().getSystemMetadataMap().put(sm.getIdentifier(), sm);
src/edu/ucsb/nceas/metacat/dataone/SystemMetadataFactory.java
25 25
 */
26 26
package edu.ucsb.nceas.metacat.dataone;
27 27

  
28
import java.io.BufferedInputStream;
29 28
import java.io.IOException;
30 29
import java.io.InputStream;
31 30
import java.math.BigInteger;
31
import java.net.URL;
32 32
import java.security.NoSuchAlgorithmException;
33 33
import java.sql.SQLException;
34 34
import java.util.ArrayList;
......
109 109
	 * @throws AccessControlException 
110 110
	 * @throws AccessException 
111 111
	 */
112
	public static SystemMetadata createSystemMetadata(String localId, boolean includeORE)
112
	public static SystemMetadata createSystemMetadata(String localId, boolean includeORE, boolean downloadData)
113 113
			throws McdbException, McdbDocNotFoundException, SQLException,
114 114
			IOException, AccessionNumberException, ClassNotFoundException,
115 115
			InsufficientKarmaException, ParseLSIDException,
......
340 340
									.getFormat("application/octet-stream")
341 341
									.getFormatId().getValue();
342 342
						}
343
						String dataDocLocalId = "";
343

  
344
						// process the data
345
						String dataDocLocalId = null;
346
						Identifier dataGuid = new Identifier();
347

  
348
						// handle ecogrid, or downloadable data
349
						String ecogridPrefix = "ecogrid://knb/";
350
						if (dataDocUrl.trim().startsWith(ecogridPrefix)) {
351
							dataDocLocalId = dataDocUrl.substring(dataDocUrl.indexOf(ecogridPrefix) + ecogridPrefix.length());
352
						} else {
353
							// should we try downloading the remote data?
354
							if (downloadData) {
355
								InputStream dataObject = null;
356
								try {
357
									// download the data from the URL
358
									URL dataURL = new URL(dataDocUrl);
359
									dataObject = dataURL.openStream();
360
									// TODO: check for valid content
361
								} catch (Exception e) {
362
									// error with the download
363
									logMetacat.warn("Error downloading remote data. " + e.getMessage());
364
								}
365
								
366
								if (dataObject != null) {
367
									// create the local version of it
368
									dataDocLocalId = DocumentUtil.generateDocumentId(1);
369
									IdentifierManager.getInstance().createMapping(dataDocLocalId, dataDocLocalId);
370
									dataGuid.setValue(dataDocLocalId);
371
									
372
									// save it locally
373
									Session session = new Session();
374
									session.setSubject(submitter);
375
									MockHttpServletRequest request = new MockHttpServletRequest(null, null, null);
376
									MNodeService.getInstance(request).insertDataObject(dataObject, dataGuid, session);
377
								}
378
							}
379
							
380
						}
381
						
344 382
						logMetacat.debug("Data local ID: " + dataDocLocalId);
345 383
						logMetacat.debug("Data URL     : " + dataDocUrl);
346 384
						logMetacat.debug("Data mime    : " + dataDocMimeType);
347
	
348
						// we only handle ecogrid urls right now
349
						String ecogridPrefix = "ecogrid://knb/";
350
						if (dataDocUrl.trim().startsWith(ecogridPrefix)) {
351
							dataDocLocalId = dataDocUrl.substring(dataDocUrl
352
									.indexOf(ecogridPrefix)
353
									+ ecogridPrefix.length());
354
	
385
						
386
						// now we have a local id for the data
387
						if (dataDocLocalId != null) {
388
							
355 389
							// look up the guid for the data
356 390
							String dataDocid = DocumentUtil.getSmartDocId(dataDocLocalId);
357 391
							int dataRev = DocumentUtil.getRevisionFromAccessionNumber(dataDocLocalId);
......
359 393
							// check if data system metadata exists already
360 394
							SystemMetadata dataSysMeta = null;
361 395
							String dataGuidString = null;
362
							Identifier dataGuid = new Identifier();
363 396
							try {
364 397
								// look for the identifier
365 398
								dataGuidString = IdentifierManager.getInstance().getGUID(dataDocid, dataRev);
......
384 417
								logMetacat.debug("There was not an existing system metadata document for " + dataDocLocalId);
385 418
								try {
386 419
									logMetacat.debug("Creating a system metadata " + "document for " + dataDocLocalId);
387
									dataSysMeta = createSystemMetadata(dataDocLocalId, includeORE);
420
									dataSysMeta = createSystemMetadata(dataDocLocalId, includeORE, false);
388 421
	
389 422
									// now look it up again
390 423
									dataGuidString = IdentifierManager.getInstance().getGUID(dataDocid, dataRev);
src/edu/ucsb/nceas/metacat/admin/upgrade/GenerateSystemMetadata.java
61 61
        boolean success = true;
62 62
        // include ORE?
63 63
        boolean includeOre = true;
64
        boolean downloadData = true;
65

  
64 66
        try {
65
			generateMissingSystemMetadata(includeOre);
67
			generateMissingSystemMetadata(includeOre, downloadData);
66 68
		} catch (Exception e) {
67 69
			String msg = "Problem generating missing system metadata: " + e.getMessage();
68 70
			log.error(msg, e);
......
95 97
     * @throws McdbDocNotFoundException 
96 98
     * @throws ServiceFailure 
97 99
     */
98
    public void generateMissingSystemMetadata(boolean includeOre) 
100
    public void generateMissingSystemMetadata(boolean includeOre, boolean downloadData) 
99 101
    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized, 
100 102
    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
101 103
    {
......
107 109
            //for each id, add a system metadata doc
108 110
        	try {
109 111
        		log.debug("generating missing system metadata for " + localId);
110
        		generateMissingSystemMetadata(localId, includeOre);
112
        		generateMissingSystemMetadata(localId, includeOre, downloadData);
111 113
        	} catch (Exception e) {
112 114
				log.error("Error generating system metadata for: " + localId, e);
113 115
			}
......
137 139
     * @throws PropertyNotFoundException 
138 140
     * @throws McdbDocNotFoundException 
139 141
     */
140
    public void generateMissingSystemMetadata(String localId, boolean includeOre) 
142
    public void generateMissingSystemMetadata(String localId, boolean includeOre, boolean downloadData) 
141 143
    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized,
142 144
    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
143 145
    {
......
147 149

  
148 150
        //generate required system metadata fields from the document
149 151
        try {
150
        	sm = SystemMetadataFactory.createSystemMetadata(localId, includeOre);
152
        	sm = SystemMetadataFactory.createSystemMetadata(localId, includeOre, downloadData);
151 153
        } catch (Exception e1) {
152 154
        	e1.printStackTrace();
153 155
        	ServiceFailure sf = new ServiceFailure("00","Exception in generateMissingSystemMetadata: " +

Also available in: Unified diff