Revision 6721
Added by ben leinfelder over 12 years ago
SystemMetadataFactory.java | ||
---|---|---|
45 | 45 |
import org.apache.commons.beanutils.BeanUtils; |
46 | 46 |
import org.apache.commons.io.IOUtils; |
47 | 47 |
import org.apache.log4j.Logger; |
48 |
import org.apache.wicket.protocol.http.MockHttpServletRequest; |
|
48 | 49 |
import org.dataone.client.ObjectFormatCache; |
49 | 50 |
import org.dataone.ore.ResourceMapFactory; |
50 | 51 |
import org.dataone.service.exceptions.BaseException; |
51 | 52 |
import org.dataone.service.exceptions.NotFound; |
53 |
import org.dataone.service.types.v1.AccessPolicy; |
|
52 | 54 |
import org.dataone.service.types.v1.Checksum; |
53 | 55 |
import org.dataone.service.types.v1.Identifier; |
54 | 56 |
import org.dataone.service.types.v1.NodeReference; |
55 | 57 |
import org.dataone.service.types.v1.ObjectFormatIdentifier; |
58 |
import org.dataone.service.types.v1.Session; |
|
56 | 59 |
import org.dataone.service.types.v1.Subject; |
57 | 60 |
import org.dataone.service.types.v1.SystemMetadata; |
58 | 61 |
import org.dataone.service.types.v1.util.ChecksumUtil; |
... | ... | |
78 | 81 |
import edu.ucsb.nceas.metacat.dataquery.MetacatDatabaseConnectionPoolFactory; |
79 | 82 |
import edu.ucsb.nceas.metacat.properties.PropertyService; |
80 | 83 |
import edu.ucsb.nceas.metacat.replication.ReplicationService; |
84 |
import edu.ucsb.nceas.metacat.shared.AccessException; |
|
81 | 85 |
import edu.ucsb.nceas.metacat.shared.HandlerException; |
82 | 86 |
import edu.ucsb.nceas.metacat.util.DocumentUtil; |
83 | 87 |
import edu.ucsb.nceas.utilities.ParseLSIDException; |
... | ... | |
101 | 105 |
* @throws SAXException |
102 | 106 |
* @throws HandlerException |
103 | 107 |
* @throws AccessControlException |
108 |
* @throws AccessException |
|
104 | 109 |
*/ |
105 | 110 |
public static SystemMetadata createSystemMetadata(String localId, boolean includeORE) |
106 | 111 |
throws McdbException, McdbDocNotFoundException, SQLException, |
107 | 112 |
IOException, AccessionNumberException, ClassNotFoundException, |
108 | 113 |
InsufficientKarmaException, ParseLSIDException, |
109 | 114 |
PropertyNotFoundException, BaseException, NoSuchAlgorithmException, |
110 |
JiBXException, AccessControlException, HandlerException, SAXException { |
|
115 |
JiBXException, AccessControlException, HandlerException, SAXException, AccessException {
|
|
111 | 116 |
|
112 | 117 |
logMetacat.debug("MetacatHandler.createSystemMetadata() called."); |
113 | 118 |
logMetacat.debug("provided localId: " + localId); |
... | ... | |
208 | 213 |
sysMeta.setFormatId(fmtid); |
209 | 214 |
logMetacat.debug("The ObjectFormat for " + localId + " is " + fmtid.getValue()); |
210 | 215 |
|
216 |
// create the checksum |
|
217 |
inputStream = MetacatHandler.read(localId); |
|
218 |
String algorithm = "MD5"; |
|
219 |
Checksum checksum = ChecksumUtil.checksum(inputStream, algorithm); |
|
220 |
sysMeta.setChecksum(checksum); |
|
221 |
|
|
222 |
// set the size |
|
223 |
inputStream = MetacatHandler.read(localId); |
|
224 |
String sizeStr = new Long(sizeOfStream(inputStream)).toString(); |
|
225 |
sysMeta.setSize(new BigInteger(sizeStr)); |
|
226 |
|
|
227 |
// submitter |
|
228 |
Subject submitter = new Subject(); |
|
229 |
submitter.setValue(docInfo.get("user_updated")); |
|
230 |
sysMeta.setSubmitter(submitter); |
|
231 |
|
|
232 |
// rights holder |
|
233 |
Subject owner = new Subject(); |
|
234 |
owner.setValue(docInfo.get("user_owner")); |
|
235 |
sysMeta.setRightsHolder(owner); |
|
236 |
|
|
237 |
// dates |
|
238 |
String createdDateString = docInfo.get("date_created"); |
|
239 |
String updatedDateString = docInfo.get("date_updated"); |
|
240 |
Date createdDate = DateTimeMarshaller.deserializeDateToUTC(createdDateString); |
|
241 |
Date updatedDate = DateTimeMarshaller.deserializeDateToUTC(updatedDateString); |
|
242 |
sysMeta.setDateUploaded(createdDate); |
|
243 |
sysMeta.setDateSysMetadataModified(updatedDate); |
|
244 |
|
|
245 |
// set the revision history |
|
246 |
String docidWithoutRev = accNum.getDocid(); |
|
247 |
Identifier obsoletedBy = null; |
|
248 |
Identifier obsoletes = null; |
|
249 |
Vector<Integer> revisions = DBUtil.getRevListFromRevisionTable(docidWithoutRev); |
|
250 |
for (int existingRev: revisions) { |
|
251 |
// use the docid+rev as the guid |
|
252 |
String existingPid = docidWithoutRev + "." + existingRev; |
|
253 |
if (existingRev < rev) { |
|
254 |
// it's the old docid, until it's not |
|
255 |
obsoletes = new Identifier(); |
|
256 |
obsoletes.setValue(existingPid); |
|
257 |
} |
|
258 |
if (existingRev > rev) { |
|
259 |
// it's the newer docid |
|
260 |
obsoletedBy = new Identifier(); |
|
261 |
obsoletedBy.setValue(existingPid); |
|
262 |
// only want the version just after it |
|
263 |
break; |
|
264 |
} |
|
265 |
} |
|
266 |
// set them |
|
267 |
sysMeta.setObsoletedBy(obsoletedBy); |
|
268 |
sysMeta.setObsoletes(obsoletes); |
|
269 |
|
|
270 |
// look up the access control policy we have in metacat already to make sure it is in the SM |
|
271 |
AccessPolicy accessPolicy = IdentifierManager.getInstance().getAccessPolicyByLocalId(docidWithoutRev); |
|
272 |
sysMeta.setAccessPolicy(accessPolicy); |
|
273 |
|
|
274 |
// authoritative node |
|
275 |
NodeReference nr = new NodeReference(); |
|
276 |
nr.setValue(PropertyService.getProperty("dataone.memberNodeId")); |
|
277 |
sysMeta.setOriginMemberNode(nr); |
|
278 |
sysMeta.setAuthoritativeMemberNode(nr); |
|
279 |
|
|
211 | 280 |
// further parse EML documents to get data object format, |
212 | 281 |
// describes and describedBy information |
213 | 282 |
if (fmtid == ObjectFormatCache.getInstance().getFormat( |
... | ... | |
220 | 289 |
"eml://ecoinformatics.org/eml-2.1.1").getFormatId()) { |
221 | 290 |
|
222 | 291 |
try { |
223 |
|
|
292 |
inputStream = MetacatHandler.read(localId); |
|
224 | 293 |
DatabaseConnectionPoolInterface connectionPool = |
225 | 294 |
MetacatDatabaseConnectionPoolFactory.getDatabaseConnectionPoolInterface(); |
226 | 295 |
DataManager dataManager = |
227 | 296 |
DataManager.getInstance(connectionPool, connectionPool.getDBAdapterName()); |
228 | 297 |
DataPackage dataPackage = dataManager.parseMetadata(inputStream); |
229 | 298 |
|
230 |
// iterate through the data objects in the EML doc and add |
|
231 |
// sysmeta |
|
299 |
// iterate through the data objects in the EML doc and add sysmeta |
|
232 | 300 |
logMetacat.debug("In createSystemMetadata() the number of data " |
233 | 301 |
+ "entities is: " |
234 | 302 |
+ dataPackage.getEntityNumber()); |
... | ... | |
264 | 332 |
String dataDocid = DocumentUtil.getSmartDocId(dataDocLocalId); |
265 | 333 |
int dataRev = DocumentUtil.getRevisionFromAccessionNumber(dataDocLocalId); |
266 | 334 |
|
335 |
// check if data system metadata exists already |
|
267 | 336 |
SystemMetadata dataSysMeta = null; |
268 |
// check if data system metadata exists |
|
269 | 337 |
String dataGuidString = null; |
338 |
Identifier dataGuid = new Identifier(); |
|
270 | 339 |
try { |
340 |
// look for the identifier |
|
271 | 341 |
dataGuidString = IdentifierManager.getInstance().getGUID(dataDocid, dataRev); |
272 |
dataSysMeta = IdentifierManager.getInstance().getSystemMetadata(dataGuidString); |
|
342 |
// set it |
|
343 |
dataGuid.setValue(dataGuidString); |
|
344 |
// look up the system metadata |
|
345 |
try { |
|
346 |
dataSysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(dataGuid); |
|
347 |
} catch (Exception e) { |
|
348 |
// probably not in the system |
|
349 |
dataSysMeta = null; |
|
350 |
} |
|
351 |
//dataSysMeta = IdentifierManager.getInstance().getSystemMetadata(dataGuidString); |
|
273 | 352 |
} catch (McdbDocNotFoundException nf) { |
274 |
// System metadata for data doesn't exist yet, so |
|
275 |
// create it |
|
276 |
logMetacat.debug("There was not an existing system metadata " |
|
277 |
+ "document for " + dataDocLocalId); |
|
353 |
// we didn't find it |
|
354 |
dataSysMeta = null; |
|
355 |
} |
|
356 |
|
|
357 |
// we'll have to generate it |
|
358 |
if (dataSysMeta == null) { |
|
359 |
// System metadata for data doesn't exist yet, so create it |
|
360 |
logMetacat.debug("There was not an existing system metadata document for " + dataDocLocalId); |
|
278 | 361 |
try { |
279 |
logMetacat.debug("Creating a system metadata " |
|
280 |
+ "document for " + dataDocLocalId); |
|
362 |
logMetacat.debug("Creating a system metadata " + "document for " + dataDocLocalId); |
|
281 | 363 |
dataSysMeta = createSystemMetadata(dataDocLocalId, includeORE); |
282 | 364 |
|
283 | 365 |
// now look it up again |
284 | 366 |
dataGuidString = IdentifierManager.getInstance().getGUID(dataDocid, dataRev); |
285 | 367 |
|
286 | 368 |
// set the guid |
287 |
Identifier dataGuid = new Identifier(); |
|
288 | 369 |
dataGuid.setValue(dataGuidString); |
289 | 370 |
|
290 | 371 |
// set object format |
... | ... | |
306 | 387 |
// update the values |
307 | 388 |
HazelcastService.getInstance().getSystemMetadataMap().put(dataSysMeta.getIdentifier(), dataSysMeta); |
308 | 389 |
|
309 |
// part of the ORE package |
|
310 |
dataIds.add(dataGuid); |
|
311 | 390 |
|
312 | 391 |
} catch (McdbDocNotFoundException mdnf) { |
313 | 392 |
mdnf.printStackTrace(); |
... | ... | |
333 | 412 |
} catch (BaseException be) { |
334 | 413 |
be.printStackTrace(); |
335 | 414 |
throw be; |
336 |
} |
|
415 |
} |
|
416 |
|
|
337 | 417 |
} |
418 |
|
|
419 |
// part of the ORE package |
|
420 |
dataIds.add(dataGuid); |
|
338 | 421 |
|
339 |
} // end if()
|
|
422 |
} // end if (EML package)
|
|
340 | 423 |
|
341 | 424 |
} // end for (data entities) |
342 | 425 |
|
... | ... | |
345 | 428 |
if (!dataIds.isEmpty()) { |
346 | 429 |
// generate the ORE map for this datapackage |
347 | 430 |
Identifier resourceMapId = new Identifier(); |
348 |
resourceMapId.setValue("resourceMap_" + sysMeta.getIdentifier().getValue()); |
|
431 |
// want to be able to run this over and over again for now |
|
432 |
resourceMapId.setValue("resourceMap_" + sysMeta.getIdentifier().getValue() + "_" + System.currentTimeMillis()); |
|
349 | 433 |
idMap.put(sysMeta.getIdentifier(), dataIds); |
350 | 434 |
ResourceMap rm = ResourceMapFactory.getInstance().createResourceMap(resourceMapId, idMap); |
351 | 435 |
String resourceMapXML = ResourceMapFactory.getInstance().serializeResourceMap(rm); |
352 | 436 |
// copy most of the same system metadata as the packaging metadata |
353 | 437 |
SystemMetadata resourceMapSysMeta = new SystemMetadata(); |
354 |
BeanUtils.copyProperties(resourceMapXML, sysMeta);
|
|
438 |
BeanUtils.copyProperties(resourceMapSysMeta, sysMeta);
|
|
355 | 439 |
resourceMapSysMeta.setIdentifier(resourceMapId); |
356 |
Checksum checksum = ChecksumUtil.checksum(new BufferedInputStream(IOUtils.toInputStream(resourceMapXML, MetaCatServlet.DEFAULT_ENCODING)), "MD5");
|
|
357 |
resourceMapSysMeta.setChecksum(checksum);
|
|
440 |
Checksum oreChecksum = ChecksumUtil.checksum(IOUtils.toInputStream(resourceMapXML, MetaCatServlet.DEFAULT_ENCODING), "MD5");
|
|
441 |
resourceMapSysMeta.setChecksum(oreChecksum);
|
|
358 | 442 |
ObjectFormatIdentifier formatId = ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms").getFormatId(); |
359 | 443 |
resourceMapSysMeta.setFormatId(formatId); |
360 | 444 |
// TODO: other fields to update? |
361 | 445 |
|
362 | 446 |
// save it locally |
363 |
MNodeService.getInstance(null).create( |
|
364 |
null, |
|
447 |
Session session = new Session(); |
|
448 |
session.setSubject(submitter); |
|
449 |
MockHttpServletRequest request = new MockHttpServletRequest(null, null, null); |
|
450 |
MNodeService.getInstance(request).create( |
|
451 |
session, |
|
365 | 452 |
resourceMapId, |
366 | 453 |
IOUtils.toInputStream(resourceMapXML, MetaCatServlet.DEFAULT_ENCODING), |
367 | 454 |
resourceMapSysMeta); |
... | ... | |
382 | 469 |
} catch (Exception e) { |
383 | 470 |
logMetacat.debug("There was a problem creating System Metadata. " |
384 | 471 |
+ "The error message was: " + e.getMessage()); |
472 |
e.printStackTrace(); |
|
385 | 473 |
} // end try() |
386 | 474 |
|
387 | 475 |
} // end if() |
388 |
|
|
389 | 476 |
|
390 |
// create the checksum |
|
391 |
inputStream = MetacatHandler.read(localId); |
|
392 |
String algorithm = "MD5"; |
|
393 |
Checksum checksum = ChecksumUtil.checksum(inputStream, algorithm); |
|
394 |
sysMeta.setChecksum(checksum); |
|
395 |
|
|
396 |
// set the size |
|
397 |
inputStream = MetacatHandler.read(localId); |
|
398 |
String sizeStr = new Long(sizeOfStream(inputStream)).toString(); |
|
399 |
sysMeta.setSize(new BigInteger(sizeStr)); |
|
400 |
|
|
401 |
// submitter |
|
402 |
Subject submitter = new Subject(); |
|
403 |
submitter.setValue(docInfo.get("user_updated")); |
|
404 |
sysMeta.setSubmitter(submitter); |
|
405 |
|
|
406 |
// rights holder |
|
407 |
Subject owner = new Subject(); |
|
408 |
owner.setValue(docInfo.get("user_owner")); |
|
409 |
sysMeta.setRightsHolder(owner); |
|
410 |
|
|
411 |
// dates |
|
412 |
String createdDateString = docInfo.get("date_created"); |
|
413 |
String updatedDateString = docInfo.get("date_updated"); |
|
414 |
Date createdDate = DateTimeMarshaller.deserializeDateToUTC(createdDateString); |
|
415 |
Date updatedDate = DateTimeMarshaller.deserializeDateToUTC(updatedDateString); |
|
416 |
sysMeta.setDateUploaded(createdDate); |
|
417 |
sysMeta.setDateSysMetadataModified(updatedDate); |
|
418 |
|
|
419 |
// set the revision history |
|
420 |
String docidWithoutRev = accNum.getDocid(); |
|
421 |
Identifier obsoletedBy = new Identifier(); |
|
422 |
Identifier obsoletes = new Identifier(); |
|
423 |
Vector<Integer> revisions = DBUtil.getRevListFromRevisionTable(docidWithoutRev); |
|
424 |
for (int existingRev: revisions) { |
|
425 |
// use the docid+rev as the guid |
|
426 |
String existingPid = docidWithoutRev + "." + existingRev; |
|
427 |
if (existingRev < rev) { |
|
428 |
// it's the old docid, until it's not |
|
429 |
obsoletes.setValue(existingPid); |
|
430 |
} |
|
431 |
if (existingRev > rev) { |
|
432 |
// it's the newer docid |
|
433 |
obsoletedBy.setValue(existingPid); |
|
434 |
// only want the version just after it |
|
435 |
break; |
|
436 |
} |
|
437 |
} |
|
438 |
// set them |
|
439 |
sysMeta.setObsoletedBy(obsoletedBy); |
|
440 |
sysMeta.setObsoletes(obsoletes); |
|
441 |
|
|
442 |
// TODO: access control? |
|
443 |
// I believe the access control will be fine since we use the same storage mechanism for Metacat/D1 |
|
444 |
|
|
445 |
// authoritative node |
|
446 |
NodeReference nr = new NodeReference(); |
|
447 |
nr.setValue(PropertyService.getProperty("dataone.memberNodeId")); |
|
448 |
sysMeta.setOriginMemberNode(nr); |
|
449 |
sysMeta.setAuthoritativeMemberNode(nr); |
|
450 |
|
|
451 | 477 |
return sysMeta; |
452 | 478 |
} |
453 | 479 |
|
Also available in: Unified diff
generating ORE maps and creating/updating system metadata now. There are some Permission conversion issues to be worked out yet