/ - Diff - Metacat - Ecoinformatics Redmine

« Previous | Next »

Revision 6721

Added by ben leinfelder over 12 years ago

generating ORE maps and creating/updating system metadata now. There are some Permission conversion issues to be worked out yet

     import org.apache.commons.beanutils.BeanUtils;
     import org.apache.commons.io.IOUtils;
     import org.apache.log4j.Logger;
     import org.apache.wicket.protocol.http.MockHttpServletRequest;
     import org.dataone.client.ObjectFormatCache;
     import org.dataone.ore.ResourceMapFactory;
     import org.dataone.service.exceptions.BaseException;
     import org.dataone.service.exceptions.NotFound;
     import org.dataone.service.types.v1.AccessPolicy;
     import org.dataone.service.types.v1.Checksum;
     import org.dataone.service.types.v1.Identifier;
     import org.dataone.service.types.v1.NodeReference;
     import org.dataone.service.types.v1.ObjectFormatIdentifier;
     import org.dataone.service.types.v1.Session;
     import org.dataone.service.types.v1.Subject;
     import org.dataone.service.types.v1.SystemMetadata;
     import org.dataone.service.types.v1.util.ChecksumUtil;
-...
     import edu.ucsb.nceas.metacat.dataquery.MetacatDatabaseConnectionPoolFactory;
     import edu.ucsb.nceas.metacat.properties.PropertyService;
     import edu.ucsb.nceas.metacat.replication.ReplicationService;
     import edu.ucsb.nceas.metacat.shared.AccessException;
     import edu.ucsb.nceas.metacat.shared.HandlerException;
     import edu.ucsb.nceas.metacat.util.DocumentUtil;
     import edu.ucsb.nceas.utilities.ParseLSIDException;
-...
     	 * @throws SAXException
     	 * @throws HandlerException
     	 * @throws AccessControlException
     	 * @throws AccessException
     	 */
     	public static SystemMetadata createSystemMetadata(String localId, boolean includeORE)
     			throws McdbException, McdbDocNotFoundException, SQLException,
     			IOException, AccessionNumberException, ClassNotFoundException,
     			InsufficientKarmaException, ParseLSIDException,
     			PropertyNotFoundException, BaseException, NoSuchAlgorithmException,
     			JiBXException, AccessControlException, HandlerException, SAXException {
     			JiBXException, AccessControlException, HandlerException, SAXException, AccessException {
     		logMetacat.debug("MetacatHandler.createSystemMetadata() called.");
     		logMetacat.debug("provided localId: " + localId);
-...
     		sysMeta.setFormatId(fmtid);
     		logMetacat.debug("The ObjectFormat for " + localId + " is " + fmtid.getValue());
     		// create the checksum
     		inputStream = MetacatHandler.read(localId);
     		String algorithm = "MD5";
     		Checksum checksum = ChecksumUtil.checksum(inputStream, algorithm);
     		sysMeta.setChecksum(checksum);
     		// set the size
     		inputStream = MetacatHandler.read(localId);
     		String sizeStr = new Long(sizeOfStream(inputStream)).toString();
     		sysMeta.setSize(new BigInteger(sizeStr));
     		// submitter
     		Subject submitter = new Subject();
     		submitter.setValue(docInfo.get("user_updated"));
     		sysMeta.setSubmitter(submitter);
     		// rights holder
     		Subject owner = new Subject();
     		owner.setValue(docInfo.get("user_owner"));
     		sysMeta.setRightsHolder(owner);
     		// dates
     		String createdDateString = docInfo.get("date_created");
     		String updatedDateString = docInfo.get("date_updated");
     		Date createdDate = DateTimeMarshaller.deserializeDateToUTC(createdDateString);
     		Date updatedDate = DateTimeMarshaller.deserializeDateToUTC(updatedDateString);
     		sysMeta.setDateUploaded(createdDate);
     		sysMeta.setDateSysMetadataModified(updatedDate);
     		// set the revision history
     		String docidWithoutRev = accNum.getDocid();
     		Identifier obsoletedBy = null;
     		Identifier obsoletes = null;
     		Vector<Integer> revisions = DBUtil.getRevListFromRevisionTable(docidWithoutRev);
     		for (int existingRev: revisions) {
     			// use the docid+rev as the guid
     			String existingPid = docidWithoutRev + "." + existingRev;
     			if (existingRev < rev) {
     				// it's the old docid, until it's not
     				obsoletes = new Identifier();
     				obsoletes.setValue(existingPid);
+    			}
     			if (existingRev > rev) {
     				// it's the newer docid
     				obsoletedBy = new Identifier();
     				obsoletedBy.setValue(existingPid);
     				// only want the version just after it
     				break;
+    			}
+    		}
     		// set them
     		sysMeta.setObsoletedBy(obsoletedBy);
     		sysMeta.setObsoletes(obsoletes);
     		// look up the access control policy we have in metacat already to make sure it is in the SM
     		AccessPolicy accessPolicy = IdentifierManager.getInstance().getAccessPolicyByLocalId(docidWithoutRev);
     		sysMeta.setAccessPolicy(accessPolicy);
     		// authoritative node
     		NodeReference nr = new NodeReference();
     		nr.setValue(PropertyService.getProperty("dataone.memberNodeId"));
     		sysMeta.setOriginMemberNode(nr);
     		sysMeta.setAuthoritativeMemberNode(nr);
     		// further parse EML documents to get data object format,
     		// describes and describedBy information
     		if (fmtid == ObjectFormatCache.getInstance().getFormat(
-...
     						"eml://ecoinformatics.org/eml-2.1.1").getFormatId()) {
     			try {
     				inputStream = MetacatHandler.read(localId);
     				DatabaseConnectionPoolInterface connectionPool =
     					MetacatDatabaseConnectionPoolFactory.getDatabaseConnectionPoolInterface();
     				DataManager dataManager =
     					DataManager.getInstance(connectionPool, connectionPool.getDBAdapterName());
     				DataPackage dataPackage = dataManager.parseMetadata(inputStream);
     				// iterate through the data objects in the EML doc and add
     				// sysmeta
     				// iterate through the data objects in the EML doc and add sysmeta
     				logMetacat.debug("In createSystemMetadata() the number of data "
     								+ "entities is: "
     								+ dataPackage.getEntityNumber());
-...
     						String dataDocid = DocumentUtil.getSmartDocId(dataDocLocalId);
     						int dataRev = DocumentUtil.getRevisionFromAccessionNumber(dataDocLocalId);
     						// check if data system metadata exists already
     						SystemMetadata dataSysMeta = null;
     						// check if data system metadata exists
     						String dataGuidString = null;
     						Identifier dataGuid = new Identifier();
     						try {
     							// look for the identifier
     							dataGuidString = IdentifierManager.getInstance().getGUID(dataDocid, dataRev);
     							dataSysMeta = IdentifierManager.getInstance().getSystemMetadata(dataGuidString);
     							// set it
     							dataGuid.setValue(dataGuidString);
     							// look up the system metadata
     							try {
     								dataSysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(dataGuid);
     							} catch (Exception e) {
     								// probably not in the system
     								dataSysMeta = null;
+    							}
     							//dataSysMeta = IdentifierManager.getInstance().getSystemMetadata(dataGuidString);
     						} catch (McdbDocNotFoundException nf) {
     							// System metadata for data doesn't exist yet, so
     							// create it
     							logMetacat.debug("There was not an existing system metadata "
     											+ "document for " + dataDocLocalId);
     							// we didn't find it
     							dataSysMeta = null;
+    						}
     						// we'll have to generate it
     						if (dataSysMeta == null) {
     							// System metadata for data doesn't exist yet, so create it
     							logMetacat.debug("There was not an existing system metadata document for " + dataDocLocalId);
     							try {
     								logMetacat.debug("Creating a system metadata "
     										+ "document for " + dataDocLocalId);
     								logMetacat.debug("Creating a system metadata " + "document for " + dataDocLocalId);
     								dataSysMeta = createSystemMetadata(dataDocLocalId, includeORE);
     								// now look it up again
     								dataGuidString = IdentifierManager.getInstance().getGUID(dataDocid, dataRev);
     								// set the guid
     								Identifier dataGuid = new Identifier();
     								dataGuid.setValue(dataGuidString);
     								// set object format
-...
     								// update the values
     								HazelcastService.getInstance().getSystemMetadataMap().put(dataSysMeta.getIdentifier(), dataSysMeta);
     								// part of the ORE package
     								dataIds.add(dataGuid);
     							} catch (McdbDocNotFoundException mdnf) {
     								mdnf.printStackTrace();
-...
     							} catch (BaseException be) {
     								be.printStackTrace();
     								throw be;
+    							}
+    							}
+    						}
     						// part of the ORE package
     						dataIds.add(dataGuid);
     					} // end if()
     					} // end if (EML package)
     				} // end for (data entities)
-...
     			        if (!dataIds.isEmpty()) {
     			            // generate the ORE map for this datapackage
     			            Identifier resourceMapId = new Identifier();
     			            resourceMapId.setValue("resourceMap_" + sysMeta.getIdentifier().getValue());
     			            // want to be able to run this over and over again for now
     			            resourceMapId.setValue("resourceMap_" + sysMeta.getIdentifier().getValue() + "_" + System.currentTimeMillis());
     			            idMap.put(sysMeta.getIdentifier(), dataIds);
     			            ResourceMap rm = ResourceMapFactory.getInstance().createResourceMap(resourceMapId, idMap);
     			            String resourceMapXML = ResourceMapFactory.getInstance().serializeResourceMap(rm);
     			            // copy most of the same system metadata as the packaging metadata
     			            SystemMetadata resourceMapSysMeta = new SystemMetadata();
     			            BeanUtils.copyProperties(resourceMapXML, sysMeta);
     			            BeanUtils.copyProperties(resourceMapSysMeta, sysMeta);
     			            resourceMapSysMeta.setIdentifier(resourceMapId);
     			            Checksum checksum = ChecksumUtil.checksum(new BufferedInputStream(IOUtils.toInputStream(resourceMapXML, MetaCatServlet.DEFAULT_ENCODING)), "MD5");
     						resourceMapSysMeta.setChecksum(checksum);
     			            Checksum oreChecksum = ChecksumUtil.checksum(IOUtils.toInputStream(resourceMapXML, MetaCatServlet.DEFAULT_ENCODING), "MD5");
     						resourceMapSysMeta.setChecksum(oreChecksum);
     			            ObjectFormatIdentifier formatId = ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms").getFormatId();
     						resourceMapSysMeta.setFormatId(formatId);
     			            // TODO: other fields to update?
     						// save it locally
     						MNodeService.getInstance(null).create(
     								null,
     						Session session = new Session();
     						session.setSubject(submitter);
     						MockHttpServletRequest request = new MockHttpServletRequest(null, null, null);
     						MNodeService.getInstance(request).create(
     								session,
     								resourceMapId,
     								IOUtils.toInputStream(resourceMapXML, MetaCatServlet.DEFAULT_ENCODING),
     								resourceMapSysMeta);
-...
     			} catch (Exception e) {
     				logMetacat.debug("There was a problem creating System Metadata. "
     								+ "The error message was: " + e.getMessage());
     				e.printStackTrace();
     			} // end try()
     		} // end if()
     		// create the checksum
     		inputStream = MetacatHandler.read(localId);
     		String algorithm = "MD5";
     		Checksum checksum = ChecksumUtil.checksum(inputStream, algorithm);
     		sysMeta.setChecksum(checksum);
     		// set the size
     		inputStream = MetacatHandler.read(localId);
     		String sizeStr = new Long(sizeOfStream(inputStream)).toString();
     		sysMeta.setSize(new BigInteger(sizeStr));
     		// submitter
     		Subject submitter = new Subject();
     		submitter.setValue(docInfo.get("user_updated"));
     		sysMeta.setSubmitter(submitter);
     		// rights holder
     		Subject owner = new Subject();
     		owner.setValue(docInfo.get("user_owner"));
     		sysMeta.setRightsHolder(owner);
     		// dates
     		String createdDateString = docInfo.get("date_created");
     		String updatedDateString = docInfo.get("date_updated");
     		Date createdDate = DateTimeMarshaller.deserializeDateToUTC(createdDateString);
     		Date updatedDate = DateTimeMarshaller.deserializeDateToUTC(updatedDateString);
     		sysMeta.setDateUploaded(createdDate);
     		sysMeta.setDateSysMetadataModified(updatedDate);
     		// set the revision history
     		String docidWithoutRev = accNum.getDocid();
     		Identifier obsoletedBy = new Identifier();
     		Identifier obsoletes = new Identifier();
     		Vector<Integer> revisions = DBUtil.getRevListFromRevisionTable(docidWithoutRev);
     		for (int existingRev: revisions) {
     			// use the docid+rev as the guid
     			String existingPid = docidWithoutRev + "." + existingRev;
     			if (existingRev < rev) {
     				// it's the old docid, until it's not
     				obsoletes.setValue(existingPid);
+    			}
     			if (existingRev > rev) {
     				// it's the newer docid
     				obsoletedBy.setValue(existingPid);
     				// only want the version just after it
     				break;
+    			}
+    		}
     		// set them
     		sysMeta.setObsoletedBy(obsoletedBy);
     		sysMeta.setObsoletes(obsoletes);
     		// TODO: access control?
     		// I believe the access control will be fine since we use the same storage mechanism for Metacat/D1
     		// authoritative node
     		NodeReference nr = new NodeReference();
     		nr.setValue(PropertyService.getProperty("dataone.memberNodeId"));
     		sysMeta.setOriginMemberNode(nr);
     		sysMeta.setAuthoritativeMemberNode(nr);
     		return sysMeta;
+    	}

     import org.dataone.service.types.v1.SystemMetadata;
     import edu.ucsb.nceas.metacat.AccessionNumberException;
     import edu.ucsb.nceas.metacat.DBUtil;
     import edu.ucsb.nceas.metacat.DocumentImpl;
     import edu.ucsb.nceas.metacat.IdentifierManager;
     import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
     import edu.ucsb.nceas.metacat.admin.AdminException;
     import edu.ucsb.nceas.metacat.dataone.SystemMetadataFactory;
     import edu.ucsb.nceas.metacat.properties.PropertyService;
     import edu.ucsb.nceas.metacat.util.DocumentUtil;
     import edu.ucsb.nceas.utilities.PropertyNotFoundException;
     import edu.ucsb.nceas.utilities.SortedProperties;
-...
         NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException
+        {
             //get the list of ids with no SM
             List<String> idList = IdentifierManager.getInstance().getLocalIdsWithNoSystemMetadata(true);
             //List<String> idList = IdentifierManager.getInstance().getLocalIdsWithNoSystemMetadata(true);
             List<String> idList = DBUtil.getAllDocidsByType(null, true);
             for (String localId : idList) {
                 //for each id, add a system metadata doc
             	try {
             		log.debug("generating missing system metadata for " + localId);
             		generateMissingSystemMetadata(localId, includeOre);
             	} catch (Exception e) {
     				log.error("Error generating system metadata for: " + localId, e);
+    			}
+            }
             log.info("generateMissingSystemMetadata(token)");
             log.info("done generating missing system metadata");
+        }
         /**
-...
             	throw sf;
+            }
             //insert the systemmetadata object or just update it
             boolean exists = IdentifierManager.getInstance().identifierExists(sm.getIdentifier().getValue());
             //insert the systemmetadata object or just update it as needed
             boolean exists = IdentifierManager.getInstance().systemMetadataExists(sm.getIdentifier().getValue());
             if (!exists) {
             	IdentifierManager.getInstance().createSystemMetadata(sm);
+            }

Also available in: Unified diff

Project

General

Profile

Metacat

Revision 6721

Added by ben leinfelder over 12 years ago