/ - Diff - Metacat - Ecoinformatics Redmine

« Previous | Next »

Revision 6964

Added by ben leinfelder almost 13 years ago

multithreaded implementation for processing docids for system metadata generation.
need to investigate ant/junit running that deadlocks hazelcast (config?)

     			PropertyNotFoundException, BaseException, NoSuchAlgorithmException,
     			JiBXException, AccessControlException, HandlerException, SAXException, AccessException {
     		logMetacat.debug("createSystemMetadata() called.");
     		logMetacat.debug("provided localId: " + localId);
     		logMetacat.debug("createSystemMetadata() called for localId " + localId);
     		// check for system metadata
     		SystemMetadata sysMeta = null;
-...
     			guid = IdentifierManager.getInstance().getGUID(accNum.getDocid(), rev);
     		} catch (McdbDocNotFoundException dnfe) {
     			// otherwise create the mapping
     			logMetacat.debug("There was a problem getting the guid from "
     							+ "the given localId (docid and revision). The error message was: "
     							+ dnfe.getMessage());
     			logMetacat.debug("No guid in the identifier table.  adding it for " + localId);
     			logMetacat.debug("No guid found in the identifier table.  Creating mapping for " + localId);
     			IdentifierManager.getInstance().createMapping(localId, localId);
     			logMetacat.debug("Mapping created for " + localId);
     			logMetacat.debug("accessionNumber: " + accNum);
     			guid = IdentifierManager.getInstance().getGUID(accNum.getDocid(), rev);
     			guid = IdentifierManager.getInstance().getGUID(accNum.getDocid(), rev);
+    		}
     		// look up existing system metadata if it exists
     		Identifier identifier = new Identifier();
     		identifier.setValue(guid);
     		try {
     			logMetacat.debug("Using hazelcast to get system metadata");
     			sysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(identifier);
     			// TODO: if this is the case, we could return here -- what else do we gain?
     			if (!updateExisting ) {
     				return sysMeta;
+    			}
     		} catch (Exception e) {
     			logMetacat.debug("No system metadata found in hz: " + e.getMessage());
     			// create system metadata
     			sysMeta = new SystemMetadata();
     			sysMeta.setIdentifier(identifier);
-...
     		try {
     			fmtid = ObjectFormatCache.getInstance().getFormat(doctype).getFormatId();
     		} catch (NotFound nfe) {
     			try {
     				// format is not registered, use default
     				if (doctype.trim().equals("BIN")) {
     					fmtid = ObjectFormatCache.getInstance().getFormat(
     							"application/octet-stream").getFormatId();
     				} else {
     					fmtid = ObjectFormatCache.getInstance().getFormat(
     							"text/plain").getFormatId();
+    				}
     			} catch (NotFound nf) {
     				logMetacat.error("There was a problem getting the default format "
     								+ "from the ObjectFormatCache: "
     								+ nf.getMessage());
     				throw nf;
     			// format is not registered, use default
     			if (doctype.trim().equals("BIN")) {
     				fmtid = ObjectFormatCache.getInstance().getFormat("application/octet-stream").getFormatId();
     			} else {
     				fmtid = ObjectFormatCache.getInstance().getFormat("text/plain").getFormatId();
+    			}
+    		}
     		sysMeta.setFormatId(fmtid);
-...
     							// we'll have to generate it
     							if (dataSysMeta == null) {
     								// System metadata for data doesn't exist yet, so create it
     								logMetacat.debug("There was not an existing system metadata document for " + dataDocLocalId);
     								logMetacat.debug("Creating a system metadata " + "document for " + dataDocLocalId);
     								logMetacat.debug("No exisiting SystemMetdata found, creating for: " + dataDocLocalId);
     								dataSysMeta = createSystemMetadata(dataDocLocalId, includeORE, false);
     								// now look it up again
-...
+    							}
     							// set object format for the data file
     							logMetacat.debug("Updating system metadata for "
     											+ dataGuid.getValue() + " to "
     											+ dataDocMimeType);
     							logMetacat.debug("Updating system metadata for " + dataGuid.getValue() + " to " + dataDocMimeType);
     							try {
     								ObjectFormatIdentifier fmt =
     									ObjectFormatCache.getInstance().getFormat(dataDocMimeType).getFormatId();
-...
          * @param idList
          * @param includeOre
          * @param downloadData
          * @throws ServiceFailure
          * @throws McdbDocNotFoundException
          * @throws PropertyNotFoundException
          * @throws InvalidToken
          * @throws NotAuthorized
          * @throws NotFound
          * @throws NotImplemented
          * @throws InvalidRequest
          * @throws NoSuchAlgorithmException
          * @throws AccessionNumberException
          * @throws SQLException
     	 * @throws SAXException
     	 * @throws HandlerException
     	 * @throws JiBXException
     	 * @throws BaseException
     	 * @throws ParseLSIDException
     	 * @throws InsufficientKarmaException
     	 * @throws ClassNotFoundException
     	 * @throws IOException
     	 * @throws McdbException
     	 * @throws AccessException
     	 * @throws AccessControlException
          */
         public static void generateSystemMetadata(List<String> idList, boolean includeOre, boolean downloadData)
         throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized,
         NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException
         throws PropertyNotFoundException, NoSuchAlgorithmException, AccessionNumberException, SQLException, AccessControlException, AccessException, McdbException, IOException, ClassNotFoundException, InsufficientKarmaException, ParseLSIDException, BaseException, JiBXException, HandlerException, SAXException
+        {
             for (String localId : idList) {
                 //for each id, add a system metadata doc
             	try {
             		logMetacat.debug("generating system metadata for " + localId);
             		generateSystemMetadata(localId, includeOre, downloadData);
             	} catch (Exception e) {
             		logMetacat.error("Error generating system metadata for: " + localId, e);
+    			}
             	generateSystemMetadata(localId, includeOre, downloadData);
+            }
             logMetacat.info("done generating system metadata");
             logMetacat.info("done generating system metadata for given list");
+        }
-...
          * @param localId
          * @param includeOre
          * @param downloadData
          * @throws ServiceFailure
          * @throws McdbDocNotFoundException
          * @throws PropertyNotFoundException
          * @throws InvalidToken
          * @throws NotAuthorized
          * @throws NotFound
          * @throws NotImplemented
          * @throws InvalidRequest
          * @throws NoSuchAlgorithmException
          * @throws AccessionNumberException
          * @throws SQLException
          * @throws InvalidSystemMetadata
          * @throws SAXException
          * @throws HandlerException
          * @throws JiBXException
          * @throws BaseException
          * @throws ParseLSIDException
          * @throws InsufficientKarmaException
          * @throws ClassNotFoundException
          * @throws IOException
          * @throws McdbException
          * @throws AccessException
          * @throws AccessControlException
          */
         protected static void generateSystemMetadata(String localId, boolean includeOre, boolean downloadData)
         throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized,
         NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException, InvalidSystemMetadata
         throws PropertyNotFoundException, NoSuchAlgorithmException, AccessionNumberException, SQLException, AccessControlException, AccessException, McdbException, IOException, ClassNotFoundException, InsufficientKarmaException, ParseLSIDException, BaseException, JiBXException, HandlerException, SAXException
+        {
         	logMetacat.debug("generateSystemMetadata() called.");
         	logMetacat.debug("Creating SystemMetadata for localId " + localId);
             SystemMetadata sm = null;
             //generate required system metadata fields from the document
             try {
             	sm = SystemMetadataFactory.createSystemMetadata(localId, includeOre, downloadData);
             } catch (Exception e1) {
             	e1.printStackTrace();
             	ServiceFailure sf = new ServiceFailure("00","Exception in generateSystemMetadata: " +
             			e1.getMessage());
             	sf.setStackTrace(e1.getStackTrace());
             	throw sf;
+            }
         	sm = SystemMetadataFactory.createSystemMetadata(localId, includeOre, downloadData);
             //insert the systemmetadata object or just update it as needed
             boolean exists = IdentifierManager.getInstance().systemMetadataExists(sm.getIdentifier().getValue());
             if (!exists) {

     import java.util.Collections;
     import java.util.List;
     import java.util.concurrent.ExecutorService;
     import java.util.concurrent.Executors;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
-...
     	private static Log log = LogFactory.getLog(GenerateSystemMetadata.class);
         public boolean upgrade() throws AdminException {
             return multiThreadUpgrade();
             //return singleThreadUpgrade();
+        }
         /**
          * Use this current single thread to process each ID in the list
          * @return
          * @throws AdminException
          * @deprecated use the multithreaded version of this upgrade method
          */
         public boolean singleThreadUpgrade() throws AdminException {
             boolean success = true;
             // do not include ORE or data, but can generate SystemMetadata for ALL records
-...
         	return success;
+        }
         /**
          * Use multiple threads to process parts of the complete ID list concurrently
          * @return
          * @throws AdminException
          */
         public boolean multiThreadUpgrade() throws AdminException {
             boolean success = true;
             // do not include ORE or data, but can generate SystemMetadata for ALL records
             final boolean includeOre = false;
             final boolean downloadData = false;
             int serverLocation = -1;
             try {
             	// get list of ALL docids at ALL server locations
                 List<String> idList = DBUtil.getAllDocidsByType(null, true, serverLocation);
                 Collections.sort(idList);
                 // executor
                 int availableProcessors = Runtime.getRuntime().availableProcessors();
                 int nThreads = availableProcessors * 1;
             	log.debug("Using nThreads: " + nThreads);
                 ExecutorService executor = Executors.newFixedThreadPool(nThreads);
                 // chunk into groups
     			int fromIndex = 0;
                 int toIndex = 0;
                 String prefix = null;
                 for (String docid: idList) {
                 	// increment the next entry, exclusive
                 	toIndex++;
                 	// use scope.docid (without revision) to determine groups
                 	if (prefix == null || !docid.startsWith(prefix)) {
                 		// construct a sublist for this previous group of docids
     					final List<String> subList = idList.subList(fromIndex, toIndex);
     	            	log.debug("Grouping docid prefix: " + prefix);
     					log.debug("subList.size: " + subList.size());
     					// add the task for this sublist
     					Runnable command = new Runnable() {
     						@Override
     						public void run() {
     							// generate based on this list
     				            try {
     				            	log.debug("Processing subList.size: " + subList.size());
     								SystemMetadataFactory.generateSystemMetadata(subList, includeOre, downloadData);
     								log.debug("Done processing subList.size: " + subList.size());
     							} catch (Exception e) {
     								throw new RuntimeException(e);
+    							}
+    						}
     					};
     					// execute the task
     					executor.execute(command);
     					// start at the end of this sublist
     					fromIndex = toIndex;
+                	}
                 	log.debug("docid: " + docid);
                 	// get the previous docid prefix
                 	String previousId = docid;
                 	prefix = previousId.substring(0, previousId.lastIndexOf("."));
+                }
                 // wait for executor to finish
                 executor.shutdown();
     		} catch (Exception e) {
     			String msg = "Problem generating missing system metadata: " + e.getMessage();
     			log.error(msg, e);
     			success = false;
     			throw new AdminException(msg);
+    		}
         	return success;
+        }
         public static void main(String [] ags){
             try {

Also available in: Unified diff

Project

General

Profile

Metacat

Revision 6964

Added by ben leinfelder almost 13 years ago