Project

General

Profile

« Previous | Next » 

Revision 6964

multithreaded implementation for processing docids for system metadata generation.
need to investigate ant/junit running that deadlocks hazelcast (config?)

View differences:

src/edu/ucsb/nceas/metacat/dataone/SystemMetadataFactory.java
129 129
			PropertyNotFoundException, BaseException, NoSuchAlgorithmException,
130 130
			JiBXException, AccessControlException, HandlerException, SAXException, AccessException {
131 131
		
132
		logMetacat.debug("createSystemMetadata() called.");
133
		logMetacat.debug("provided localId: " + localId);
132
		logMetacat.debug("createSystemMetadata() called for localId " + localId);
134 133

  
135 134
		// check for system metadata
136 135
		SystemMetadata sysMeta = null;
......
145 144
			guid = IdentifierManager.getInstance().getGUID(accNum.getDocid(), rev);
146 145
		} catch (McdbDocNotFoundException dnfe) {
147 146
			// otherwise create the mapping
148
			logMetacat.debug("There was a problem getting the guid from "
149
							+ "the given localId (docid and revision). The error message was: "
150
							+ dnfe.getMessage());
151
			logMetacat.debug("No guid in the identifier table.  adding it for " + localId);
147
			logMetacat.debug("No guid found in the identifier table.  Creating mapping for " + localId);
152 148
			IdentifierManager.getInstance().createMapping(localId, localId);
153
			logMetacat.debug("Mapping created for " + localId);
154
			logMetacat.debug("accessionNumber: " + accNum);
155
			guid = IdentifierManager.getInstance().getGUID(accNum.getDocid(), rev);
156
			
149
			guid = IdentifierManager.getInstance().getGUID(accNum.getDocid(), rev);			
157 150
		}
158 151
		
159 152
		// look up existing system metadata if it exists
160 153
		Identifier identifier = new Identifier();
161 154
		identifier.setValue(guid);
162 155
		try {
156
			logMetacat.debug("Using hazelcast to get system metadata");
163 157
			sysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(identifier);
164 158
			// TODO: if this is the case, we could return here -- what else do we gain?
165 159
			if (!updateExisting ) {
166 160
				return sysMeta;
167 161
			}
168 162
		} catch (Exception e) {
163
			logMetacat.debug("No system metadata found in hz: " + e.getMessage());
164

  
169 165
			// create system metadata
170 166
			sysMeta = new SystemMetadata();
171 167
			sysMeta.setIdentifier(identifier);
......
183 179
		try {
184 180
			fmtid = ObjectFormatCache.getInstance().getFormat(doctype).getFormatId();
185 181
		} catch (NotFound nfe) {
186

  
187
			try {
188
				// format is not registered, use default
189
				if (doctype.trim().equals("BIN")) {
190
					fmtid = ObjectFormatCache.getInstance().getFormat(
191
							"application/octet-stream").getFormatId();
192

  
193
				} else {
194
					fmtid = ObjectFormatCache.getInstance().getFormat(
195
							"text/plain").getFormatId();
196
				}
197

  
198
			} catch (NotFound nf) {
199
				logMetacat.error("There was a problem getting the default format "
200
								+ "from the ObjectFormatCache: "
201
								+ nf.getMessage());
202
				throw nf;
182
			// format is not registered, use default
183
			if (doctype.trim().equals("BIN")) {
184
				fmtid = ObjectFormatCache.getInstance().getFormat("application/octet-stream").getFormatId();
185
			} else {
186
				fmtid = ObjectFormatCache.getInstance().getFormat("text/plain").getFormatId();
203 187
			}
204

  
205 188
		}
206 189

  
207 190
		sysMeta.setFormatId(fmtid);
......
440 423
							// we'll have to generate it	
441 424
							if (dataSysMeta == null) {
442 425
								// System metadata for data doesn't exist yet, so create it
443
								logMetacat.debug("There was not an existing system metadata document for " + dataDocLocalId);
444
								logMetacat.debug("Creating a system metadata " + "document for " + dataDocLocalId);
426
								logMetacat.debug("No exisiting SystemMetdata found, creating for: " + dataDocLocalId);
445 427
								dataSysMeta = createSystemMetadata(dataDocLocalId, includeORE, false);
446 428

  
447 429
								// now look it up again
......
459 441
							}
460 442
							
461 443
							// set object format for the data file
462
							logMetacat.debug("Updating system metadata for "
463
											+ dataGuid.getValue() + " to "
464
											+ dataDocMimeType);
444
							logMetacat.debug("Updating system metadata for " + dataGuid.getValue() + " to " + dataDocMimeType);
465 445
							try {
466 446
								ObjectFormatIdentifier fmt = 
467 447
									ObjectFormatCache.getInstance().getFormat(dataDocMimeType).getFormatId();
......
579 559
     * @param idList
580 560
     * @param includeOre
581 561
     * @param downloadData
582
     * @throws ServiceFailure
583
     * @throws McdbDocNotFoundException
584 562
     * @throws PropertyNotFoundException
585
     * @throws InvalidToken
586
     * @throws NotAuthorized
587
     * @throws NotFound
588
     * @throws NotImplemented
589
     * @throws InvalidRequest
590 563
     * @throws NoSuchAlgorithmException
591 564
     * @throws AccessionNumberException
592 565
     * @throws SQLException
566
	 * @throws SAXException 
567
	 * @throws HandlerException 
568
	 * @throws JiBXException 
569
	 * @throws BaseException 
570
	 * @throws ParseLSIDException 
571
	 * @throws InsufficientKarmaException 
572
	 * @throws ClassNotFoundException 
573
	 * @throws IOException 
574
	 * @throws McdbException 
575
	 * @throws AccessException 
576
	 * @throws AccessControlException 
593 577
     */
594 578
    public static void generateSystemMetadata(List<String> idList, boolean includeOre, boolean downloadData) 
595
    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized, 
596
    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
579
    throws PropertyNotFoundException, NoSuchAlgorithmException, AccessionNumberException, SQLException, AccessControlException, AccessException, McdbException, IOException, ClassNotFoundException, InsufficientKarmaException, ParseLSIDException, BaseException, JiBXException, HandlerException, SAXException 
597 580
    {
598 581
        
599 582
        for (String localId : idList) { 
600 583
            //for each id, add a system metadata doc
601
        	try {
602
        		logMetacat.debug("generating system metadata for " + localId);
603
        		generateSystemMetadata(localId, includeOre, downloadData);
604
        	} catch (Exception e) {
605
        		logMetacat.error("Error generating system metadata for: " + localId, e);
606
			}
584
        	generateSystemMetadata(localId, includeOre, downloadData);
607 585
        }
608
        logMetacat.info("done generating system metadata");
586
        logMetacat.info("done generating system metadata for given list");
609 587
    }
610 588
    
611 589

  
......
616 594
     * @param localId
617 595
     * @param includeOre
618 596
     * @param downloadData
619
     * @throws ServiceFailure
620
     * @throws McdbDocNotFoundException
621 597
     * @throws PropertyNotFoundException
622
     * @throws InvalidToken
623
     * @throws NotAuthorized
624
     * @throws NotFound
625
     * @throws NotImplemented
626
     * @throws InvalidRequest
627 598
     * @throws NoSuchAlgorithmException
628 599
     * @throws AccessionNumberException
629 600
     * @throws SQLException
630
     * @throws InvalidSystemMetadata
601
     * @throws SAXException 
602
     * @throws HandlerException 
603
     * @throws JiBXException 
604
     * @throws BaseException 
605
     * @throws ParseLSIDException 
606
     * @throws InsufficientKarmaException 
607
     * @throws ClassNotFoundException 
608
     * @throws IOException 
609
     * @throws McdbException 
610
     * @throws AccessException 
611
     * @throws AccessControlException 
631 612
     */
632 613
    protected static void generateSystemMetadata(String localId, boolean includeOre, boolean downloadData) 
633
    throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized,
634
    NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException, InvalidSystemMetadata 
614
    throws PropertyNotFoundException, NoSuchAlgorithmException, AccessionNumberException, SQLException, AccessControlException, AccessException, McdbException, IOException, ClassNotFoundException, InsufficientKarmaException, ParseLSIDException, BaseException, JiBXException, HandlerException, SAXException 
635 615
    {
636
    	logMetacat.debug("generateSystemMetadata() called.");
637 616
    	logMetacat.debug("Creating SystemMetadata for localId " + localId);
638 617
        SystemMetadata sm = null;
639 618

  
640 619
        //generate required system metadata fields from the document
641
        try {
642
        	sm = SystemMetadataFactory.createSystemMetadata(localId, includeOre, downloadData);
643
        } catch (Exception e1) {
644
        	e1.printStackTrace();
645
        	ServiceFailure sf = new ServiceFailure("00","Exception in generateSystemMetadata: " +
646
        			e1.getMessage());
647
        	sf.setStackTrace(e1.getStackTrace());
648
        	throw sf;
649
        }
650
        
620
    	sm = SystemMetadataFactory.createSystemMetadata(localId, includeOre, downloadData);
621
    	
651 622
        //insert the systemmetadata object or just update it as needed
652 623
        boolean exists = IdentifierManager.getInstance().systemMetadataExists(sm.getIdentifier().getValue());
653 624
        if (!exists) {
src/edu/ucsb/nceas/metacat/admin/upgrade/GenerateSystemMetadata.java
28 28

  
29 29
import java.util.Collections;
30 30
import java.util.List;
31
import java.util.concurrent.ExecutorService;
32
import java.util.concurrent.Executors;
31 33

  
32 34
import org.apache.commons.logging.Log;
33 35
import org.apache.commons.logging.LogFactory;
......
43 45
	private static Log log = LogFactory.getLog(GenerateSystemMetadata.class);
44 46
	
45 47
    public boolean upgrade() throws AdminException {
48
        return multiThreadUpgrade();
49
        //return singleThreadUpgrade();
50
    }
51
    
52
    /**
53
     * Use this current single thread to process each ID in the list
54
     * @return
55
     * @throws AdminException
56
     * @deprecated use the multithreaded version of this upgrade method
57
     */
58
    public boolean singleThreadUpgrade() throws AdminException {
46 59
        boolean success = true;
47 60
        
48 61
        // do not include ORE or data, but can generate SystemMetadata for ALL records
......
66 79
    	return success;
67 80
    }
68 81
    
82
    /**
83
     * Use multiple threads to process parts of the complete ID list concurrently
84
     * @return
85
     * @throws AdminException
86
     */
87
    public boolean multiThreadUpgrade() throws AdminException {
88
    	
89
        boolean success = true;
90
        
91
        // do not include ORE or data, but can generate SystemMetadata for ALL records
92
        final boolean includeOre = false;
93
        final boolean downloadData = false;
94
        int serverLocation = -1;
95
        
96
        try {
97
        	
98
        	// get list of ALL docids at ALL server locations
99
            List<String> idList = DBUtil.getAllDocidsByType(null, true, serverLocation);
100
            Collections.sort(idList);
101
            
102
            // executor
103
            int availableProcessors = Runtime.getRuntime().availableProcessors();
104
            int nThreads = availableProcessors * 1;
105
        	log.debug("Using nThreads: " + nThreads);
106

  
107
            ExecutorService executor = Executors.newFixedThreadPool(nThreads);
108
            
109
            // chunk into groups
110
			int fromIndex = 0;
111
            int toIndex = 0;
112
            String prefix = null;
113
            for (String docid: idList) {
114

  
115
            	// increment the next entry, exclusive
116
            	toIndex++;
117
            	
118
            	// use scope.docid (without revision) to determine groups
119
            	if (prefix == null || !docid.startsWith(prefix)) {
120
            		
121
            		// construct a sublist for this previous group of docids
122
					final List<String> subList = idList.subList(fromIndex, toIndex);
123
	            	log.debug("Grouping docid prefix: " + prefix);
124
					log.debug("subList.size: " + subList.size());
125
					
126
					// add the task for this sublist
127
					Runnable command = new Runnable() {
128
						@Override
129
						public void run() {
130
							// generate based on this list
131
				            try {
132
				            	log.debug("Processing subList.size: " + subList.size());
133
								SystemMetadataFactory.generateSystemMetadata(subList, includeOre, downloadData);
134
								log.debug("Done processing subList.size: " + subList.size());
135
								
136
							} catch (Exception e) {
137
								throw new RuntimeException(e);
138
							}
139
						}
140
					};
141
					
142
					// execute the task 
143
					executor.execute(command);
144
					
145
					// start at the end of this sublist
146
					fromIndex = toIndex;
147

  
148
            	}
149

  
150
            	log.debug("docid: " + docid);
151

  
152
            	// get the previous docid prefix
153
            	String previousId = docid;
154
            	prefix = previousId.substring(0, previousId.lastIndexOf("."));
155
				
156
            }
157
            
158
            // wait for executor to finish
159
            executor.shutdown();
160
            
161
		} catch (Exception e) {
162
			String msg = "Problem generating missing system metadata: " + e.getMessage();
163
			log.error(msg, e);
164
			success = false;
165
			throw new AdminException(msg);
166
		}
167
    	return success;
168
    }
169
    
69 170
    public static void main(String [] ags){
70 171

  
71 172
        try {

Also available in: Unified diff