Revision 6980
Added by ben leinfelder almost 13 years ago
| src/edu/ucsb/nceas/metacat/admin/upgrade/GenerateSystemMetadata.java | ||
|---|---|---|
| 35 | 35 |
import org.apache.commons.logging.Log; |
| 36 | 36 |
import org.apache.commons.logging.LogFactory; |
| 37 | 37 |
|
| 38 |
import com.hazelcast.core.Hazelcast; |
|
| 39 |
|
|
| 40 | 38 |
import edu.ucsb.nceas.metacat.DBUtil; |
| 41 | 39 |
import edu.ucsb.nceas.metacat.admin.AdminException; |
| 42 | 40 |
import edu.ucsb.nceas.metacat.dataone.SystemMetadataFactory; |
| ... | ... | |
| 49 | 47 |
private static Log log = LogFactory.getLog(GenerateSystemMetadata.class); |
| 50 | 48 |
|
| 51 | 49 |
public boolean upgrade() throws AdminException {
|
| 52 |
return multiThreadUpgrade(); |
|
| 50 |
|
|
| 51 |
// do this in a thread too so that we don't have to hang the UI (web) |
|
| 52 |
ExecutorService executor = Executors.newSingleThreadExecutor(); |
|
| 53 |
Runnable command = new Runnable() {
|
|
| 54 |
@Override |
|
| 55 |
public void run() {
|
|
| 56 |
// just run it |
|
| 57 |
try {
|
|
| 58 |
boolean success = multiThreadUpgrade(); |
|
| 59 |
} catch (AdminException e) {
|
|
| 60 |
throw new RuntimeException(e); |
|
| 61 |
} |
|
| 62 |
} |
|
| 63 |
}; |
|
| 64 |
executor.execute(command); |
|
| 65 |
executor.shutdown(); |
|
| 66 |
|
|
| 67 |
// wait for it to finish before returning? |
|
| 68 |
boolean wait = true; |
|
| 69 |
if (wait) {
|
|
| 70 |
log.debug("Waiting for upgrade to complete");
|
|
| 71 |
try {
|
|
| 72 |
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS); |
|
| 73 |
} catch (InterruptedException e) {
|
|
| 74 |
AdminException ae = new AdminException(e.getMessage()); |
|
| 75 |
ae.initCause(e); |
|
| 76 |
throw ae; |
|
| 77 |
} |
|
| 78 |
log.debug("Done waiting for upgrade thread");
|
|
| 79 |
} |
|
| 80 |
|
|
| 81 |
return true; |
|
| 53 | 82 |
//return singleThreadUpgrade(); |
| 54 | 83 |
} |
| 55 | 84 |
|
| 56 | 85 |
/** |
| 57 |
* Use this current single thread to process each ID in the list |
|
| 58 |
* @return |
|
| 59 |
* @throws AdminException |
|
| 60 |
* @deprecated use the multithreaded version of this upgrade method |
|
| 61 |
*/ |
|
| 62 |
public boolean singleThreadUpgrade() throws AdminException {
|
|
| 63 |
boolean success = true; |
|
| 64 |
|
|
| 65 |
// do not include ORE or data, but can generate SystemMetadata for ALL records |
|
| 66 |
boolean includeOre = false; |
|
| 67 |
boolean downloadData = false; |
|
| 68 |
int serverLocation = -1; |
|
| 69 |
|
|
| 70 |
try {
|
|
| 71 |
// get list of ALL docids at ALL server locations |
|
| 72 |
List<String> idList = DBUtil.getAllDocidsByType(null, true, serverLocation); |
|
| 73 |
Collections.sort(idList); |
|
| 74 |
|
|
| 75 |
// generate based on this list |
|
| 76 |
SystemMetadataFactory.generateSystemMetadata(idList, includeOre, downloadData); |
|
| 77 |
} catch (Exception e) {
|
|
| 78 |
String msg = "Problem generating missing system metadata: " + e.getMessage(); |
|
| 79 |
log.error(msg, e); |
|
| 80 |
success = false; |
|
| 81 |
throw new AdminException(msg); |
|
| 82 |
} |
|
| 83 |
return success; |
|
| 84 |
} |
|
| 85 |
|
|
| 86 |
/** |
|
| 87 | 86 |
* Use multiple threads to process parts of the complete ID list concurrently |
| 88 | 87 |
* @return |
| 89 | 88 |
* @throws AdminException |
| ... | ... | |
| 106 | 105 |
// executor |
| 107 | 106 |
int availableProcessors = Runtime.getRuntime().availableProcessors(); |
| 108 | 107 |
int nThreads = availableProcessors * 1; |
| 108 |
//availableProcessors++; |
|
| 109 | 109 |
log.debug("Using nThreads: " + nThreads);
|
| 110 | 110 |
|
| 111 | 111 |
ExecutorService executor = Executors.newFixedThreadPool(nThreads); |
| 112 |
|
|
| 112 |
int taskCount = 0; |
|
| 113 |
|
|
| 113 | 114 |
// init HZ |
| 114 |
log.debug("STARTING HAZELCAST");
|
|
| 115 |
log.debug("Making sure Hazelcast is up");
|
|
| 115 | 116 |
HazelcastService.getInstance(); |
| 116 | 117 |
|
| 117 | 118 |
// chunk into groups |
| ... | ... | |
| 149 | 150 |
|
| 150 | 151 |
// execute the task |
| 151 | 152 |
executor.execute(command); |
| 153 |
taskCount++; |
|
| 152 | 154 |
|
| 153 | 155 |
// start at the end of this sublist |
| 154 | 156 |
fromIndex = toIndex; |
| ... | ... | |
| 163 | 165 |
|
| 164 | 166 |
} |
| 165 | 167 |
|
| 166 |
log.debug("DONE LAUNCHING THREADS");
|
|
| 168 |
log.info("done launching threaded tasks, count: " + taskCount);
|
|
| 167 | 169 |
|
| 168 | 170 |
// wait for executor to finish |
| 169 | 171 |
executor.shutdown(); |
| 170 | 172 |
|
| 171 |
// wait a long time |
|
| 173 |
// wait a long time |
|
| 174 |
log.debug("Waiting for all threads to complete");
|
|
| 172 | 175 |
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS); |
| 173 |
|
|
| 174 |
log.debug("DONE WAITING FOR THREAD");
|
|
| 176 |
log.debug("Done waiting for all threads to complete");
|
|
| 177 |
// now we are ready to be a data one node |
|
| 178 |
PropertyService.setProperty("dataone.systemmetadata.generated", Boolean.TRUE.toString());
|
|
| 175 | 179 |
|
| 176 |
|
|
| 177 | 180 |
} catch (Exception e) {
|
| 178 | 181 |
String msg = "Problem generating missing system metadata: " + e.getMessage(); |
| 179 | 182 |
log.error(msg, e); |
Also available in: Unified diff