Revision 2031
Added by Duane Costa about 20 years ago
Harvester.java | ||
---|---|---|
6 | 6 |
|
7 | 7 |
package edu.ucsb.nceas.metacat.harvesterClient; |
8 | 8 |
|
9 |
import java.io.*; |
|
10 |
import java.sql.*; |
|
11 |
import java.util.*; |
|
12 |
import javax.xml.parsers.*; |
|
13 |
import org.xml.sax.*; |
|
14 |
import org.xml.sax.helpers.*; |
|
9 |
import java.io.File; |
|
10 |
import java.io.FileInputStream; |
|
11 |
import java.io.IOException; |
|
12 |
import java.sql.Connection; |
|
13 |
import java.sql.DriverManager; |
|
14 |
import java.sql.ResultSet; |
|
15 |
import java.sql.SQLException; |
|
16 |
import java.sql.SQLWarning; |
|
17 |
import java.sql.Statement; |
|
18 |
import java.util.ArrayList; |
|
19 |
import java.text.SimpleDateFormat; |
|
20 |
import java.util.Date; |
|
21 |
import java.util.Properties; |
|
15 | 22 |
|
16 |
import edu.ucsb.nceas.metacat.client.*; |
|
23 |
import edu.ucsb.nceas.metacat.client.Metacat; |
|
24 |
import edu.ucsb.nceas.metacat.client.MetacatFactory; |
|
25 |
import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException; |
|
17 | 26 |
|
18 | 27 |
/** |
19 | 28 |
* Harvester is the main class for the Harvester application. The main |
... | ... | |
33 | 42 |
* Class methods |
34 | 43 |
*/ |
35 | 44 |
|
45 |
|
|
36 | 46 |
/** |
37 | 47 |
* Constructor. Creates a new instance of Harvester. |
38 | 48 |
*/ |
39 | 49 |
public Harvester() { |
40 | 50 |
} |
41 | 51 |
|
52 |
|
|
42 | 53 |
/** |
43 | 54 |
* Determines whether Harvester is running on a Win32 platform. Used |
44 |
* during development to aid in resolving platform dependencies.
|
|
55 |
* during development. |
|
45 | 56 |
* |
46 | 57 |
* @return true if this in Win32, false otherwise |
47 | 58 |
*/ |
... | ... | |
76 | 87 |
* Object fields |
77 | 88 |
*/ |
78 | 89 |
|
90 |
/** Database connection */ |
|
79 | 91 |
Connection conn; |
80 |
private Object harvestEndTime; |
|
81 |
private Object harvestLogList; |
|
82 |
private HarvestSiteSchedule[] harvestSiteScheduleList = |
|
83 |
new HarvestSiteSchedule[20]; |
|
84 |
private int harvestSiteScheduleIndex = 0; |
|
85 |
private Object harvestStartTime; |
|
92 |
|
|
93 |
/** Used during development to determine whether to connect to metacat */ |
|
94 |
private boolean connectToMetacat; |
|
95 |
|
|
96 |
/** Highest DETAIL_LOG_ID primary key in the HARVEST_DETAIL_LOG table */ |
|
97 |
private int detailLogID; |
|
98 |
|
|
99 |
/** Highest HARVEST_LOG_ID primary key in the HARVEST_LOG table */ |
|
100 |
private int harvestLogID; |
|
101 |
|
|
102 |
/** End time of this harvest session */ |
|
103 |
private Date harvestEndTime; |
|
104 |
|
|
105 |
/** List of HarvestLog objects. Stores log entries for report generation. */ |
|
106 |
private ArrayList harvestLogList = new ArrayList(); |
|
107 |
|
|
108 |
/** List of HarvestSiteSchedule objects */ |
|
109 |
private ArrayList harvestSiteScheduleList = new ArrayList(); |
|
110 |
|
|
111 |
/** Start time of this harvest session */ |
|
112 |
private Date harvestStartTime; |
|
113 |
|
|
114 |
/** Number of days to save log records. Any that are older are purged. */ |
|
115 |
int logPeriod; |
|
116 |
|
|
117 |
/** Metacat client object */ |
|
86 | 118 |
Metacat metacat; |
119 |
|
|
120 |
/** Loads harvester properties from configuration file */ |
|
87 | 121 |
Properties properties; |
88 | 122 |
|
89 | 123 |
|
90 | 124 |
/* |
91 | 125 |
* Object methods |
92 | 126 |
*/ |
127 |
|
|
128 |
|
|
129 |
/* |
|
130 |
Harvester harvester, |
|
131 |
Date harvestDate, |
|
132 |
int status, |
|
133 |
String message, |
|
134 |
String harvestOperationCode, |
|
135 |
int siteScheduleID, |
|
136 |
HarvestDocument harvestDocument, |
|
137 |
String errorMessage |
|
138 |
*/ |
|
139 |
/** |
|
140 |
* Creates a new HarvestLog object and adds it to the harvestLogList. |
|
141 |
* |
|
142 |
* @param status the status of the harvest operation |
|
143 |
* @param message the message text of the harvest operation |
|
144 |
* @param harvestOperationCode the harvest operation code |
|
145 |
* @param siteScheduleID the siteScheduleID for which this operation was |
|
146 |
* performed. 0 indicates that the operation did not |
|
147 |
* involve a particular harvest site. |
|
148 |
* @param harvestDocument the associated HarvestDocument object. May be null. |
|
149 |
* @param errorMessage additional error message pertaining to document |
|
150 |
* error. |
|
151 |
*/ |
|
152 |
void addLogEntry(int status, |
|
153 |
String message, |
|
154 |
String harvestOperationCode, |
|
155 |
int siteScheduleID, |
|
156 |
HarvestDocument harvestDocument, |
|
157 |
String errorMessage |
|
158 |
) { |
|
159 |
HarvestLog harvestLog; |
|
93 | 160 |
|
161 |
/* If there is no associated harvest document, call the basic constructor; |
|
162 |
* else call the extended constructor. |
|
163 |
*/ |
|
164 |
if (harvestDocument == null) { |
|
165 |
harvestLog = new HarvestLog(this, harvestStartTime, status, message, |
|
166 |
harvestOperationCode, siteScheduleID); |
|
167 |
} |
|
168 |
else { |
|
169 |
harvestLog = new HarvestLog(this, harvestStartTime, status, message, |
|
170 |
harvestOperationCode, siteScheduleID, |
|
171 |
harvestDocument, errorMessage); |
|
172 |
} |
|
173 |
|
|
174 |
harvestLogList.add(harvestLog); |
|
175 |
} |
|
176 |
|
|
177 |
|
|
94 | 178 |
/** |
95 | 179 |
* Determines whether Harvester should attempt to connect to Metacat. |
96 | 180 |
* Used during development and testing. |
... | ... | |
98 | 182 |
* @return true if Harvester should connect, otherwise false |
99 | 183 |
*/ |
100 | 184 |
boolean connectToMetacat () { |
101 |
boolean connectToMetacat = Harvester.isWin32() ? false : true; |
|
102 |
|
|
103 | 185 |
return connectToMetacat; |
104 | 186 |
} |
105 | 187 |
|
106 | 188 |
|
107 | 189 |
/** |
190 |
* Gets the current value of the detailLogID for storage as a primary key in |
|
191 |
* the DETAIL_LOG_ID field of the HARVEST_DETAIL_LOG table. |
|
192 |
* |
|
193 |
* @return the current value of the detailLogID |
|
194 |
*/ |
|
195 |
int getDetailLogID() { |
|
196 |
int currentValue = detailLogID; |
|
197 |
|
|
198 |
detailLogID++; |
|
199 |
return currentValue; |
|
200 |
} |
|
201 |
|
|
202 |
|
|
203 |
/** |
|
204 |
* Gets the current value of the harvestLogID for storage as a primary key in |
|
205 |
* the HARVEST_LOG_ID field of the HARVEST_LOG table. |
|
206 |
* |
|
207 |
* @return the current value of the detailLogID |
|
208 |
*/ |
|
209 |
int getHarvestLogID() { |
|
210 |
int currentValue = harvestLogID; |
|
211 |
|
|
212 |
harvestLogID++; |
|
213 |
return currentValue; |
|
214 |
} |
|
215 |
|
|
216 |
|
|
217 |
/** |
|
218 |
* Gets the maximum value of an integer field from a table. |
|
219 |
* |
|
220 |
* @param tableName the database table name |
|
221 |
* @param fieldName the field name of the integer field in the table |
|
222 |
* @return the maximum integer stored in the fieldName field of tableName |
|
223 |
*/ |
|
224 |
private int getMaxValue(String tableName, String fieldName) { |
|
225 |
int maxValue = 100; |
|
226 |
int fieldValue; |
|
227 |
String query = "SELECT " + fieldName + " FROM " + tableName; |
|
228 |
Statement stmt; |
|
229 |
|
|
230 |
try { |
|
231 |
stmt = conn.createStatement(); |
|
232 |
ResultSet rs = stmt.executeQuery(query); |
|
233 |
|
|
234 |
while (rs.next()) { |
|
235 |
fieldValue = rs.getInt(fieldName); |
|
236 |
maxValue = Math.max(maxValue, fieldValue); |
|
237 |
} |
|
238 |
|
|
239 |
stmt.close(); |
|
240 |
} |
|
241 |
catch(SQLException ex) { |
|
242 |
System.out.println("SQLException: " + ex.getMessage()); |
|
243 |
} |
|
244 |
|
|
245 |
return maxValue; |
|
246 |
} |
|
247 |
|
|
248 |
|
|
249 |
/** |
|
250 |
* Gets the minimum value of an integer field from a table. |
|
251 |
* |
|
252 |
* @param tableName the database table name |
|
253 |
* @param fieldName the field name of the integer field in the table |
|
254 |
* @return the minimum integer stored in the fieldName field of tableName |
|
255 |
*/ |
|
256 |
private int getMinValue(String tableName, String fieldName) { |
|
257 |
int minValue = 0; |
|
258 |
int fieldValue; |
|
259 |
String query = "SELECT " + fieldName + " FROM " + tableName; |
|
260 |
Statement stmt; |
|
261 |
|
|
262 |
try { |
|
263 |
stmt = conn.createStatement(); |
|
264 |
ResultSet rs = stmt.executeQuery(query); |
|
265 |
|
|
266 |
while (rs.next()) { |
|
267 |
fieldValue = rs.getInt(fieldName); |
|
268 |
|
|
269 |
if (minValue == 0) { |
|
270 |
minValue = fieldValue; |
|
271 |
} |
|
272 |
else { |
|
273 |
minValue = Math.min(minValue, fieldValue); |
|
274 |
} |
|
275 |
} |
|
276 |
|
|
277 |
stmt.close(); |
|
278 |
} |
|
279 |
catch(SQLException ex) { |
|
280 |
System.out.println("SQLException: " + ex.getMessage()); |
|
281 |
} |
|
282 |
|
|
283 |
return minValue; |
|
284 |
} |
|
285 |
|
|
286 |
|
|
287 |
/** |
|
108 | 288 |
* For every Harvest site schedule in the database, harvest the |
109 | 289 |
* documents for that site if they are due to be harvested. |
110 | 290 |
* |
... | ... | |
115 | 295 |
private void harvest() { |
116 | 296 |
HarvestSiteSchedule harvestSiteSchedule; |
117 | 297 |
|
118 |
for (int i = 0; i < harvestSiteScheduleList.length; i++) { |
|
119 |
harvestSiteSchedule = harvestSiteScheduleList[i]; |
|
120 |
if (harvestSiteSchedule != null) { |
|
121 |
harvestSiteSchedule.printOutput(); |
|
122 |
harvestSiteSchedule.harvestDocumentList(); |
|
123 |
} |
|
298 |
for (int i = 0; i < harvestSiteScheduleList.size(); i++) { |
|
299 |
harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i); |
|
300 |
harvestSiteSchedule.harvestDocumentList(); |
|
124 | 301 |
} |
125 |
|
|
126 |
reportToAdministrator(); |
|
127 | 302 |
} |
128 | 303 |
|
129 | 304 |
|
130 | 305 |
/** |
131 |
* Loads Harvester properties |
|
306 |
* Initializes the detailLogID and harvestLogID values to their current |
|
307 |
* maximums + 1. |
|
132 | 308 |
*/ |
309 |
private void initLogIDs() { |
|
310 |
detailLogID = getMaxValue("HARVEST_DETAIL_LOG", "DETAIL_LOG_ID") + 1; |
|
311 |
harvestLogID = getMaxValue("HARVEST_LOG", "HARVEST_LOG_ID") + 1; |
|
312 |
} |
|
313 |
|
|
314 |
|
|
315 |
/** |
|
316 |
* Loads Harvester properties from a configuration file. |
|
317 |
*/ |
|
133 | 318 |
private void loadProperties() { |
134 | 319 |
String homedir = System.getProperty("user.home"); |
135 | 320 |
File configfile = new File(homedir, "harvester.properties"); |
... | ... | |
141 | 326 |
properties.list(System.out); |
142 | 327 |
} |
143 | 328 |
catch (IOException e) { |
144 |
System.err.println("IOException: " + e.getMessage());
|
|
329 |
System.out.println("IOException: " + e.getMessage());
|
|
145 | 330 |
System.exit(1); |
146 | 331 |
} |
147 | 332 |
} |
333 |
|
|
334 |
|
|
335 |
void printHarvestSiteSchedule(int siteScheduleID) { |
|
336 |
HarvestSiteSchedule harvestSiteSchedule; |
|
337 |
|
|
338 |
for (int i = 0; i < harvestSiteScheduleList.size(); i++) { |
|
339 |
harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i); |
|
340 |
if (harvestSiteSchedule.siteScheduleID == siteScheduleID) { |
|
341 |
harvestSiteSchedule.printOutput(); |
|
342 |
} |
|
343 |
} |
|
344 |
} |
|
345 |
|
|
346 |
|
|
347 |
/** |
|
348 |
* Prunes old records from the HARVEST_DETAIL_LOG table. Records are |
|
349 |
* removed if the HARVEST_LOG_ID foreign key is less than the lowest |
|
350 |
* HARVEST_LOG_ID primary key in the HARVEST_LOG table. |
|
351 |
*/ |
|
352 |
private void pruneHarvestDetailLog() { |
|
353 |
String deleteString; |
|
354 |
int minHarvestLogID; |
|
355 |
int recordsDeleted; |
|
356 |
Statement stmt; |
|
148 | 357 |
|
358 |
minHarvestLogID = getMinValue("HARVEST_LOG", "HARVEST_LOG_ID"); |
|
359 |
deleteString = "DELETE FROM HARVEST_DETAIL_LOG WHERE HARVEST_LOG_ID < " + |
|
360 |
minHarvestLogID; |
|
149 | 361 |
|
362 |
try { |
|
363 |
System.out.print("Pruning log entries from HARVEST_DETAIL_LOG: "); |
|
364 |
stmt = conn.createStatement(); |
|
365 |
recordsDeleted = stmt.executeUpdate(deleteString); |
|
366 |
System.out.println(recordsDeleted + " records deleted"); |
|
367 |
stmt.close(); |
|
368 |
} |
|
369 |
catch(SQLException e) { |
|
370 |
System.out.println("SQLException: " + e.getMessage()); |
|
371 |
} |
|
372 |
} |
|
373 |
|
|
374 |
|
|
150 | 375 |
/** |
376 |
* Prunes old records from the HARVEST_LOG table. Records are removed if |
|
377 |
* their HARVEST_DATE is older than a given number of days, as stored in the |
|
378 |
* logPeriod object field. |
|
379 |
*/ |
|
380 |
private void pruneHarvestLog() { |
|
381 |
long currentTime = harvestStartTime.getTime(); // time in milliseconds |
|
382 |
Date dateLastLog; // Prune everything prior to this date |
|
383 |
String deleteString; |
|
384 |
long delta; |
|
385 |
final long millisecondsPerDay = (1000 * 60 * 60 * 24); |
|
386 |
int recordsDeleted; |
|
387 |
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MMM-yyyy"); |
|
388 |
String dateString; |
|
389 |
Statement stmt; |
|
390 |
long timeLastLog = 0; |
|
391 |
|
|
392 |
delta = logPeriod * millisecondsPerDay; |
|
393 |
deleteString = "DELETE FROM HARVEST_LOG WHERE HARVEST_DATE < "; |
|
394 |
timeLastLog = currentTime - delta; |
|
395 |
dateLastLog = new Date(timeLastLog); |
|
396 |
dateString = "'" + simpleDateFormat.format(dateLastLog) + "'"; |
|
397 |
deleteString += dateString; |
|
398 |
|
|
399 |
try { |
|
400 |
System.out.print("Pruning log entries from HARVEST_LOG: "); |
|
401 |
stmt = conn.createStatement(); |
|
402 |
recordsDeleted = stmt.executeUpdate(deleteString); |
|
403 |
System.out.println(recordsDeleted + " records deleted"); |
|
404 |
stmt.close(); |
|
405 |
} |
|
406 |
catch (SQLException e) { |
|
407 |
System.out.println("SQLException: " + e.getMessage()); |
|
408 |
} |
|
409 |
} |
|
410 |
|
|
411 |
|
|
412 |
/** |
|
151 | 413 |
* Reads the HARVEST_SITE_SCHEDULE table in the database, creating |
152 | 414 |
* a HarvestSiteSchedule object for each row in the table. |
153 | 415 |
*/ |
... | ... | |
162 | 424 |
String dateNextHarvest; |
163 | 425 |
String documentListURL; |
164 | 426 |
String ldapDN; |
165 |
String ldapPassword;
|
|
427 |
String ldapPwd;
|
|
166 | 428 |
int siteScheduleID; |
167 | 429 |
String unit; |
168 | 430 |
int updateFrequency; |
... | ... | |
190 | 452 |
siteScheduleID = rs.getInt("SITE_SCHEDULE_ID"); |
191 | 453 |
documentListURL = rs.getString("DOCUMENTLISTURL"); |
192 | 454 |
ldapDN = rs.getString("LDAPDN"); |
193 |
// ldapPassword = rs.getString("LDAPPASSWORD"); |
|
194 |
ldapPassword = "ntre4dc"; |
|
455 |
ldapPwd = rs.getString("LDAPPWD"); |
|
195 | 456 |
dateNextHarvest = rs.getString("DATENEXTHARVEST"); |
196 | 457 |
dateLastHarvest = rs.getString("DATELASTHARVEST"); |
197 | 458 |
updateFrequency = rs.getInt("UPDATEFREQUENCY"); |
... | ... | |
213 | 474 |
} |
214 | 475 |
} |
215 | 476 |
|
216 |
harvestSiteSchedule = new HarvestSiteSchedule( |
|
217 |
this, |
|
477 |
harvestSiteSchedule = new HarvestSiteSchedule(this, |
|
218 | 478 |
siteScheduleID, |
219 | 479 |
documentListURL, |
220 | 480 |
ldapDN, |
221 |
ldapPassword,
|
|
481 |
ldapPwd,
|
|
222 | 482 |
dateNextHarvest, |
223 | 483 |
dateLastHarvest, |
224 | 484 |
updateFrequency, |
225 | 485 |
unit, |
226 | 486 |
contactEmail |
227 | 487 |
); |
228 |
|
|
229 |
harvestSiteScheduleList[harvestSiteScheduleIndex] = harvestSiteSchedule; |
|
230 |
harvestSiteScheduleIndex++; |
|
488 |
harvestSiteScheduleList.add(harvestSiteSchedule); |
|
231 | 489 |
} |
232 |
} |
|
490 |
|
|
491 |
rs.close(); |
|
492 |
stmt.close(); |
|
493 |
} |
|
233 | 494 |
catch (SQLException e) { |
234 | 495 |
System.out.println("Database access failed " + e); |
235 | 496 |
System.exit(1); |
... | ... | |
253 | 514 |
private void shutdown() { |
254 | 515 |
// Log shutdown operation |
255 | 516 |
System.out.println("Shutting Down Harvester"); |
517 |
addLogEntry(0, "Shutting Down Harvester", "HarvesterShutdown", 0, null, ""); |
|
518 |
pruneHarvestLog(); |
|
519 |
pruneHarvestDetailLog(); |
|
256 | 520 |
|
257 | 521 |
try { |
258 | 522 |
// Close the database connection |
... | ... | |
262 | 526 |
catch (SQLException e) { |
263 | 527 |
System.out.println("Database access failed " + e); |
264 | 528 |
} |
529 |
|
|
530 |
writeHarvestLog(); |
|
531 |
reportToAdministrator(); |
|
265 | 532 |
} |
266 | 533 |
|
267 | 534 |
|
... | ... | |
269 | 536 |
* Initializes Harvester at startup. Connects to the database and to Metacat. |
270 | 537 |
*/ |
271 | 538 |
private void startup() { |
539 |
Boolean ctm; |
|
272 | 540 |
String dbDriver; |
541 |
Integer lp; |
|
273 | 542 |
String metacatURL; |
274 | 543 |
String osName = Harvester.isWin32() ? "Windows" : "Unix"; |
275 | 544 |
String password; |
276 |
// String response;
|
|
545 |
//String response;
|
|
277 | 546 |
String sessionId; |
278 | 547 |
String url; |
279 | 548 |
String user; |
... | ... | |
283 | 552 |
// Log startup operation |
284 | 553 |
System.out.println("*****************************************************"); |
285 | 554 |
System.out.println("Starting Up Harvester"); |
286 |
System.out.println("OS is " + osName); |
|
287 |
|
|
288 | 555 |
loadProperties(); |
289 |
|
|
556 |
ctm = Boolean.valueOf(properties.getProperty("connectToMetacat", "true")); |
|
557 |
connectToMetacat = ctm.booleanValue(); |
|
290 | 558 |
dbDriver = properties.getProperty("dbDriver"); |
559 |
try { |
|
560 |
lp = Integer.valueOf(properties.getProperty("logPeriod", "90")); |
|
561 |
logPeriod = lp.intValue(); |
|
562 |
} |
|
563 |
catch (NumberFormatException e) { |
|
564 |
System.err.println("NumberFormatException: Error parsing logPeriod " + |
|
565 |
logPeriod + e.getMessage()); |
|
566 |
System.err.println("Defaulting to logPeriod of 90 days"); |
|
567 |
logPeriod = 90; |
|
568 |
} |
|
569 |
metacatURL = properties.getProperty("metacatURL"); |
|
570 |
password = properties.getProperty("password"); |
|
291 | 571 |
url = properties.getProperty("url"); |
292 | 572 |
user = properties.getProperty("user"); |
293 |
password = properties.getProperty("password"); |
|
294 |
metacatURL = properties.getProperty("metacatURL"); |
|
295 | 573 |
|
296 | 574 |
// Load the jdbc driver |
297 | 575 |
try { |
... | ... | |
325 | 603 |
System.out.println("Database access failed " + e); |
326 | 604 |
System.exit(1); |
327 | 605 |
} |
606 |
|
|
607 |
initLogIDs(); |
|
608 |
harvestStartTime = new Date(); |
|
609 |
addLogEntry(0, "Starting Up Harvester", "HarvesterStartup", 0, null, ""); |
|
328 | 610 |
|
329 | 611 |
if (connectToMetacat()) { |
330 | 612 |
try { |
... | ... | |
338 | 620 |
System.out.println("Metacat connection failed." + e.getMessage()); |
339 | 621 |
} |
340 | 622 |
} |
341 |
else { |
|
342 |
System.out.println("Not connecting to Metacat"); |
|
343 |
} |
|
344 | 623 |
} |
345 | 624 |
|
346 | 625 |
|
... | ... | |
348 | 627 |
* Writes one or more log entries to the HARVEST_LOG table. |
349 | 628 |
*/ |
350 | 629 |
private void writeHarvestLog() { |
630 |
HarvestLog harvestLog; |
|
631 |
|
|
632 |
for (int i = 0; i < harvestLogList.size(); i++) { |
|
633 |
harvestLog = (HarvestLog) harvestLogList.get(i); |
|
634 |
harvestLog.printOutput(); |
|
635 |
} |
|
351 | 636 |
} |
352 | 637 |
|
353 | 638 |
} |
Also available in: Unified diff
Additional development of Harvester implementation