Revision 4125
Added by daigle almost 16 years ago
| Harvester.java | ||
|---|---|---|
| 25 | 25 |
package edu.ucsb.nceas.metacat.harvesterClient; |
| 26 | 26 |
|
| 27 | 27 |
import com.oreilly.servlet.MailMessage; |
| 28 |
import edu.ucsb.nceas.utilities.Options; |
|
| 29 |
import java.io.File; |
|
| 30 | 28 |
import java.io.IOException; |
| 31 | 29 |
import java.io.PrintStream; |
| 32 | 30 |
import java.sql.Connection; |
| ... | ... | |
| 43 | 41 |
import edu.ucsb.nceas.metacat.client.Metacat; |
| 44 | 42 |
import edu.ucsb.nceas.metacat.client.MetacatFactory; |
| 45 | 43 |
import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException; |
| 44 |
import edu.ucsb.nceas.metacat.service.PropertyService; |
|
| 45 |
import edu.ucsb.nceas.metacat.service.ServiceException; |
|
| 46 | 46 |
import edu.ucsb.nceas.metacat.util.SystemUtil; |
| 47 |
import edu.ucsb.nceas.utilities.PropertyNotFoundException; |
|
| 47 | 48 |
|
| 48 | 49 |
/** |
| 49 | 50 |
* Harvester is the main class for the Harvester application. The main |
| ... | ... | |
| 59 | 60 |
*/ |
| 60 | 61 |
private static final String CONFIG_DIR = "../../build/war/WEB-INF"; |
| 61 | 62 |
private static final String CONFIG_DIR_TEST = "./build/war/WEB-INF"; |
| 62 |
private static final String CONFIG_NAME = "metacat.properties"; |
|
| 63 |
// private static final String CONFIG_NAME = "metacat.properties";
|
|
| 63 | 64 |
public static final String filler = "*"; |
| 64 | 65 |
private static boolean keepRunning = true; |
| 65 | 66 |
public static final String marker = |
| 66 | 67 |
"*****************************************************************************"; |
| 67 |
public static Options options = null;
|
|
| 68 |
// public static PropertyService propertyService = null;
|
|
| 68 | 69 |
private static String schemaLocation = null; |
| 69 | 70 |
|
| 70 | 71 |
|
| ... | ... | |
| 83 | 84 |
/** |
| 84 | 85 |
* Loads Harvester options from a configuration file. |
| 85 | 86 |
*/ |
| 86 |
public static void loadOptions(boolean test) {
|
|
| 87 |
public static void loadProperties(boolean test) {
|
|
| 87 | 88 |
String configDir = test ? CONFIG_DIR_TEST : CONFIG_DIR; |
| 88 |
File propertyFile = new File(configDir, CONFIG_NAME); |
|
| 89 | 89 |
|
| 90 | 90 |
try {
|
| 91 |
options = Options.initialize(propertyFile);
|
|
| 91 |
PropertyService.getInstance(configDir);
|
|
| 92 | 92 |
} |
| 93 |
catch (IOException e) {
|
|
| 94 |
System.out.println("Error in loading options: " + e.getMessage());
|
|
| 93 |
catch (ServiceException e) {
|
|
| 94 |
System.out.println("Error in loading properties: " + e.getMessage());
|
|
| 95 | 95 |
} |
| 96 | 96 |
} |
| 97 | 97 |
|
| 98 | 98 |
|
| 99 |
/** |
|
| 100 |
* Harvester main method. |
|
| 101 |
* |
|
| 102 |
* @param args the command line arguments |
|
| 103 |
* @throws SAXException |
|
| 104 |
* @throws IOException |
|
| 105 |
* @throws ParserConfigurationException |
|
| 106 |
*/ |
|
| 107 |
public static void main(String[] args) {
|
|
| 108 |
Integer delayDefault = new Integer(0); // Default number of hours delay |
|
| 109 |
int delay = delayDefault.intValue(); // Delay in hours before first harvest |
|
| 110 |
Integer d; // Used for determining delay |
|
| 111 |
long delta; // endTime - startTime |
|
| 112 |
long endTime; // time that a harvest completes |
|
| 113 |
Harvester harvester; // object for a single harvest run |
|
| 114 |
Integer maxHarvestsDefault = new Integer(0); // Default max harvests |
|
| 115 |
int maxHarvests = maxHarvestsDefault.intValue(); // Max number of harvests |
|
| 116 |
Integer mh; // used in determining max harvests |
|
| 117 |
int nHarvests = 0; // counts the number of harvest runs |
|
| 118 |
final long oneHour = (60 * 60 * 1000); // milliseconds in one hour |
|
| 119 |
Integer periodDefault = new Integer(24); // Default hours between harvests |
|
| 120 |
int period = periodDefault.intValue(); // Hours between harvests |
|
| 121 |
Integer p; // Used in determining the period |
|
| 122 |
long startTime; // time that a harvest run starts |
|
| 123 |
boolean test = false; // Passed to loadOption() |
|
| 124 |
|
|
| 125 |
if (args.length > 0) {
|
|
| 126 |
schemaLocation = args[0]; |
|
| 127 |
System.err.println("schemaLocation: " + schemaLocation);
|
|
| 128 |
|
|
| 129 |
try {
|
|
| 130 |
Thread.sleep(10000); |
|
| 131 |
} |
|
| 132 |
catch (InterruptedException e) {
|
|
| 133 |
e.printStackTrace(); |
|
| 134 |
} |
|
| 135 |
} |
|
| 99 |
/** |
|
| 100 |
* Harvester main method. |
|
| 101 |
* |
|
| 102 |
* @param args |
|
| 103 |
* the command line arguments |
|
| 104 |
* @throws SAXException |
|
| 105 |
* @throws IOException |
|
| 106 |
* @throws ParserConfigurationException |
|
| 107 |
*/ |
|
| 108 |
public static void main(String[] args) {
|
|
| 109 |
Integer delayDefault = new Integer(0); // Default number of hours delay |
|
| 110 |
int delay = delayDefault.intValue(); // Delay in hours before first |
|
| 111 |
// harvest |
|
| 112 |
Integer d; // Used for determining delay |
|
| 113 |
long delta; // endTime - startTime |
|
| 114 |
long endTime; // time that a harvest completes |
|
| 115 |
Harvester harvester; // object for a single harvest run |
|
| 116 |
Integer maxHarvestsDefault = new Integer(0); // Default max harvests |
|
| 117 |
int maxHarvests = maxHarvestsDefault.intValue(); // Max number of |
|
| 118 |
// harvests |
|
| 119 |
Integer mh; // used in determining max harvests |
|
| 120 |
int nHarvests = 0; // counts the number of harvest runs |
|
| 121 |
final long oneHour = (60 * 60 * 1000); // milliseconds in one hour |
|
| 122 |
Integer periodDefault = new Integer(24); // Default hours between |
|
| 123 |
// harvests |
|
| 124 |
int period = periodDefault.intValue(); // Hours between harvests |
|
| 125 |
Integer p; // Used in determining the period |
|
| 126 |
long startTime; // time that a harvest run starts |
|
| 127 |
boolean test = false; // Passed to loadOption() |
|
| 136 | 128 |
|
| 137 |
System.out.println(marker);
|
|
| 138 |
System.out.println("Starting Harvester");
|
|
| 139 |
Harvester.loadOptions(test);
|
|
| 129 |
if (args.length > 0) {
|
|
| 130 |
schemaLocation = args[0];
|
|
| 131 |
System.err.println("schemaLocation: " + schemaLocation);
|
|
| 140 | 132 |
|
| 141 |
// Parse the delay property. Use default if necessary. |
|
| 142 |
try {
|
|
| 143 |
d = Integer.valueOf(options.getOption("delay"));
|
|
| 144 |
delay = d.intValue(); |
|
| 145 |
} |
|
| 146 |
catch (NumberFormatException e) {
|
|
| 147 |
System.out.println("NumberFormatException: Error parsing delay: " +
|
|
| 148 |
e.getMessage()); |
|
| 149 |
System.out.println("Defaulting to delay=" + delayDefault);
|
|
| 150 |
delay = delayDefault.intValue(); |
|
| 151 |
} |
|
| 133 |
try {
|
|
| 134 |
Thread.sleep(10000); |
|
| 135 |
} catch (InterruptedException e) {
|
|
| 136 |
e.printStackTrace(); |
|
| 137 |
} |
|
| 138 |
} |
|
| 152 | 139 |
|
| 153 |
// Parse the maxHarvests property. Use default if necessary. |
|
| 154 |
try {
|
|
| 155 |
mh = Integer.valueOf(options.getOption("maxHarvests"));
|
|
| 156 |
maxHarvests = mh.intValue(); |
|
| 157 |
} |
|
| 158 |
catch (NumberFormatException e) {
|
|
| 159 |
System.out.println("NumberFormatException: Error parsing maxHarvests: " +
|
|
| 160 |
e.getMessage()); |
|
| 161 |
System.out.println("Defaulting to maxHarvests=" + maxHarvestsDefault);
|
|
| 162 |
maxHarvests = maxHarvestsDefault.intValue(); |
|
| 163 |
} |
|
| 140 |
System.out.println(marker); |
|
| 141 |
System.out.println("Starting Harvester");
|
|
| 142 |
Harvester.loadProperties(test); |
|
| 164 | 143 |
|
| 165 |
// Parse the period property. Use default if necessary. |
|
| 166 |
try {
|
|
| 167 |
p = Integer.valueOf(options.getOption("period"));
|
|
| 168 |
period = p.intValue(); |
|
| 169 |
} |
|
| 170 |
catch (NumberFormatException e) {
|
|
| 171 |
System.out.println("NumberFormatException: Error parsing period: " +
|
|
| 172 |
e.getMessage()); |
|
| 173 |
System.out.println("Defaulting to period=" + periodDefault);
|
|
| 174 |
period = periodDefault.intValue(); |
|
| 175 |
} |
|
| 176 |
|
|
| 177 |
// Sleep for delay number of hours prior to starting first harvest |
|
| 178 |
if (delay > 0) {
|
|
| 179 |
try {
|
|
| 180 |
System.out.print("First harvest will begin in " + delay);
|
|
| 181 |
if (delay == 1) {
|
|
| 182 |
System.out.println(" hour.");
|
|
| 183 |
} |
|
| 184 |
else {
|
|
| 185 |
System.out.println(" hours.");
|
|
| 186 |
} |
|
| 187 |
Thread.sleep(delay * oneHour); |
|
| 188 |
} |
|
| 189 |
catch (InterruptedException e) {
|
|
| 190 |
System.err.println("InterruptedException: " + e.getMessage());
|
|
| 191 |
System.exit(1); |
|
| 192 |
} |
|
| 193 |
} |
|
| 144 |
// Parse the delay property. Use default if necessary. |
|
| 145 |
try {
|
|
| 146 |
d = Integer.valueOf(PropertyService.getProperty("delay"));
|
|
| 147 |
delay = d.intValue(); |
|
| 148 |
} catch (NumberFormatException e) {
|
|
| 149 |
System.out.println("NumberFormatException: Error parsing delay: "
|
|
| 150 |
+ e.getMessage()); |
|
| 151 |
System.out.println("Defaulting to delay=" + delayDefault);
|
|
| 152 |
delay = delayDefault.intValue(); |
|
| 153 |
} catch (PropertyNotFoundException pnfe) {
|
|
| 154 |
System.out.println("PropertyNotFoundException: Error finding delay: "
|
|
| 155 |
+ pnfe.getMessage()); |
|
| 156 |
System.out.println("Defaulting to delay=" + delayDefault);
|
|
| 157 |
delay = delayDefault.intValue(); |
|
| 158 |
} |
|
| 194 | 159 |
|
| 160 |
// Parse the maxHarvests property. Use default if necessary. |
|
| 161 |
try {
|
|
| 162 |
mh = Integer.valueOf(PropertyService.getProperty("maxHarvests"));
|
|
| 163 |
maxHarvests = mh.intValue(); |
|
| 164 |
} catch (NumberFormatException e) {
|
|
| 165 |
System.out.println("NumberFormatException: Error parsing maxHarvests: "
|
|
| 166 |
+ e.getMessage()); |
|
| 167 |
System.out.println("Defaulting to maxHarvests=" + maxHarvestsDefault);
|
|
| 168 |
maxHarvests = maxHarvestsDefault.intValue(); |
|
| 169 |
} catch (PropertyNotFoundException pnfe) {
|
|
| 170 |
System.out.println("PropertyNotFoundException: Error finding maxHarvests: "
|
|
| 171 |
+ pnfe.getMessage()); |
|
| 172 |
System.out.println("Defaulting to maxHarvests=" + maxHarvestsDefault);
|
|
| 173 |
maxHarvests = maxHarvestsDefault.intValue(); |
|
| 174 |
} |
|
| 175 |
|
|
| 176 |
// Parse the period property. Use default if necessary. |
|
| 177 |
try {
|
|
| 178 |
p = Integer.valueOf(PropertyService.getProperty("period"));
|
|
| 179 |
period = p.intValue(); |
|
| 180 |
} catch (NumberFormatException e) {
|
|
| 181 |
System.out.println("NumberFormatException: Error parsing period: "
|
|
| 182 |
+ e.getMessage()); |
|
| 183 |
System.out.println("Defaulting to period=" + periodDefault);
|
|
| 184 |
period = periodDefault.intValue(); |
|
| 185 |
} catch (PropertyNotFoundException pnfe) {
|
|
| 186 |
System.out.println("PropertyNotFoundException: Error finding period: "
|
|
| 187 |
+ pnfe.getMessage()); |
|
| 188 |
System.out.println("Defaulting to period=" + periodDefault);
|
|
| 189 |
period = periodDefault.intValue(); |
|
| 190 |
} |
|
| 191 |
|
|
| 192 |
// Sleep for delay number of hours prior to starting first harvest |
|
| 193 |
if (delay > 0) {
|
|
| 194 |
try {
|
|
| 195 |
System.out.print("First harvest will begin in " + delay);
|
|
| 196 |
if (delay == 1) {
|
|
| 197 |
System.out.println(" hour.");
|
|
| 198 |
} else {
|
|
| 199 |
System.out.println(" hours.");
|
|
| 200 |
} |
|
| 201 |
Thread.sleep(delay * oneHour); |
|
| 202 |
} catch (InterruptedException e) {
|
|
| 203 |
System.err.println("InterruptedException: " + e.getMessage());
|
|
| 204 |
System.exit(1); |
|
| 205 |
} |
|
| 206 |
} |
|
| 207 |
|
|
| 195 | 208 |
// Repeat a new harvest once every period number of hours, until we reach |
| 196 | 209 |
// the maximum number of harvests, or indefinitely if maxHarvests <= 0. |
| 197 |
// Subtract delta from the time period so
|
|
| 210 |
// Subtract delta from the time period so |
|
| 198 | 211 |
// that each harvest will start at a fixed interval. |
| 199 | 212 |
// |
| 200 | 213 |
while (keepRunning && ((nHarvests < maxHarvests) || (maxHarvests <= 0))) {
|
| 201 | 214 |
nHarvests++; |
| 202 | 215 |
startTime = System.currentTimeMillis(); |
| 203 |
harvester = new Harvester(); // New object for this harvest |
|
| 216 |
harvester = new Harvester(); // New object for this |
|
| 217 |
// harvest |
|
| 204 | 218 |
harvester.startup(nHarvests, maxHarvests); // Start up Harvester |
| 205 | 219 |
harvester.readHarvestSiteSchedule(); // Read the database table |
| 206 | 220 |
harvester.harvest(); // Harvest the documents |
| ... | ... | |
| 395 | 409 |
*/ |
| 396 | 410 |
public Connection getConnection() {
|
| 397 | 411 |
String dbDriver = ""; |
| 398 |
String defaultDB; |
|
| 399 |
String password; |
|
| 400 |
String user; |
|
| 412 |
String defaultDB = null;
|
|
| 413 |
String password = null;
|
|
| 414 |
String user = null;
|
|
| 401 | 415 |
SQLWarning warn; |
| 402 | 416 |
|
| 403 | 417 |
if (conn == null) {
|
| 404 |
dbDriver = options.getOption("database.driver");
|
|
| 405 |
defaultDB = options.getOption("database.connectionURI");
|
|
| 406 |
password = options.getOption("database.password");
|
|
| 407 |
user = options.getOption("database.user");
|
|
| 418 |
try {
|
|
| 419 |
dbDriver = PropertyService.getProperty("database.driver");
|
|
| 420 |
defaultDB = PropertyService.getProperty("database.connectionURI");
|
|
| 421 |
password = PropertyService.getProperty("database.password");
|
|
| 422 |
user = PropertyService.getProperty("database.user");
|
|
| 423 |
} catch (PropertyNotFoundException pnfe) {
|
|
| 424 |
System.out.println("Can't find property " + pnfe);
|
|
| 425 |
System.exit(1); |
|
| 426 |
} |
|
| 408 | 427 |
|
| 409 | 428 |
// Load the jdbc driver |
| 410 | 429 |
try {
|
| ... | ... | |
| 950 | 969 |
} |
| 951 | 970 |
|
| 952 | 971 |
|
| 953 |
/** |
|
| 954 |
* Initializes Harvester at startup. Connects to the database and to Metacat. |
|
| 955 |
* |
|
| 956 |
* @param nHarvests the nth harvest |
|
| 957 |
* @param maxHarvests the maximum number of harvests that this process |
|
| 958 |
* can run |
|
| 959 |
*/ |
|
| 960 |
private void startup(int nHarvests, int maxHarvests) {
|
|
| 961 |
Boolean ctm; |
|
| 962 |
Integer lp; |
|
| 963 |
String metacatURL; |
|
| 964 |
Date now = new Date(); |
|
| 965 |
|
|
| 966 |
timestamp = now.toString(); |
|
| 967 |
System.out.println(Harvester.marker); |
|
| 968 |
System.out.print(timestamp + ": Starting Next Harvest"); |
|
| 969 |
if (maxHarvests > 0) {
|
|
| 970 |
System.out.print(" (" + nHarvests + "/" + maxHarvests + ")");
|
|
| 971 |
} |
|
| 972 |
System.out.print("\n");
|
|
| 973 |
ctm = Boolean.valueOf(options.getOption("connectToMetacat"));
|
|
| 974 |
connectToMetacat = ctm.booleanValue(); |
|
| 975 |
harvesterAdministrator = options.getOption("harvesterAdministrator");
|
|
| 976 |
smtpServer = options.getOption("smtpServer");
|
|
| 972 |
/** |
|
| 973 |
* Initializes Harvester at startup. Connects to the database and to Metacat. |
|
| 974 |
* |
|
| 975 |
* @param nHarvests the nth harvest |
|
| 976 |
* @param maxHarvests the maximum number of harvests that this process |
|
| 977 |
* can run |
|
| 978 |
*/ |
|
| 979 |
private void startup(int nHarvests, int maxHarvests) {
|
|
| 980 |
Boolean ctm; |
|
| 981 |
Integer lp; |
|
| 982 |
String metacatURL; |
|
| 983 |
Date now = new Date(); |
|
| 977 | 984 |
|
| 978 |
try {
|
|
| 979 |
lp = Integer.valueOf(options.getOption("logPeriod"));
|
|
| 980 |
logPeriod = lp.intValue(); |
|
| 981 |
} |
|
| 982 |
catch (NumberFormatException e) {
|
|
| 983 |
System.err.println("NumberFormatException: Error parsing logPeriod " +
|
|
| 984 |
logPeriod + e.getMessage()); |
|
| 985 |
System.err.println("Defaulting to logPeriod of 90 days");
|
|
| 986 |
logPeriod = 90; |
|
| 987 |
} |
|
| 985 |
timestamp = now.toString(); |
|
| 986 |
System.out.println(Harvester.marker); |
|
| 987 |
System.out.print(timestamp + ": Starting Next Harvest"); |
|
| 988 |
if (maxHarvests > 0) {
|
|
| 989 |
System.out.print(" (" + nHarvests + "/" + maxHarvests + ")");
|
|
| 990 |
} |
|
| 991 |
System.out.print("\n");
|
|
| 992 |
try {
|
|
| 993 |
ctm = Boolean.valueOf(PropertyService.getProperty("connectToMetacat"));
|
|
| 994 |
connectToMetacat = ctm.booleanValue(); |
|
| 995 |
harvesterAdministrator = PropertyService |
|
| 996 |
.getProperty("harvesterAdministrator");
|
|
| 997 |
smtpServer = PropertyService.getProperty("smtpServer");
|
|
| 988 | 998 |
|
| 989 |
conn = getConnection(); |
|
| 990 |
initLogIDs(); |
|
| 991 |
setHarvestStartTime(now); |
|
| 992 |
// Log startup operation |
|
| 993 |
addLogEntry(0, "Starting Up Harvester", "HarvesterStartup", 0, null, ""); |
|
| 994 |
|
|
| 995 |
if (connectToMetacat()) {
|
|
| 996 |
try {
|
|
| 997 |
metacatURL = SystemUtil.getServletURL(); |
|
| 998 |
System.out.println("Connecting to Metacat: " + metacatURL);
|
|
| 999 |
metacat = MetacatFactory.createMetacatConnection(metacatURL); |
|
| 1000 |
} |
|
| 1001 |
catch (MetacatInaccessibleException e) {
|
|
| 1002 |
System.out.println("Metacat connection failed." + e.getMessage());
|
|
| 1003 |
} |
|
| 1004 |
catch (Exception e) {
|
|
| 1005 |
System.out.println("Metacat connection failed." + e.getMessage());
|
|
| 1006 |
} |
|
| 1007 |
} |
|
| 1008 |
} |
|
| 999 |
lp = Integer.valueOf(PropertyService.getProperty("logPeriod"));
|
|
| 1000 |
logPeriod = lp.intValue(); |
|
| 1001 |
} catch (NumberFormatException e) {
|
|
| 1002 |
System.err.println("NumberFormatException: Error parsing logPeriod "
|
|
| 1003 |
+ logPeriod + e.getMessage()); |
|
| 1004 |
System.err.println("Defaulting to logPeriod of 90 days");
|
|
| 1005 |
logPeriod = 90; |
|
| 1006 |
} catch (PropertyNotFoundException pnfe) {
|
|
| 1007 |
System.out.println("PropertyNotFoundException: Error getting property: "
|
|
| 1008 |
+ pnfe.getMessage()); |
|
| 1009 |
return; |
|
| 1010 |
} |
|
| 1009 | 1011 |
|
| 1012 |
conn = getConnection(); |
|
| 1013 |
initLogIDs(); |
|
| 1014 |
setHarvestStartTime(now); |
|
| 1015 |
// Log startup operation |
|
| 1016 |
addLogEntry(0, "Starting Up Harvester", "HarvesterStartup", 0, null, ""); |
|
| 1017 |
|
|
| 1018 |
if (connectToMetacat()) {
|
|
| 1019 |
try {
|
|
| 1020 |
metacatURL = SystemUtil.getServletURL(); |
|
| 1021 |
System.out.println("Connecting to Metacat: " + metacatURL);
|
|
| 1022 |
metacat = MetacatFactory.createMetacatConnection(metacatURL); |
|
| 1023 |
} catch (MetacatInaccessibleException e) {
|
|
| 1024 |
System.out.println("Metacat connection failed." + e.getMessage());
|
|
| 1025 |
} catch (Exception e) {
|
|
| 1026 |
System.out.println("Metacat connection failed." + e.getMessage());
|
|
| 1027 |
} |
|
| 1028 |
} |
|
| 1029 |
} |
|
| 1030 |
|
|
| 1010 | 1031 |
} |
Also available in: Unified diff
Replace Options class with PropertyService