Revision 2155
Added by Duane Costa over 20 years ago
src/edu/ucsb/nceas/metacat/harvesterClient/Harvester.java | ||
---|---|---|
25 | 25 |
package edu.ucsb.nceas.metacat.harvesterClient; |
26 | 26 |
|
27 | 27 |
import com.oreilly.servlet.MailMessage; |
28 |
import edu.ucsb.nceas.utilities.Options; |
|
28 | 29 |
import java.io.File; |
29 | 30 |
import java.io.FileInputStream; |
30 | 31 |
import java.io.IOException; |
... | ... | |
38 | 39 |
import java.util.ArrayList; |
39 | 40 |
import java.text.SimpleDateFormat; |
40 | 41 |
import java.util.Date; |
41 |
import java.util.Properties; |
|
42 | 42 |
|
43 | 43 |
import edu.ucsb.nceas.metacat.client.Metacat; |
44 | 44 |
import edu.ucsb.nceas.metacat.client.MetacatFactory; |
... | ... | |
56 | 56 |
/* |
57 | 57 |
* Class fields |
58 | 58 |
*/ |
59 |
private static final String marker = |
|
59 |
private static final String CONFIG_DIR = "../../build/war/WEB-INF"; |
|
60 |
private static final String CONFIG_DIR_TEST = "./build/war/WEB-INF"; |
|
61 |
private static final String CONFIG_NAME = "metacat.properties"; |
|
62 |
public static final String filler = "*"; |
|
63 |
public static final String marker = |
|
60 | 64 |
"*****************************************************************************"; |
61 |
|
|
62 |
public static Properties properties; |
|
65 |
public static Options options = null; |
|
63 | 66 |
|
64 | 67 |
|
65 | 68 |
/* |
... | ... | |
75 | 78 |
|
76 | 79 |
|
77 | 80 |
/** |
78 |
* Loads Harvester properties from a configuration file.
|
|
81 |
* Loads Harvester options from a configuration file.
|
|
79 | 82 |
*/ |
80 |
public static void loadProperties(File configfile) { |
|
81 |
properties = new Properties(); |
|
83 |
public static void loadOptions(boolean test) { |
|
84 |
String configDir = test ? CONFIG_DIR_TEST : CONFIG_DIR; |
|
85 |
File propertyFile = new File(configDir, CONFIG_NAME); |
|
82 | 86 |
|
83 | 87 |
try { |
84 |
properties.load(new FileInputStream(configfile));
|
|
85 |
} |
|
88 |
options = Options.initialize(propertyFile);
|
|
89 |
}
|
|
86 | 90 |
catch (IOException e) { |
87 |
System.out.println("IOException: " + e.getMessage()); |
|
88 |
System.exit(1); |
|
91 |
System.out.println("Error in loading options: " + e.getMessage()); |
|
89 | 92 |
} |
93 |
|
|
90 | 94 |
} |
91 | 95 |
|
92 | 96 |
|
... | ... | |
99 | 103 |
* @throws ParserConfigurationException |
100 | 104 |
*/ |
101 | 105 |
public static void main(String[] args) { |
102 |
File configfile = new File("../../lib/harvester", "harvester.properties"); |
|
103 | 106 |
Integer delayDefault = new Integer(0); // Default number of hours delay |
104 | 107 |
int delay = delayDefault.intValue(); // Delay in hours before first harvest |
105 | 108 |
Integer d; // Used for determining delay |
... | ... | |
115 | 118 |
int period = periodDefault.intValue(); // Hours between harvests |
116 | 119 |
Integer p; // Used in determining the period |
117 | 120 |
long startTime; // time that a harvest run starts |
121 |
boolean test = false; // Passed to loadOption() |
|
118 | 122 |
|
119 | 123 |
System.out.println(marker); |
120 | 124 |
System.out.println("Starting Harvester"); |
121 |
Harvester.loadProperties(configfile);
|
|
125 |
Harvester.loadOptions(test);
|
|
122 | 126 |
|
123 | 127 |
// Parse the delay property. Use default if necessary. |
124 | 128 |
try { |
125 |
d = Integer.valueOf(properties.getProperty( |
|
126 |
"delay", |
|
127 |
delayDefault.toString() |
|
128 |
)); |
|
129 |
d = Integer.valueOf(options.getOption("delay")); |
|
129 | 130 |
delay = d.intValue(); |
130 | 131 |
} |
131 | 132 |
catch (NumberFormatException e) { |
... | ... | |
137 | 138 |
|
138 | 139 |
// Parse the maxHarvests property. Use default if necessary. |
139 | 140 |
try { |
140 |
mh = Integer.valueOf(properties.getProperty( |
|
141 |
"maxHarvests", |
|
142 |
maxHarvestsDefault.toString() |
|
143 |
)); |
|
141 |
mh = Integer.valueOf(options.getOption("maxHarvests")); |
|
144 | 142 |
maxHarvests = mh.intValue(); |
145 | 143 |
} |
146 | 144 |
catch (NumberFormatException e) { |
... | ... | |
152 | 150 |
|
153 | 151 |
// Parse the period property. Use default if necessary. |
154 | 152 |
try { |
155 |
p = Integer.valueOf(properties.getProperty("period", |
|
156 |
periodDefault.toString() |
|
157 |
)); |
|
153 |
p = Integer.valueOf(options.getOption("period")); |
|
158 | 154 |
period = p.intValue(); |
159 | 155 |
} |
160 | 156 |
catch (NumberFormatException e) { |
... | ... | |
367 | 363 |
* @return conn the database Connection object |
368 | 364 |
*/ |
369 | 365 |
public Connection getConnection() { |
370 |
String database; |
|
371 | 366 |
String dbDriver = ""; |
372 | 367 |
String defaultDB; |
373 | 368 |
String password; |
374 |
Properties properties = Harvester.properties; |
|
375 | 369 |
String user; |
376 | 370 |
SQLWarning warn; |
377 | 371 |
|
378 | 372 |
if (conn == null) { |
379 |
database = properties.getProperty("database"); |
|
380 |
|
|
381 |
if (database.equalsIgnoreCase("oracle")) { |
|
382 |
dbDriver = "oracle.jdbc.driver.OracleDriver"; |
|
383 |
} |
|
384 |
else if (database.equalsIgnoreCase("postgresql")) { |
|
385 |
dbDriver = "org.postgresql.Driver"; |
|
386 |
} |
|
387 |
else if (database.equalsIgnoreCase("sqlserver")) { |
|
388 |
dbDriver = "com.microsoft.jdbc.sqlserver.SQLServerDriver"; |
|
389 |
} |
|
373 |
dbDriver = options.getOption("dbDriver"); |
|
374 |
defaultDB = options.getOption("defaultDB"); |
|
375 |
password = options.getOption("password"); |
|
376 |
user = options.getOption("user"); |
|
390 | 377 |
|
391 |
defaultDB = properties.getProperty("defaultDB"); |
|
392 |
password = properties.getProperty("password"); |
|
393 |
user = properties.getProperty("user"); |
|
394 |
|
|
395 | 378 |
// Load the jdbc driver |
396 | 379 |
try { |
397 | 380 |
Class.forName(dbDriver); |
... | ... | |
557 | 540 |
|
558 | 541 |
|
559 | 542 |
/** |
560 |
* Prints harvest log entries for this harvest run. Entries may be filtered |
|
561 |
* for a particular site, or all entries may be printed. |
|
543 |
* Prints the header of the harvest report. |
|
562 | 544 |
* |
563 |
* @param out the PrintStream object to write to |
|
564 |
* @param maxCodeLevel the maximum code level that should be printed, |
|
565 |
* e.g. "warning". Any log entries higher than this |
|
566 |
* level will not be printed. |
|
567 |
* @param siteScheduleID if greater than 0, indicates that the log |
|
568 |
* entry should only be printed for a particular site |
|
569 |
* as identified by its siteScheduleID. if 0, then |
|
570 |
* print output for all sites. |
|
545 |
* @param out the PrintStream object to print to |
|
546 |
* @param siteScheduleID the siteScheduleId of the HarvestSiteSchedule. Will |
|
547 |
* have a value of 0 if no particular site is involved, |
|
548 |
* which indicates that the report is being prepared |
|
549 |
* for the Harvester Administrator rather than for a |
|
550 |
* particular Site Contact. |
|
571 | 551 |
*/ |
572 |
void printHarvestLog(PrintStream out, |
|
573 |
String maxCodeLevel, |
|
574 |
int siteScheduleID |
|
575 |
) { |
|
552 |
void printHarvestHeader(PrintStream out, int siteScheduleID) { |
|
576 | 553 |
HarvestLog harvestLog; |
577 | 554 |
int logSiteScheduleID; |
578 | 555 |
int nErrors = 0; |
... | ... | |
590 | 567 |
} |
591 | 568 |
|
592 | 569 |
out.println(marker); |
593 |
out.println("*");
|
|
570 |
out.println(filler);
|
|
594 | 571 |
out.println("* METACAT HARVESTER REPORT: " + timestamp); |
595 |
out.println("*");
|
|
572 |
out.println(filler);
|
|
596 | 573 |
|
597 | 574 |
if (nErrors > 0) { |
598 | 575 |
phrase = (nErrors == 1) ? " ERROR WAS " : " ERRORS WERE "; |
... | ... | |
603 | 580 |
out.println("* NO ERRORS WERE DETECTED DURING THIS HARVEST."); |
604 | 581 |
} |
605 | 582 |
|
606 |
out.println("*");
|
|
583 |
out.println(filler);
|
|
607 | 584 |
out.println(marker); |
585 |
} |
|
586 |
|
|
608 | 587 |
|
588 |
/** |
|
589 |
* Prints harvest log entries for this harvest run. Entries may be filtered |
|
590 |
* for a particular site, or all entries may be printed. |
|
591 |
* |
|
592 |
* @param out the PrintStream object to write to |
|
593 |
* @param maxCodeLevel the maximum code level that should be printed, |
|
594 |
* e.g. "warning". Any log entries higher than this |
|
595 |
* level will not be printed. |
|
596 |
* @param siteScheduleID if greater than 0, indicates that the log |
|
597 |
* entry should only be printed for a particular site |
|
598 |
* as identified by its siteScheduleID. if 0, then |
|
599 |
* print output for all sites. |
|
600 |
*/ |
|
601 |
void printHarvestLog(PrintStream out, String maxCodeLevel, int siteScheduleID |
|
602 |
) { |
|
603 |
HarvestLog harvestLog; |
|
604 |
int logSiteScheduleID; |
|
605 |
int nErrors = 0; |
|
606 |
String phrase; |
|
607 |
|
|
608 |
out.println(""); |
|
609 |
out.println(marker); |
|
610 |
out.println(filler); |
|
611 |
out.println("* LOG ENTRIES"); |
|
612 |
out.println(filler); |
|
613 |
out.println(marker); |
|
614 |
|
|
609 | 615 |
for (int i = 0; i < harvestLogList.size(); i++) { |
610 | 616 |
harvestLog = (HarvestLog) harvestLogList.get(i); |
611 | 617 |
logSiteScheduleID = harvestLog.getSiteScheduleID(); |
... | ... | |
790 | 796 |
|
791 | 797 |
|
792 | 798 |
/** |
793 |
* Sends a report to the Harvester administrator. The report prints each log
|
|
799 |
* Sends a report to the Harvester Administrator. The report prints each log
|
|
794 | 800 |
* entry pertaining to this harvest run. |
795 | 801 |
* |
796 | 802 |
* @param maxCodeLevel the maximum code level that should be printed, |
... | ... | |
815 | 821 |
msg.to(to); |
816 | 822 |
msg.setSubject(subject); |
817 | 823 |
body = msg.getPrintStream(); |
824 |
printHarvestHeader(body, siteScheduleID); |
|
818 | 825 |
printHarvestLog(body, maxCodeLevel, siteScheduleID); |
819 | 826 |
msg.sendAndClose(); |
820 | 827 |
} |
... | ... | |
865 | 872 |
* can run |
866 | 873 |
*/ |
867 | 874 |
private void startup(int nHarvests, int maxHarvests) { |
875 |
String context; |
|
868 | 876 |
Boolean ctm; |
877 |
String httpserver; |
|
869 | 878 |
Integer lp; |
870 | 879 |
String metacatURL; |
871 | 880 |
Date now = new Date(); |
872 |
Properties properties = Harvester.properties; |
|
873 | 881 |
|
874 | 882 |
timestamp = now.toString(); |
875 | 883 |
System.out.println(Harvester.marker); |
876 | 884 |
System.out.println(timestamp + ": Starting Next Harvest (" + |
877 | 885 |
nHarvests + "/" + maxHarvests + ")"); |
878 |
ctm = Boolean.valueOf(properties.getProperty("connectToMetacat", "true"));
|
|
886 |
ctm = Boolean.valueOf(options.getOption("connectToMetacat"));
|
|
879 | 887 |
connectToMetacat = ctm.booleanValue(); |
880 |
harvesterAdministrator = properties.getProperty("harvesterAdministrator");
|
|
881 |
smtpServer = properties.getProperty("smtpServer", "localhost");
|
|
888 |
harvesterAdministrator = options.getOption("harvesterAdministrator");
|
|
889 |
smtpServer = options.getOption("smtpServer");
|
|
882 | 890 |
|
883 | 891 |
try { |
884 |
lp = Integer.valueOf(properties.getProperty("logPeriod", "90"));
|
|
892 |
lp = Integer.valueOf(options.getOption("logPeriod"));
|
|
885 | 893 |
logPeriod = lp.intValue(); |
886 | 894 |
} |
887 | 895 |
catch (NumberFormatException e) { |
... | ... | |
899 | 907 |
|
900 | 908 |
if (connectToMetacat()) { |
901 | 909 |
try { |
902 |
metacatURL = properties.getProperty("metacatURL"); |
|
910 |
httpserver = options.getOption("httpserver"); |
|
911 |
context = options.getOption("context"); |
|
912 |
metacatURL = httpserver + "/" + context + "/servlet/metacat"; |
|
903 | 913 |
System.out.println("Connecting to Metacat: " + metacatURL); |
904 | 914 |
metacat = MetacatFactory.createMetacatConnection(metacatURL); |
905 | 915 |
} |
src/edu/ucsb/nceas/metacat/harvesterClient/MetUpload.java | ||
---|---|---|
24 | 24 |
|
25 | 25 |
package edu.ucsb.nceas.metacat.harvesterClient; |
26 | 26 |
|
27 |
import edu.ucsb.nceas.utilities.Options; |
|
27 | 28 |
import java.io.*; |
28 | 29 |
import java.util.*; |
29 | 30 |
import javax.servlet.ServletConfig; |
... | ... | |
49 | 50 |
*/ |
50 | 51 |
public class MetUpload extends HttpServlet { |
51 | 52 |
|
53 |
/* |
|
54 |
* Class fields |
|
55 |
*/ |
|
56 |
private static final String CONFIG_DIR = "WEB-INF"; |
|
57 |
private static final String CONFIG_NAME = "metacat.properties"; |
|
58 |
|
|
59 |
/* Object fields */ |
|
60 |
private ServletConfig config = null; |
|
61 |
private ServletContext context = null; |
|
62 |
private String metacatURL; |
|
63 |
|
|
52 | 64 |
/** |
53 | 65 |
* Handle POST requests made to the Harvester MetUpload servlet. |
54 | 66 |
* |
... | ... | |
65 | 77 |
String fieldValue; |
66 | 78 |
String fileName = ""; |
67 | 79 |
FilePart fPart = null; |
68 |
String metacatUrl; |
|
69 | 80 |
PrintWriter out = res.getWriter(); |
70 | 81 |
MultipartParser parser = new MultipartParser(req, 1024 * 1024); |
71 | 82 |
ParamPart pPart; |
72 | 83 |
HttpSession sess = req.getSession(true); |
73 | 84 |
String password = (String) sess.getAttribute("Mpassword"); |
74 |
Properties properties = loadProperties(out); |
|
75 | 85 |
StringReader sr = null; |
76 | 86 |
boolean upload = false; |
77 | 87 |
String username = (String) sess.getAttribute("Musername"); |
78 | 88 |
|
79 | 89 |
res.setContentType("text/plain"); |
80 |
metacatUrl = properties.getProperty("metacatURL", ""); |
|
81 | 90 |
|
82 | 91 |
try { |
83 | 92 |
while ((aPart = parser.readNextPart()) != null) { |
... | ... | |
112 | 121 |
} |
113 | 122 |
|
114 | 123 |
if (upload) { |
115 |
upload(out, docid, sr, metacatUrl, username, password);
|
|
124 |
upload(out, docid, sr, username, password); |
|
116 | 125 |
} |
117 | 126 |
if (delete) { |
118 |
delete(out, docid, metacatUrl, username, password);
|
|
127 |
delete(out, docid, username, password); |
|
119 | 128 |
} |
120 | 129 |
} |
121 | 130 |
|
... | ... | |
125 | 134 |
* |
126 | 135 |
* @param out the PrintWriter output stream |
127 | 136 |
* @param docid the Metacat document id, e.g. document.1.1 |
128 |
* @param metacatUrl the Metacat URL |
|
129 | 137 |
* @param username the Metacat username |
130 | 138 |
* @param password the Metacat password |
131 | 139 |
*/ |
132 | 140 |
private void delete(PrintWriter out, |
133 | 141 |
String docid, |
134 |
String metacatUrl, |
|
135 | 142 |
String username, |
136 | 143 |
String password |
137 | 144 |
) { |
... | ... | |
144 | 151 |
} |
145 | 152 |
|
146 | 153 |
try { |
147 |
metacat = MetacatFactory.createMetacatConnection(metacatUrl);
|
|
154 |
metacat = MetacatFactory.createMetacatConnection(metacatURL);
|
|
148 | 155 |
metacat.login(username, password); |
149 | 156 |
metacatResponse = metacat.delete(docid); |
150 | 157 |
out.println(metacatResponse); |
... | ... | |
170 | 177 |
|
171 | 178 |
|
172 | 179 |
/** |
173 |
* Loads Harvester properties from a configuration file.
|
|
180 |
* Initializes the servlet. Reads properties and initializes object fields.
|
|
174 | 181 |
* |
175 |
* @param out the PrintWriter output stream |
|
176 |
* @return properties Properties object containing the loaded properties |
|
182 |
* @throws ServletException |
|
177 | 183 |
*/ |
178 |
private Properties loadProperties(PrintWriter out) {
|
|
179 |
InputStream inputStream;
|
|
180 |
String path = "/harvester/harvester.properties";
|
|
181 |
Properties properties = new Properties();
|
|
182 |
ServletContext servletContext = getServletContext();
|
|
184 |
public void init(ServletConfig config) throws ServletException {
|
|
185 |
String contextString;
|
|
186 |
String dirPath;
|
|
187 |
String httpserver;
|
|
188 |
Options options = null;
|
|
183 | 189 |
|
190 |
super.init(config); |
|
191 |
this.config = config; |
|
192 |
this.context = config.getServletContext(); |
|
193 |
dirPath = context.getRealPath(CONFIG_DIR); |
|
194 |
File propertyFile = new File(dirPath, CONFIG_NAME); |
|
195 |
|
|
184 | 196 |
try { |
185 |
inputStream = servletContext.getResourceAsStream(path); |
|
186 |
properties.load(inputStream); |
|
197 |
options = Options.initialize(propertyFile); |
|
187 | 198 |
} |
188 | 199 |
catch (IOException e) { |
189 |
out.println("IOException: " + e.getMessage());
|
|
200 |
System.out.println("Error in loading options: " + e.getMessage());
|
|
190 | 201 |
} |
191 | 202 |
|
192 |
return properties; |
|
203 |
httpserver = options.getOption("httpserver"); |
|
204 |
contextString = options.getOption("context"); |
|
205 |
metacatURL = httpserver + "/" + contextString + "/servlet/metacat"; |
|
206 |
System.out.println("metacatURL: " + metacatURL); |
|
193 | 207 |
} |
194 |
|
|
195 | 208 |
|
209 |
|
|
196 | 210 |
/** |
197 | 211 |
* Uploads a file to Metacat for insertion or updating. This implementation |
198 | 212 |
* is limited by the fact that all ".1" documents are inserted while all |
... | ... | |
203 | 217 |
* @param out the PrintWriter output stream |
204 | 218 |
* @param docid the docid, e.g. "document.1.1" |
205 | 219 |
* @param sr the StringReader containing the contents of the document |
206 |
* @param metacatUrl the Metacat URL |
|
207 | 220 |
* @param username the Metacat username |
208 | 221 |
* @param password the Metacat password |
209 | 222 |
*/ |
210 | 223 |
private void upload(PrintWriter out, |
211 | 224 |
String docid, |
212 | 225 |
StringReader sr, |
213 |
String metacatUrl, |
|
214 | 226 |
String username, |
215 | 227 |
String password |
216 | 228 |
) { |
... | ... | |
231 | 243 |
} |
232 | 244 |
|
233 | 245 |
try { |
234 |
metacat = MetacatFactory.createMetacatConnection(metacatUrl);
|
|
246 |
metacat = MetacatFactory.createMetacatConnection(metacatURL);
|
|
235 | 247 |
metacat.login(username, password); |
236 | 248 |
strLen = docid.length(); |
237 | 249 |
revision = docid.substring((strLen - 2), strLen); |
src/edu/ucsb/nceas/metacat/harvesterClient/HarvestDocument.java | ||
---|---|---|
61 | 61 |
int identifier; |
62 | 62 |
int revision; |
63 | 63 |
String scope; |
64 |
|
|
65 |
/* These booleans keep track of status information. They are used when |
|
66 |
* generating email reports. |
|
67 |
*/ |
|
68 |
boolean accessError = false; |
|
69 |
boolean inserted = false; |
|
70 |
boolean metacatHasIt = false; |
|
71 |
boolean updated = false; |
|
72 |
boolean uploadError = false; |
|
64 | 73 |
|
65 | 74 |
|
66 | 75 |
/** |
... | ... | |
123 | 132 |
""); |
124 | 133 |
} |
125 | 134 |
catch (MalformedURLException e) { |
135 |
accessError = true; |
|
126 | 136 |
harvester.addLogEntry(1, "MalformedURLException", "GetDocError", |
127 | 137 |
harvestSiteSchedule.siteScheduleID, this, |
128 | 138 |
"MalformedURLException: " + e.getMessage()); |
129 | 139 |
} |
130 | 140 |
catch (IOException e) { |
141 |
accessError = true; |
|
131 | 142 |
harvester.addLogEntry(1, "IOException", "GetDocError", |
132 | 143 |
harvestSiteSchedule.siteScheduleID, this, |
133 | 144 |
"IOException: " + e.getMessage()); |
... | ... | |
164 | 175 |
update = true; |
165 | 176 |
} |
166 | 177 |
else { |
178 |
metacatHasIt = true; |
|
167 | 179 |
harvester.addLogEntry(0, |
168 | 180 |
"Attempting to update " + docid + " to revision " + |
169 | 181 |
revision + ". Metacat has document revision " + |
... | ... | |
198 | 210 |
String exceptionName, |
199 | 211 |
Exception e |
200 | 212 |
) { |
213 |
uploadError = true; |
|
214 |
|
|
201 | 215 |
if (insert) { |
202 | 216 |
harvester.addLogEntry(1, |
203 | 217 |
metacatReturnString, |
... | ... | |
264 | 278 |
|
265 | 279 |
|
266 | 280 |
/** |
281 |
* Print the document URL following by its scope.identifier.revision. |
|
282 |
* Used for report generation. |
|
283 |
* |
|
284 |
* @param out the PrintStream to write to |
|
285 |
*/ |
|
286 |
public void prettyPrint(PrintStream out) { |
|
287 |
out.println("* " + docidFull + " (" + documentURL + ")"); |
|
288 |
} |
|
289 |
|
|
290 |
|
|
291 |
/** |
|
267 | 292 |
* Insert or update this document to Metacat. If revision equals 1, do an |
268 | 293 |
* insert; otherwise, do an update. |
269 | 294 |
* |
... | ... | |
281 | 306 |
try { |
282 | 307 |
if (insert) { |
283 | 308 |
metacatReturnString = metacat.insert(docidFull, stringReader, null); |
309 |
inserted = true; |
|
284 | 310 |
harvester.addLogEntry(0, docidFull + " : " + metacatReturnString, |
285 | 311 |
"InsertDocSuccess", |
286 | 312 |
harvestSiteSchedule.siteScheduleID, |
... | ... | |
288 | 314 |
} |
289 | 315 |
else if (update) { |
290 | 316 |
metacatReturnString = metacat.update(docidFull, stringReader, null); |
317 |
updated = true; |
|
291 | 318 |
harvester.addLogEntry(0, docidFull + " : " + metacatReturnString, |
292 | 319 |
"UpdateDocSuccess", |
293 | 320 |
harvestSiteSchedule.siteScheduleID, |
src/edu/ucsb/nceas/metacat/harvesterClient/HarvestLog.java | ||
---|---|---|
24 | 24 |
|
25 | 25 |
package edu.ucsb.nceas.metacat.harvesterClient; |
26 | 26 |
|
27 |
import edu.ucsb.nceas.utilities.Options; |
|
27 | 28 |
import java.io.PrintStream; |
28 | 29 |
import java.sql.Connection; |
29 | 30 |
import java.sql.ResultSet; |
... | ... | |
32 | 33 |
import java.sql.Statement; |
33 | 34 |
import java.text.SimpleDateFormat; |
34 | 35 |
import java.util.Date; |
35 |
import java.util.Properties; |
|
36 | 36 |
import java.util.StringTokenizer; |
37 | 37 |
|
38 | 38 |
|
... | ... | |
241 | 241 |
* Returns either an explanation string or a harvest operation code level |
242 | 242 |
* string based on the value of a harvest operation code property. The |
243 | 243 |
* explanation and the code level are stored as comma-separated strings in |
244 |
* the harvester.properties file. For example, the HarvesterStartup
|
|
244 |
* the metacat.properties file. For example, the HarvesterStartup
|
|
245 | 245 |
* harvest operation code stores its explanation and code level like so: |
246 | 246 |
* |
247 | 247 |
* HarvesterStartup=Harvester start up,Info |
... | ... | |
253 | 253 |
String getHarvestOperation(String fieldName, String harvestOperationCode) { |
254 | 254 |
String explanation = "No explanation available"; |
255 | 255 |
String harvestOperationCodeLevel = "debug"; |
256 |
Properties properties = Harvester.properties;
|
|
256 |
Options options = Harvester.options;
|
|
257 | 257 |
String propertyValue; |
258 | 258 |
String returnString = ""; |
259 | 259 |
StringTokenizer stringTokenizer; |
260 | 260 |
|
261 |
propertyValue = properties.getProperty(harvestOperationCode);
|
|
261 |
propertyValue = options.getOption(harvestOperationCode);
|
|
262 | 262 |
stringTokenizer = new StringTokenizer(propertyValue, ","); |
263 | 263 |
|
264 | 264 |
explanation = (String) stringTokenizer.nextElement(); |
src/edu/ucsb/nceas/metacat/harvesterClient/HarvestSiteSchedule.java | ||
---|---|---|
259 | 259 |
System.out.println("ParserConfigurationException: " + e.getMessage()); |
260 | 260 |
} |
261 | 261 |
|
262 |
reportToSite(); |
|
262 |
reportToSiteContact();
|
|
263 | 263 |
} |
264 | 264 |
} |
265 | 265 |
|
... | ... | |
384 | 384 |
out.println("* contactEmail: " + contactEmail); |
385 | 385 |
} |
386 | 386 |
|
387 |
/** |
|
388 |
* Reports a summary of the site harvest. Includes the following: |
|
389 |
* A list of documents that were successfully inserted. |
|
390 |
* A list of documents that were successfully updated. |
|
391 |
* A list of documents that could not be accessed at the site. |
|
392 |
* A list of documents that could not be uploaded to Metacat. |
|
393 |
* A list of documents that were already found in Metacat. |
|
394 |
* |
|
395 |
* @param out the PrintStream to write to |
|
396 |
*/ |
|
397 |
void printSiteSummary(PrintStream out) { |
|
398 |
HarvestDocument harvestDocument; |
|
399 |
int nAccessError = 0; |
|
400 |
int nInserted = 0; |
|
401 |
int nMetacatHasIt = 0; |
|
402 |
int nUpdated = 0; |
|
403 |
int nUploadError = 0; |
|
404 |
|
|
405 |
for (int i = 0; i < harvestDocumentList.size(); i++) { |
|
406 |
harvestDocument = (HarvestDocument) harvestDocumentList.get(i); |
|
407 |
|
|
408 |
if (harvestDocument != null) { |
|
409 |
if (harvestDocument.accessError) { nAccessError++; } |
|
410 |
if (harvestDocument.inserted) { nInserted++; } |
|
411 |
if (harvestDocument.metacatHasIt) { nMetacatHasIt++; } |
|
412 |
if (harvestDocument.updated) { nUpdated++; } |
|
413 |
if (harvestDocument.uploadError) { nUploadError++; } |
|
414 |
} |
|
415 |
} |
|
416 |
|
|
417 |
if (nInserted > 0) { |
|
418 |
printSiteSummaryHeader(out); |
|
419 |
out.println("* The following document(s) were successfully inserted:"); |
|
420 |
for (int i = 0; i < harvestDocumentList.size(); i++) { |
|
421 |
harvestDocument = (HarvestDocument) harvestDocumentList.get(i); |
|
422 |
if (harvestDocument != null) { |
|
423 |
if (harvestDocument.inserted) { |
|
424 |
harvestDocument.prettyPrint(out); |
|
425 |
} |
|
426 |
} |
|
427 |
} |
|
428 |
printSiteSummaryTrailer(out); |
|
429 |
} |
|
387 | 430 |
|
431 |
if (nUpdated > 0) { |
|
432 |
printSiteSummaryHeader(out); |
|
433 |
out.println("* The following document(s) were successfully updated:"); |
|
434 |
for (int i = 0; i < harvestDocumentList.size(); i++) { |
|
435 |
harvestDocument = (HarvestDocument) harvestDocumentList.get(i); |
|
436 |
if (harvestDocument != null) { |
|
437 |
if (harvestDocument.updated) { |
|
438 |
harvestDocument.prettyPrint(out); |
|
439 |
} |
|
440 |
} |
|
441 |
} |
|
442 |
printSiteSummaryTrailer(out); |
|
443 |
} |
|
444 |
|
|
445 |
if (nAccessError > 0) { |
|
446 |
printSiteSummaryHeader(out); |
|
447 |
out.println("* The following document(s) could not be accessed"); |
|
448 |
out.println("* at the site. Please check the URL to ensure that it is"); |
|
449 |
out.println("* accessible at the site."); |
|
450 |
for (int i = 0; i < harvestDocumentList.size(); i++) { |
|
451 |
harvestDocument = (HarvestDocument) harvestDocumentList.get(i); |
|
452 |
if (harvestDocument != null) { |
|
453 |
if (harvestDocument.accessError) { |
|
454 |
harvestDocument.prettyPrint(out); |
|
455 |
} |
|
456 |
} |
|
457 |
} |
|
458 |
printSiteSummaryTrailer(out); |
|
459 |
} |
|
460 |
|
|
461 |
if (nUploadError > 0) { |
|
462 |
printSiteSummaryHeader(out); |
|
463 |
out.println("* The following document(s) could not be uploaded to"); |
|
464 |
out.println("* Metacat because an error of some kind occurred."); |
|
465 |
out.println("* See log entries below for additional details.) :"); |
|
466 |
for (int i = 0; i < harvestDocumentList.size(); i++) { |
|
467 |
harvestDocument = (HarvestDocument) harvestDocumentList.get(i); |
|
468 |
if (harvestDocument != null) { |
|
469 |
if (harvestDocument.uploadError) { |
|
470 |
harvestDocument.prettyPrint(out); |
|
471 |
} |
|
472 |
} |
|
473 |
} |
|
474 |
printSiteSummaryTrailer(out); |
|
475 |
} |
|
476 |
|
|
477 |
if (nMetacatHasIt > 0) { |
|
478 |
printSiteSummaryHeader(out); |
|
479 |
out.println("* The following document(s) were already found in Metacat."); |
|
480 |
out.println("* See log entries below for additional details.) :"); |
|
481 |
for (int i = 0; i < harvestDocumentList.size(); i++) { |
|
482 |
harvestDocument = (HarvestDocument) harvestDocumentList.get(i); |
|
483 |
if (harvestDocument != null) { |
|
484 |
if (harvestDocument.metacatHasIt) { |
|
485 |
harvestDocument.prettyPrint(out); |
|
486 |
} |
|
487 |
} |
|
488 |
} |
|
489 |
printSiteSummaryTrailer(out); |
|
490 |
} |
|
491 |
|
|
492 |
} |
|
493 |
|
|
494 |
|
|
388 | 495 |
/** |
389 |
* Sends a report to the site summarizing the results of the harvest at |
|
390 |
* that site. |
|
496 |
* Prints the header lines of a site summary entry. |
|
497 |
* |
|
498 |
* @param out the PrintStream to write to |
|
391 | 499 |
*/ |
392 |
void reportToSite() { |
|
500 |
void printSiteSummaryHeader(PrintStream out) { |
|
501 |
final String filler = Harvester.filler; |
|
502 |
final String marker = Harvester.marker; |
|
503 |
|
|
504 |
out.println(""); |
|
505 |
out.println(marker); |
|
506 |
out.println(filler); |
|
507 |
} |
|
508 |
|
|
509 |
|
|
510 |
/** |
|
511 |
* Prints the trailing lines of a site summary entry. |
|
512 |
* |
|
513 |
* @param out the PrintStream to write to |
|
514 |
*/ |
|
515 |
void printSiteSummaryTrailer(PrintStream out) { |
|
516 |
final String filler = Harvester.filler; |
|
517 |
final String marker = Harvester.marker; |
|
518 |
|
|
519 |
out.println(filler); |
|
520 |
out.println(marker); |
|
521 |
} |
|
522 |
|
|
523 |
|
|
524 |
/** |
|
525 |
* Sends a report to the Site Contact summarizing the results of the harvest |
|
526 |
* at that site. |
|
527 |
*/ |
|
528 |
void reportToSiteContact() { |
|
393 | 529 |
PrintStream body; |
394 | 530 |
String from = harvester.harvesterAdministrator; |
395 | 531 |
String maxCodeLevel = "info"; |
... | ... | |
405 | 541 |
msg = new MailMessage(harvester.smtpServer); |
406 | 542 |
msg.from(from); |
407 | 543 |
msg.to(to); |
544 |
msg.cc(from); |
|
408 | 545 |
msg.setSubject(subject); |
409 | 546 |
body = msg.getPrintStream(); |
547 |
harvester.printHarvestHeader(body, siteScheduleID); |
|
548 |
printSiteSummary(body); |
|
410 | 549 |
harvester.printHarvestLog(body, maxCodeLevel, siteScheduleID); |
411 | 550 |
msg.sendAndClose(); |
412 | 551 |
} |
src/edu/ucsb/nceas/metacat/harvesterClient/HarvesterRegistration.java | ||
---|---|---|
25 | 25 |
package edu.ucsb.nceas.metacat.harvesterClient; |
26 | 26 |
|
27 | 27 |
import com.oreilly.servlet.ParameterParser; |
28 |
import edu.ucsb.nceas.utilities.Options; |
|
28 | 29 |
import java.io.File; |
29 | 30 |
import java.io.InputStream; |
30 | 31 |
import java.io.IOException; |
... | ... | |
39 | 40 |
import java.text.SimpleDateFormat; |
40 | 41 |
import java.util.Date; |
41 | 42 |
import java.util.Enumeration; |
42 |
import java.util.Properties;
|
|
43 |
import javax.servlet.ServletConfig;
|
|
43 | 44 |
import javax.servlet.ServletContext; |
44 | 45 |
import javax.servlet.ServletException; |
45 | 46 |
import javax.servlet.http.HttpServlet; |
... | ... | |
58 | 59 |
* |
59 | 60 |
*/ |
60 | 61 |
public class HarvesterRegistration extends HttpServlet { |
62 |
|
|
63 |
/* |
|
64 |
* Class fields |
|
65 |
*/ |
|
66 |
private static final String CONFIG_DIR = "WEB-INF"; |
|
67 |
private static final String CONFIG_NAME = "metacat.properties"; |
|
61 | 68 |
|
62 | 69 |
/* |
63 | 70 |
* Object fields |
64 | 71 |
*/ |
72 |
private ServletConfig config = null; |
|
73 |
private ServletContext context = null; |
|
65 | 74 |
private String defaultDB; // database connection, from properties file |
66 | 75 |
final private long millisecondsPerDay = (1000 * 60 * 60 * 24); |
67 | 76 |
private String password; // database password, from properties file |
... | ... | |
685 | 694 |
* |
686 | 695 |
* @throws ServletException |
687 | 696 |
*/ |
688 |
public void init() throws ServletException { |
|
697 |
public void init(ServletConfig config) throws ServletException {
|
|
689 | 698 |
String database; |
690 | 699 |
String dbDriver = ""; |
691 |
Properties properties = loadProperties(); |
|
700 |
String dirPath; |
|
701 |
Options options = null; |
|
692 | 702 |
|
693 |
database = properties.getProperty("database"); |
|
703 |
super.init(config); |
|
704 |
this.config = config; |
|
705 |
this.context = config.getServletContext(); |
|
706 |
dirPath = context.getRealPath(CONFIG_DIR); |
|
707 |
File propertyFile = new File(dirPath, CONFIG_NAME); |
|
694 | 708 |
|
695 |
if (database.equalsIgnoreCase("oracle")) {
|
|
696 |
dbDriver = "oracle.jdbc.driver.OracleDriver";
|
|
709 |
try {
|
|
710 |
options = Options.initialize(propertyFile);
|
|
697 | 711 |
} |
698 |
else if (database.equalsIgnoreCase("postgresql")) {
|
|
699 |
dbDriver = "org.postgresql.Driver";
|
|
712 |
catch (IOException e) {
|
|
713 |
System.out.println("Error in loading options: " + e.getMessage());
|
|
700 | 714 |
} |
701 |
else if (database.equalsIgnoreCase("sqlserver")) { |
|
702 |
dbDriver = "com.microsoft.jdbc.sqlserver.SQLServerDriver"; |
|
703 |
} |
|
704 | 715 |
|
705 |
defaultDB = properties.getProperty("defaultDB"); |
|
706 |
password = properties.getProperty("password"); |
|
707 |
user = properties.getProperty("user"); |
|
716 |
dbDriver = options.getOption("dbDriver"); |
|
717 |
defaultDB = options.getOption("defaultDB"); |
|
718 |
password = options.getOption("password"); |
|
719 |
user = options.getOption("user"); |
|
708 | 720 |
|
709 | 721 |
// Load the jdbc driver |
710 | 722 |
try { |
... | ... | |
717 | 729 |
|
718 | 730 |
|
719 | 731 |
/** |
720 |
* Loads Harvester properties from a configuration file. |
|
721 |
*/ |
|
722 |
private Properties loadProperties() { |
|
723 |
InputStream inputStream; |
|
724 |
String path = "/harvester/harvester.properties"; |
|
725 |
Properties properties = new Properties(); |
|
726 |
ServletContext servletContext = getServletContext(); |
|
727 |
|
|
728 |
try { |
|
729 |
inputStream = servletContext.getResourceAsStream(path); |
|
730 |
properties.load(inputStream); |
|
731 |
properties.list(System.out); |
|
732 |
} |
|
733 |
catch (IOException e) { |
|
734 |
System.out.println("IOException: " + e.getMessage()); |
|
735 |
} |
|
736 |
|
|
737 |
return properties; |
|
738 |
} |
|
739 |
|
|
740 |
|
|
741 |
/** |
|
742 | 732 |
* Surrounds a string with single quotes. |
743 | 733 |
* @param str the original string |
744 | 734 |
* @return the quoted string |
Also available in: Unified diff
Integrate Harvester properties with build.properties and metacat.properties.
Eliminate the need for a separate harvester.properties file.
Improve email message to the site to include more summary information.