Project

General

Profile

« Previous | Next » 

Revision 2031

Additional development of Harvester implementation

View differences:

src/edu/ucsb/nceas/metacat/harvesterClient/HarvestDetailLog.java
6 6

  
7 7
package edu.ucsb.nceas.metacat.harvesterClient;
8 8

  
9
import java.sql.Connection;
10
import java.sql.SQLException;
11
import java.sql.Statement;
12

  
9 13
/**
10 14
 * HarvestDetailLog manages data and operations corresponding to the
11 15
 * HARVEST_DETAIL_LOG table. It records errors encountered while attempting
12
 * to harvest a particular document.
16
 * to harvest a particular EML document.
13 17
 * 
14 18
 * @author  costa
15 19
 */
16 20
public class HarvestDetailLog {
17 21
    
22
  private Harvester harvester;              // The parent Harvester object
18 23
  private int detailLogID;
24
  private int harvestLogID;
25
  private HarvestDocument harvestDocument;  // The associated HarvestDocument
19 26
  private String errorMessage;
20
  private Object harvestDocument;
21
  private int harvestLogID;
22 27
    
23 28

  
24 29
  /** 
25
   * Creates a new instance of HarvestDetailLog.
30
   * Creates a new instance of HarvestDetailLog and inserts the data into
31
   * the HARVEST_DETAIL_LOG table.
32
   *
33
   * @param  harvester       the Harvester parent object
34
   * @param  harvestLogID    foreign key value matching the HARVEST_LOG table
35
   * @param  harvestDocument HarvestDocument object that generated an error
36
   * @param  errorMessage    text of the error message
26 37
   */
27
  public HarvestDetailLog() {
38
  public HarvestDetailLog(Harvester       harvester,
39
                          int             harvestLogID,
40
                          HarvestDocument harvestDocument,
41
                          String          errorMessage
42
                         ) {
43
    this.harvester = harvester;
44
    this.detailLogID = harvester.getDetailLogID();  // Primary key for record
45
    this.harvestLogID = harvestLogID;
46
    this.harvestDocument = harvestDocument;
47
    this.errorMessage = errorMessage;
28 48
  }
49
    
29 50

  
30

  
31 51
  /**
32 52
   * Inserts a new entry into the HARVEST_DETAIL_LOG table, based on the 
33
   * contents of this HarvestDetailLog object. Not yet implemented.
53
   * contents of this HarvestDetailLog object.
34 54
   */
35
  public void dnInsertHarvestDetailLogEntry() {
55
  void dbInsertHarvestDetailLogEntry() {
56
    String insertString;
57
		Statement stmt;
58

  
59
    // Set the value of the HARVEST_LOG_ID to the current time in UTC seconds
60
    insertString = "INSERT INTO HARVEST_DETAIL_LOG " +
61
                   "(DETAIL_LOG_ID, HARVEST_LOG_ID, SCOPE," + 
62
                   " IDENTIFIER, REVISION," +
63
                   " DOCUMENT_URL, ERROR_MESSAGE, DOCUMENT_TYPE) " +
64
                   "values(" +
65
                   detailLogID + ", " +
66
                   harvestLogID + ", " +
67
                   "'" + harvestDocument.scope + "', " +
68
                   harvestDocument.identifier + ", " +
69
                   harvestDocument.revision + ", " +
70
                   "'" + harvestDocument.documentURL + "', " +
71
                   "'" + errorMessage + "'," +
72
                   "'" + harvestDocument.documentType + "'" +
73
                   ")";
74
                   
75
		try {
76
			stmt = harvester.conn.createStatement();						
77
			stmt.executeUpdate(insertString);
78
			stmt.close();
79
		}
80
    catch(SQLException e) {
81
			System.out.println("SQLException: " + e.getMessage());
82
		}
36 83
  }
37
    
84

  
85

  
86
  /**
87
   * Prints the contents of this HarvestLog object. Used in generating reports.
88
   */
89
  void printOutput() {
90
    System.out.println("detailLogID:          " + detailLogID);
91
    System.out.println("errorMessage:         " + errorMessage);
92
    harvestDocument.printOutput();
93
  }
94

  
38 95
}
src/edu/ucsb/nceas/metacat/harvesterClient/Harvester.java
6 6

  
7 7
package edu.ucsb.nceas.metacat.harvesterClient;
8 8

  
9
import java.io.*;
10
import java.sql.*;
11
import java.util.*;
12
import javax.xml.parsers.*;
13
import org.xml.sax.*;
14
import org.xml.sax.helpers.*;
9
import java.io.File;
10
import java.io.FileInputStream;
11
import java.io.IOException;
12
import java.sql.Connection;
13
import java.sql.DriverManager;
14
import java.sql.ResultSet;
15
import java.sql.SQLException;
16
import java.sql.SQLWarning;
17
import java.sql.Statement;
18
import java.util.ArrayList;
19
import java.text.SimpleDateFormat;
20
import java.util.Date;
21
import java.util.Properties;
15 22

  
16
import edu.ucsb.nceas.metacat.client.*;
23
import edu.ucsb.nceas.metacat.client.Metacat;
24
import edu.ucsb.nceas.metacat.client.MetacatFactory;
25
import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;
17 26

  
18 27
/**
19 28
 * Harvester is the main class for the Harvester application. The main
......
33 42
   * Class methods
34 43
   */
35 44
   
45

  
36 46
  /**
37 47
   * Constructor. Creates a new instance of Harvester.
38 48
   */
39 49
  public Harvester() {
40 50
  }
41 51
    
52

  
42 53
  /**
43 54
   * Determines whether Harvester is running on a Win32 platform. Used
44
   * during development to aid in resolving platform dependencies.
55
   * during development.
45 56
   * 
46 57
   * @return    true if this in Win32, false otherwise
47 58
   */
......
76 87
   * Object fields
77 88
   */
78 89

  
90
  /** Database connection */
79 91
  Connection conn;
80
  private Object harvestEndTime;
81
  private Object harvestLogList;
82
  private HarvestSiteSchedule[] harvestSiteScheduleList = 
83
          new HarvestSiteSchedule[20];
84
  private int harvestSiteScheduleIndex = 0;
85
  private Object harvestStartTime;
92
  
93
  /** Used during development to determine whether to connect to metacat */
94
  private boolean connectToMetacat;
95

  
96
  /** Highest DETAIL_LOG_ID primary key in the HARVEST_DETAIL_LOG table */
97
  private int detailLogID;
98
  
99
  /** Highest HARVEST_LOG_ID primary key in the HARVEST_LOG table */
100
  private int harvestLogID;
101
  
102
  /** End time of this harvest session */
103
  private Date harvestEndTime;
104
  
105
  /** List of HarvestLog objects. Stores log entries for report generation. */
106
  private ArrayList harvestLogList = new ArrayList();
107
  
108
  /** List of HarvestSiteSchedule objects */
109
  private ArrayList harvestSiteScheduleList = new ArrayList();
110
  
111
  /** Start time of this harvest session */
112
  private Date harvestStartTime;
113
  
114
  /** Number of days to save log records. Any that are older are purged. */
115
  int logPeriod;
116
  
117
  /** Metacat client object */
86 118
  Metacat metacat;
119
  
120
  /** Loads harvester properties from configuration file */
87 121
  Properties properties;
88 122
    
89 123

  
90 124
  /*
91 125
   * Object methods
92 126
   */
127
   
128
   
129
  /*                
130
                    Harvester  harvester,
131
                    Date       harvestDate,
132
                    int        status,
133
                    String     message, 
134
                    String     harvestOperationCode,
135
                    int        siteScheduleID,
136
                    HarvestDocument harvestDocument,
137
                    String     errorMessage 
138
*/
139
  /**
140
   * Creates a new HarvestLog object and adds it to the harvestLogList.
141
   * 
142
   * @param  status          the status of the harvest operation
143
   * @param  message         the message text of the harvest operation
144
   * @param  harvestOperationCode  the harvest operation code
145
   * @param  siteScheduleID  the siteScheduleID for which this operation was
146
   *                         performed. 0 indicates that the operation did not
147
   *                         involve a particular harvest site.
148
   * @param  harvestDocument the associated HarvestDocument object. May be null.
149
   * @param  errorMessage    additional error message pertaining to document
150
   *                         error.
151
   */
152
  void addLogEntry(int    status,
153
                   String message,
154
                   String harvestOperationCode,
155
                   int    siteScheduleID,
156
                   HarvestDocument harvestDocument,
157
                   String errorMessage
158
                  ) {
159
    HarvestLog harvestLog;
93 160

  
161
    /* If there is no associated harvest document, call the basic constructor;
162
     * else call the extended constructor.
163
     */
164
    if (harvestDocument == null) {    
165
      harvestLog = new HarvestLog(this, harvestStartTime, status, message,
166
                                  harvestOperationCode, siteScheduleID);
167
    }
168
    else {
169
      harvestLog = new HarvestLog(this, harvestStartTime, status, message,
170
                                  harvestOperationCode, siteScheduleID,
171
                                  harvestDocument, errorMessage);
172
    }
173
    
174
    harvestLogList.add(harvestLog);
175
  }
176

  
177

  
94 178
  /**
95 179
   * Determines whether Harvester should attempt to connect to Metacat.
96 180
   * Used during development and testing.
......
98 182
   * @return     true if Harvester should connect, otherwise false
99 183
   */
100 184
  boolean connectToMetacat () {
101
    boolean connectToMetacat = Harvester.isWin32() ? false : true;
102
    
103 185
    return connectToMetacat;
104 186
  }
105 187
    
106 188

  
107 189
  /**
190
   * Gets the current value of the detailLogID for storage as a primary key in
191
   * the DETAIL_LOG_ID field of the HARVEST_DETAIL_LOG table.
192
   * 
193
   * @return  the current value of the detailLogID
194
   */
195
  int getDetailLogID() {
196
    int currentValue = detailLogID;
197
    
198
    detailLogID++;
199
    return currentValue;
200
  }
201
  
202
  
203
  /**
204
   * Gets the current value of the harvestLogID for storage as a primary key in
205
   * the HARVEST_LOG_ID field of the HARVEST_LOG table.
206
   * 
207
   * @return  the current value of the detailLogID
208
   */
209
  int getHarvestLogID() {
210
    int currentValue = harvestLogID;
211
    
212
    harvestLogID++;
213
    return currentValue;
214
  }
215
  
216

  
217
  /** 
218
   * Gets the maximum value of an integer field from a table.
219
   * 
220
   * @param tableName  the database table name
221
   * @param fieldName  the field name of the integer field in the table
222
   * @return  the maximum integer stored in the fieldName field of tableName
223
   */
224
  private int getMaxValue(String tableName, String fieldName) {
225
    int maxValue = 100;
226
    int fieldValue;
227
		String query = "SELECT " + fieldName + " FROM " + tableName;
228
		Statement stmt;
229
    
230
		try {
231
			stmt = conn.createStatement();							
232
			ResultSet rs = stmt.executeQuery(query);
233
	
234
			while (rs.next()) {
235
				fieldValue = rs.getInt(fieldName);
236
        maxValue = Math.max(maxValue, fieldValue);
237
			}
238
	
239
			stmt.close();	
240
		} 
241
    catch(SQLException ex) {
242
			System.out.println("SQLException: " + ex.getMessage());
243
		}
244
    
245
    return maxValue;
246
  }
247
  
248
  
249
  /** 
250
   * Gets the minimum value of an integer field from a table.
251
   * 
252
   * @param tableName  the database table name
253
   * @param fieldName  the field name of the integer field in the table
254
   * @return  the minimum integer stored in the fieldName field of tableName
255
   */
256
  private int getMinValue(String tableName, String fieldName) {
257
    int minValue = 0;
258
    int fieldValue;
259
		String query = "SELECT " + fieldName + " FROM " + tableName;
260
		Statement stmt;
261
    
262
		try {
263
			stmt = conn.createStatement();							
264
			ResultSet rs = stmt.executeQuery(query);
265
	
266
			while (rs.next()) {
267
				fieldValue = rs.getInt(fieldName);
268

  
269
        if (minValue == 0) {
270
          minValue = fieldValue;
271
        }
272
        else {
273
          minValue = Math.min(minValue, fieldValue);
274
        }
275
			}
276
	
277
			stmt.close();	
278
		} 
279
    catch(SQLException ex) {
280
			System.out.println("SQLException: " + ex.getMessage());
281
		}
282
    
283
    return minValue;
284
  }
285
  
286
  
287
  /**
108 288
   * For every Harvest site schedule in the database, harvest the
109 289
   * documents for that site if they are due to be harvested.
110 290
   * 
......
115 295
  private void harvest() {
116 296
    HarvestSiteSchedule harvestSiteSchedule;
117 297

  
118
    for (int i = 0; i < harvestSiteScheduleList.length; i++) {
119
      harvestSiteSchedule = harvestSiteScheduleList[i];
120
      if (harvestSiteSchedule != null) {
121
        harvestSiteSchedule.printOutput();
122
        harvestSiteSchedule.harvestDocumentList();
123
      }
298
    for (int i = 0; i < harvestSiteScheduleList.size(); i++) {
299
      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);
300
      harvestSiteSchedule.harvestDocumentList();
124 301
    }
125
        
126
    reportToAdministrator();
127 302
  }
128 303
  
129 304
  
130 305
  /**
131
   * Loads Harvester properties
306
   * Initializes the detailLogID and harvestLogID values to their current
307
   * maximums + 1.
132 308
   */
309
  private void initLogIDs() {
310
    detailLogID = getMaxValue("HARVEST_DETAIL_LOG", "DETAIL_LOG_ID") + 1;
311
    harvestLogID = getMaxValue("HARVEST_LOG", "HARVEST_LOG_ID") + 1;
312
  }
313
  
314
  
315
  /**
316
   * Loads Harvester properties from a configuration file.
317
   */
133 318
  private void loadProperties() {
134 319
    String homedir = System.getProperty("user.home");
135 320
    File configfile = new File(homedir, "harvester.properties");
......
141 326
      properties.list(System.out);
142 327
    }
143 328
    catch (IOException e) {
144
      System.err.println("IOException: " + e.getMessage());
329
      System.out.println("IOException: " + e.getMessage());
145 330
      System.exit(1);
146 331
    }
147 332
  }
333
  
334
  
335
  void printHarvestSiteSchedule(int siteScheduleID) {
336
     HarvestSiteSchedule harvestSiteSchedule;
337

  
338
    for (int i = 0; i < harvestSiteScheduleList.size(); i++) {
339
      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);
340
      if (harvestSiteSchedule.siteScheduleID == siteScheduleID) {
341
        harvestSiteSchedule.printOutput();
342
      }
343
    }
344
  }
345
  
346

  
347
  /**
348
   * Prunes old records from the HARVEST_DETAIL_LOG table. Records are
349
   * removed if the HARVEST_LOG_ID foreign key is less than the lowest
350
   * HARVEST_LOG_ID primary key in the HARVEST_LOG table.
351
   */
352
  private void pruneHarvestDetailLog() {
353
		String deleteString;
354
    int minHarvestLogID;
355
    int recordsDeleted;
356
		Statement stmt;
148 357
    
358
    minHarvestLogID = getMinValue("HARVEST_LOG", "HARVEST_LOG_ID");
359
    deleteString = "DELETE FROM HARVEST_DETAIL_LOG WHERE HARVEST_LOG_ID < " +
360
                   minHarvestLogID;
149 361

  
362
		try {
363
			System.out.print("Pruning log entries from HARVEST_DETAIL_LOG: ");
364
			stmt = conn.createStatement();							
365
			recordsDeleted = stmt.executeUpdate(deleteString);
366
			System.out.println(recordsDeleted + " records deleted");
367
			stmt.close();
368
		}
369
    catch(SQLException e) {
370
			System.out.println("SQLException: " + e.getMessage());
371
		}
372
  }
373
    
374

  
150 375
  /**
376
   * Prunes old records from the HARVEST_LOG table. Records are removed if
377
   * their HARVEST_DATE is older than a given number of days, as stored in the
378
   * logPeriod object field.
379
   */
380
  private void pruneHarvestLog() {
381
    long currentTime = harvestStartTime.getTime(); // time in milliseconds
382
    Date dateLastLog;                    // Prune everything prior to this date
383
		String deleteString;
384
    long delta;
385
    final long millisecondsPerDay = (1000 * 60 * 60 * 24);
386
    int recordsDeleted;
387
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MMM-yyyy");
388
    String dateString;
389
		Statement stmt;
390
    long timeLastLog = 0;
391
    
392
    delta = logPeriod * millisecondsPerDay;
393
    deleteString = "DELETE FROM HARVEST_LOG WHERE HARVEST_DATE < ";
394
    timeLastLog = currentTime - delta;
395
    dateLastLog = new Date(timeLastLog);
396
    dateString = "'" + simpleDateFormat.format(dateLastLog) + "'";
397
    deleteString += dateString;
398

  
399
		try {
400
			System.out.print("Pruning log entries from HARVEST_LOG: ");
401
			stmt = conn.createStatement();							
402
			recordsDeleted = stmt.executeUpdate(deleteString);
403
			System.out.println(recordsDeleted + " records deleted");
404
			stmt.close();
405
		}
406
    catch (SQLException e) {
407
			System.out.println("SQLException: " + e.getMessage());
408
		}
409
  }
410
    
411

  
412
  /**
151 413
   * Reads the HARVEST_SITE_SCHEDULE table in the database, creating
152 414
   * a HarvestSiteSchedule object for each row in the table.
153 415
   */
......
162 424
    String dateNextHarvest;
163 425
    String documentListURL;
164 426
    String ldapDN;
165
    String ldapPassword;
427
    String ldapPwd;
166 428
    int siteScheduleID;
167 429
    String unit;
168 430
    int updateFrequency;
......
190 452
        siteScheduleID = rs.getInt("SITE_SCHEDULE_ID");
191 453
        documentListURL = rs.getString("DOCUMENTLISTURL");
192 454
        ldapDN = rs.getString("LDAPDN");
193
//        ldapPassword = rs.getString("LDAPPASSWORD");
194
        ldapPassword = "ntre4dc";
455
        ldapPwd = rs.getString("LDAPPWD");
195 456
        dateNextHarvest = rs.getString("DATENEXTHARVEST");
196 457
        dateLastHarvest = rs.getString("DATELASTHARVEST");
197 458
        updateFrequency = rs.getInt("UPDATEFREQUENCY");
......
213 474
          }
214 475
        }
215 476
      
216
        harvestSiteSchedule = new HarvestSiteSchedule(
217
                                                      this,
477
        harvestSiteSchedule = new HarvestSiteSchedule(this,
218 478
                                                      siteScheduleID,
219 479
                                                      documentListURL,
220 480
                                                      ldapDN,
221
                                                      ldapPassword,
481
                                                      ldapPwd,
222 482
                                                      dateNextHarvest,
223 483
                                                      dateLastHarvest,
224 484
                                                      updateFrequency,
225 485
                                                      unit,
226 486
                                                      contactEmail
227 487
                                                     );
228

  
229
        harvestSiteScheduleList[harvestSiteScheduleIndex] = harvestSiteSchedule;
230
        harvestSiteScheduleIndex++;
488
        harvestSiteScheduleList.add(harvestSiteSchedule);
231 489
      }
232
    } 
490
      
491
      rs.close();
492
      stmt.close();
493
    }
233 494
    catch (SQLException e) {
234 495
      System.out.println("Database access failed " + e);
235 496
      System.exit(1);
......
253 514
  private void shutdown() {
254 515
    // Log shutdown operation
255 516
    System.out.println("Shutting Down Harvester");
517
    addLogEntry(0, "Shutting Down Harvester", "HarvesterShutdown", 0, null, "");
518
    pruneHarvestLog();
519
    pruneHarvestDetailLog();
256 520

  
257 521
    try {
258 522
      // Close the database connection
......
262 526
    catch (SQLException e) {
263 527
      System.out.println("Database access failed " + e);
264 528
    }
529
    
530
    writeHarvestLog();
531
    reportToAdministrator();
265 532
  }
266 533
    
267 534

  
......
269 536
   * Initializes Harvester at startup. Connects to the database and to Metacat.
270 537
   */
271 538
  private void startup() {
539
    Boolean ctm;
272 540
    String dbDriver;
541
    Integer lp;
273 542
    String metacatURL;
274 543
    String osName = Harvester.isWin32() ? "Windows" : "Unix";
275 544
    String password;
276
//    String response;
545
    //String response;
277 546
    String sessionId;
278 547
		String url;
279 548
    String user;
......
283 552
    // Log startup operation
284 553
    System.out.println("*****************************************************");
285 554
    System.out.println("Starting Up Harvester");
286
    System.out.println("OS is " + osName);
287
    
288 555
    loadProperties();
289
    
556
    ctm = Boolean.valueOf(properties.getProperty("connectToMetacat", "true"));
557
    connectToMetacat = ctm.booleanValue();
290 558
    dbDriver = properties.getProperty("dbDriver");
559
    try {
560
      lp = Integer.valueOf(properties.getProperty("logPeriod", "90"));
561
      logPeriod = lp.intValue();
562
    }
563
    catch (NumberFormatException e) {
564
      System.err.println("NumberFormatException: Error parsing logPeriod " +
565
                         logPeriod + e.getMessage());
566
      System.err.println("Defaulting to logPeriod of 90 days");
567
      logPeriod = 90;
568
    }
569
    metacatURL = properties.getProperty("metacatURL");
570
    password = properties.getProperty("password");
291 571
    url = properties.getProperty("url");
292 572
    user = properties.getProperty("user");
293
    password = properties.getProperty("password");
294
    metacatURL = properties.getProperty("metacatURL");
295 573

  
296 574
    // Load the jdbc driver
297 575
    try {
......
325 603
      System.out.println("Database access failed " + e);
326 604
      System.exit(1);
327 605
    }
606
    
607
    initLogIDs();
608
    harvestStartTime = new Date();
609
    addLogEntry(0, "Starting Up Harvester", "HarvesterStartup", 0, null, "");
328 610
      
329 611
    if (connectToMetacat()) {      
330 612
      try {
......
338 620
        System.out.println("Metacat connection failed." + e.getMessage());
339 621
      }
340 622
    }
341
    else {
342
      System.out.println("Not connecting to Metacat");
343
    }
344 623
  }
345 624

  
346 625

  
......
348 627
   * Writes one or more log entries to the HARVEST_LOG table.
349 628
   */
350 629
  private void writeHarvestLog() {
630
    HarvestLog harvestLog;
631
    
632
    for (int i = 0; i < harvestLogList.size(); i++) {
633
      harvestLog = (HarvestLog) harvestLogList.get(i);
634
      harvestLog.printOutput();
635
    }
351 636
  }
352 637
    
353 638
}
src/edu/ucsb/nceas/metacat/harvesterClient/HarvestDocument.java
13 13
import java.net.MalformedURLException;
14 14
import java.net.URL;
15 15

  
16
import edu.ucsb.nceas.metacat.client.*;
16
import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;
17
import edu.ucsb.nceas.metacat.client.Metacat;
18
import edu.ucsb.nceas.metacat.client.MetacatException;
19
import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;
17 20
import edu.ucsb.nceas.utilities.IOUtil;
18 21

  
19 22

  
......
26 29
public class HarvestDocument {
27 30
  
28 31
  private String documentName;  
29
  private String documentType;
30
  private String documentURL;
32
  String documentType;
33
  String documentURL;
31 34
  private Harvester harvester;
32 35
  private HarvestSiteSchedule harvestSiteSchedule;
33
  private int identifier;
34
  private int revision;
35
  private String scope;
36
  int identifier;
37
  int revision;
38
  String scope;
36 39
    
37 40

  
38 41
  /**
......
86 89
      inputStreamReader = new InputStreamReader(inputStream);
87 90
      documentString = IOUtil.getAsString(inputStreamReader, true);
88 91
      stringReader = new StringReader(documentString);
89
      System.out.println("  Successfully read document: " + documentURL);
92
      harvester.addLogEntry(0, "", "GetDocSuccess", 
93
                            harvestSiteSchedule.siteScheduleID, null, "");
90 94
    }
91 95
    catch (MalformedURLException e) {
92
      System.err.println("MalformedURLException: " + e.getMessage());
96
      harvester.addLogEntry(1, "MalformedURLException", "GetDocError", 
97
                            harvestSiteSchedule.siteScheduleID, this, 
98
                            "MalformedURLException: " + e.getMessage());
93 99
    }
94 100
    catch (IOException e) {
95
      System.err.println("IOException: " + e.getMessage());
101
      harvester.addLogEntry(1, "IOException", "GetDocError", 
102
                            harvestSiteSchedule.siteScheduleID, this, 
103
                            "IOException: " + e.getMessage());
96 104
    }
97 105
    
98 106
    return stringReader;
......
116 124
     * into metacat.
117 125
     */
118 126
    if (metacatHasDocument()) {
119
      System.out.println("  metacat has document");
120 127
      highestRevision = metacatHighestRevision();
121
      System.out.println("  metacatHighestRevision: " + highestRevision);
128
      harvester.addLogEntry(0, 
129
                            "Metacat has document: " + documentName +
130
                            ", highest revision: " + highestRevision, 
131
                            "MetacatHasDoc", 
132
                            harvestSiteSchedule.siteScheduleID, null, "");
122 133
    }
123 134
    else {
124 135
      stringReader = getSiteDocument();
125 136
      if (stringReader != null) {
126 137
        if (parseDocument()) {
127
          metacatReturnString = putMetacatDocument(stringReader);
128
          System.out.println("  " + metacatReturnString);
138
          putMetacatDocument(stringReader);
129 139
        }
130
        else {
131
          System.out.println("Error parsing document.");
132
        }
133 140
      }
134
      else {
135
        System.out.print("  Error reading document at URL: ");
136
        System.out.println(documentURL);
137
      }
138 141
    }
139 142
  }
140
    
143
  
141 144

  
142 145
  /**
146
   * Logs a metacat document error to the harvest detail log. 
147
   *
148
   * @param insert               true if insert operation, false is update
149
   * @param metacatReturnString  string returned from the insert or update
150
   * @param exceptionName        name of the exception class
151
   * @param e                    the exception object
152
   */
153
  private void logMetacatError (boolean insert, 
154
                                String metacatReturnString,
155
                                String exceptionName,
156
                                Exception e
157
                               ) {
158
    if (insert) {
159
      harvester.addLogEntry(1, metacatReturnString, "InsertDocError", 
160
                                harvestSiteSchedule.siteScheduleID, 
161
                                this, exceptionName + ": " + e.getMessage());
162
    }
163
    else {
164
      harvester.addLogEntry(1, metacatReturnString, "UpdateDocError", 
165
                                harvestSiteSchedule.siteScheduleID, 
166
                                this, exceptionName + ": " + e.getMessage());
167
    }
168
  }
169
  
170

  
171
  /**
143 172
   * Boolean to determine whether Metacat already has this document.
144 173
   * 
145 174
   * @return  true if Metacat has the document, otherwise false
......
172 201
  private boolean parseDocument () {
173 202
    boolean success = true;
174 203
    
204
    if (success) {
205
      harvester.addLogEntry(0, "", "ValidateDocSuccess", 
206
                            harvestSiteSchedule.siteScheduleID, null, "");
207
    }
208
    else {
209
      harvester.addLogEntry(1, "Error validating document", "ValidateDocError", 
210
                            harvestSiteSchedule.siteScheduleID, this, "");
211
    }
212
    
175 213
    return success;
176 214
  }
177 215
  
......
179 217
  /**
180 218
   * Print the data fields and values in this HarvestDocument object.
181 219
   */
182
  void printOutput() 
183
  {
184
    System.out.println("");
185
    System.out.println("  scope: " + scope);
186
    System.out.println("  identifier: " + identifier);
187
    System.out.println("  revision: " + revision);
188
    System.out.println("  documentType: " + documentType);
189
    System.out.println("  documentURL: " + documentURL);
190
    System.out.println("  documentName: " + documentName);
220
  void printOutput() {
221
    System.out.println("scope:                " + scope);
222
    System.out.println("identifier:           " + identifier);
223
    System.out.println("revision:             " + revision);
224
    System.out.println("documentType:         " + documentType);
225
    System.out.println("documentURL:          " + documentURL);
226
    System.out.println("documentName:         " + documentName);
191 227
  }
192 228
 
193 229
 
194 230
  /**
195 231
   * Insert or update this document to Metacat. If revision equals 1, do an
196 232
   * insert; otherwise, do an update.
197
   * 
198
   * @return  the Metacat return string from the insert or update operation
199 233
   */
200
  private String putMetacatDocument(StringReader stringReader) {
234
  private void putMetacatDocument(StringReader stringReader) {
201 235
    String docid = scope + "." + identifier + "." + revision;
236
    boolean insert = (revision == 1);
202 237
    Metacat metacat = harvester.metacat;
203 238
    String metacatReturnString = "";
204

  
239
    
205 240
    if (harvester.connectToMetacat()) {
206 241
      try {
207
        if (revision == 1) {
208
          System.out.println("  Inserting document to metacat: " + docid);
242
        if (insert) {
209 243
          metacatReturnString = metacat.insert(docid, stringReader, null);
244
          harvester.addLogEntry(0, docid + " : " + metacatReturnString, 
245
                                "InsertDocSuccess", 
246
                                harvestSiteSchedule.siteScheduleID, 
247
                                null, "");
210 248
        }
211 249
        else {
212
          System.out.println("  Updating document to metacat: " + docid);
213 250
          metacatReturnString = metacat.update(docid, stringReader, null);
251
          harvester.addLogEntry(0, docid + " : " + metacatReturnString, 
252
                                "UpdateDocSuccess", 
253
                                harvestSiteSchedule.siteScheduleID, 
254
                                null, "");
214 255
        }
215 256
      }
216 257
      catch (MetacatInaccessibleException e) {
217
        System.err.println("MetacatInaccessibleException: " + e.getMessage());
258
        logMetacatError(insert, metacatReturnString, 
259
                        "MetacatInaccessibleException", e);
218 260
      }
219 261
      catch (InsufficientKarmaException e) {
220
        System.err.println("InsufficientKarmaException: " + e.getMessage());
262
        logMetacatError(insert, metacatReturnString, 
263
                        "InsufficientKarmaException", e);
221 264
      }
222 265
      catch (MetacatException e) {
223
        System.err.println("MetacatException: " + e.getMessage());
266
        logMetacatError(insert, metacatReturnString, "MetacatException", e);
224 267
      }
225 268
      catch (IOException e) {
226
        System.err.println("IOException: " + e.getMessage());
269
        logMetacatError(insert, metacatReturnString, "IOException", e);
227 270
      }
228 271
    }
229
    else {
230
      metacatReturnString = "Not putting document to metacat";
231
    }
232
        
233
    return metacatReturnString;
234 272
  }
235
    
236 273
}
src/edu/ucsb/nceas/metacat/harvesterClient/HarvestLog.java
6 6

  
7 7
package edu.ucsb.nceas.metacat.harvesterClient;
8 8

  
9
import java.sql.Connection;
10
import java.sql.ResultSet;
11
import java.sql.SQLException;
12
import java.sql.SQLWarning;
13
import java.sql.Statement;
14
import java.text.SimpleDateFormat;
15
import java.util.Date;
16

  
17

  
9 18
/**
10 19
 * Manages log entries to be inserted to the HARVEST_LOG table.
11 20
 *
......
13 22
 */
14 23
public class HarvestLog {
15 24
    
16
  private Object harvestDate;
25
  private Harvester harvester;              // The parent Harvester object
26
  private int harvestLogID;
27
  private Date harvestDate;
28
  private int status;
29
  private String message;
17 30
  private String harvestOperationCode;
18
  private String message;
19 31
  private int siteScheduleID;
20
  private String status;
32
  private String timestamp;
33
  private HarvestDetailLog harvestDetailLog;  // Associated detail log, if any
21 34
    
35

  
22 36
  /** 
23
    * Creates a new instance of HarvestLog.
24
    */
25
  public HarvestLog() {
37
   * Creates a new instance of HarvestLog. This constructor is used when
38
   * creating log entries that do not involve an error on a harvest document.
39
   * (For that type of log entry, use the alternate constructor below.)
40
   *
41
   * @param  harvester       the parent Harvester object
42
   * @param  harvestDate     the date of this harvest
43
   * @param  status          the status of the harvest operation
44
   * @param  message         the message text of the harvest operation
45
   * @param  harvestOperationCode  the harvest operation code
46
   * @param  siteScheduleID  the siteScheduleID for which this operation was
47
   *                         performed. 0 indicates that the operation did not
48
   *                         involve a particular harvest site.
49
   */
50
  public HarvestLog(Harvester  harvester,
51
                    Date       harvestDate,
52
                    int        status,
53
                    String     message, 
54
                    String     harvestOperationCode,
55
                    int        siteScheduleID
56
                   ) {
57
    Date now = new Date();
58
    timestamp = now.toString();
59

  
60
    this.harvester = harvester;
61
    this.harvestLogID = harvester.getHarvestLogID();
62
    this.harvestDate = harvestDate;
63
    this.status = status;
64
    this.message = message;
65
    this.harvestOperationCode = harvestOperationCode;
66
    this.siteScheduleID = siteScheduleID;
67
    
68
    dbInsertHarvestLogEntry();   // Insert this entry to the HARVEST_LOG table
26 69
  }
27 70
    
28 71

  
72
  /** 
73
   * Creates a new instance of HarvestLog and inserts this entry to the
74
   * HARVEST_LOG table. This version of the constructor also instantiates a 
75
   * HarvestDetailLog object and inserts it to the HARVEST_DETAIL_LOG table.
76
   *
77
   * @param  harvester       the parent Harvester object
78
   * @param  harvestDate     the date of this harvest
79
   * @param  status          the status of the harvest operation
80
   * @param  message         the message text of the harvest operation
81
   * @param  harvestOperationCode  the harvest operation code
82
   * @param  siteScheduleID  the siteScheduleID for which this operation was
83
   *                         performed. 0 indicates that the operation did not
84
   *                         involve a particular harvest site.
85
   */
86
  public HarvestLog(Harvester  harvester,
87
                    Date       harvestDate,
88
                    int        status,
89
                    String     message, 
90
                    String     harvestOperationCode,
91
                    int        siteScheduleID,
92
                    HarvestDocument harvestDocument,
93
                    String     errorMessage
94
                   ) {
95
    Date now = new Date();
96
    timestamp = now.toString();
97

  
98
    this.harvester = harvester;
99
    this.harvestLogID = harvester.getHarvestLogID();
100
    this.harvestDate = harvestDate;
101
    this.status = status;
102
    this.message = message;
103
    this.harvestOperationCode = harvestOperationCode;
104
    this.siteScheduleID = siteScheduleID;
105
    this.harvestDetailLog = new HarvestDetailLog(harvester, harvestLogID, 
106
                                                 harvestDocument, errorMessage);
107
    
108
    dbInsertHarvestLogEntry();               // Insert to the HARVEST_LOG table
109
    harvestDetailLog.dbInsertHarvestDetailLogEntry(); // and HARVEST_DETAIL_LOG
110
  }
111
    
112

  
29 113
  /**
30 114
   * Retrieves the value of the EXPLANATION field of the HARVEST_OPERATION
31 115
   * table based on the value of the HARVEST_OPERATION_CODE field. 
32
   * Not yet implemented.
33 116
   * 
34
   * @param harvestOperationCode  string value of the harvest operation code
35
   * @return           the explanation for this harvest operation, a String
117
   * @param  harvestOperationCode  string value of the harvest operation code
118
   * @return the explanation for this harvest operation, a String
36 119
   */
37
  public String dbGetExplanation(String harvestOperationCode) {
38
    String explanation = "";
120
  String dbGetExplanation(String harvestOperationCode) {
121
    String explanation;
122
    String fieldName = "EXPLANATION";
123
    
124
    explanation = dbQueryHarvestOperation(fieldName, harvestOperationCode);
39 125
        
40 126
    return explanation;
41 127
  }
......
44 130
  /**
45 131
   * Retrieves the value of the HARVEST_OPERATION_CODE_LEVEL field of the
46 132
   * HARVEST_OPERATION table based on the value of the HARVEST_OPERATION_CODE 
47
   * field. Not yet implemented.
133
   * field.
48 134
   * 
49
   * @param harvestOperationCode  string value of the harvest operation code
50
   * @return          the code level value, an int
135
   * @param  harvestOperationCode  string value of the harvest operation code
136
   * @return the code level value, a String, one of the following:
137
   *         "error", "warning", "notice", "info", or "debug"
51 138
   */
52
  public int dbGetHarvestOperationCodeLevel(String harvestOperationCode) {
53
    int codeLevel = 0;
139
  String dbGetHarvestOperationCodeLevel(String harvestOperationCode) {
140
    String harvestOperationCodeLevel;
141
    String fieldName = "HARVEST_OPERATION_CODE_LEVEL";
142
    
143
    harvestOperationCodeLevel = 
144
            dbQueryHarvestOperation(fieldName, harvestOperationCode);
54 145
        
55
    return codeLevel;
146
    return harvestOperationCodeLevel;
56 147
  }
57
    
148
  
58 149

  
59 150
  /**
60 151
   * Inserts a new entry into the HARVEST_LOG table, based on the contents of
61 152
   * this HarvestLog object. Not yet implemented.
62 153
   */
63
  public void dbInsertHarvestLogEntry() {
154
  void dbInsertHarvestLogEntry() {
155
    String insertString;
156
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MMM-yyyy");
157
		Statement stmt;
158

  
159
    insertString = "INSERT INTO HARVEST_LOG " +
160
                   "(HARVEST_LOG_ID, HARVEST_DATE, STATUS, MESSAGE," +
161
                   " HARVEST_OPERATION_CODE, SITE_SCHEDULE_ID) " +
162
                   "values(" +
163
                   harvestLogID + ", " +
164
                   "'" + simpleDateFormat.format(harvestDate) + "', " +
165
                   status + ", " +
166
                   "'" + timestamp + ": " + message + "', " +
167
                   "'" + harvestOperationCode + "', " +
168
                   siteScheduleID +
169
                   ")";
170
                   
171
		try {
172
			stmt = harvester.conn.createStatement();							
173
			stmt.executeUpdate(insertString);
174
			stmt.close();
175
		}
176
    catch(SQLException e) {
177
			System.out.println("SQLException: " + e.getMessage());
178
		}
64 179
  }
180
  
65 181

  
182
  /**
183
   * Retrieves the value of the either the EXPLANATION field or the
184
   * HARVEST_OPERATION_CODE_LEVEL field of the HARVEST_OPERATION table based on
185
   * the value of the HARVEST_OPERATION_CODE field.
186
   *
187
   * @param  fieldName  the field name to match, e.g. "EXPLANATION"
188
   * @param  harvestOperationCode  string value of the harvest operation code
189
   * @return the explanation string or the harvestOperationCodeLevel string
190
   */
191
  String dbQueryHarvestOperation(String fieldName,
192
                                 String harvestOperationCode
193
                                ) {
194
    String explanation = "No explanation available";
195
    String harvestOperationCodeLevel = "debug";
196
    String queryString;
197
    String returnString = "";
198
    ResultSet rs;
199
    Statement stmt;
200
    SQLWarning warn;
201
    
202
    queryString = "SELECT EXPLANATION, HARVEST_OPERATION_CODE_LEVEL " + 
203
                  "FROM HARVEST_OPERATION " +
204
                  "WHERE HARVEST_OPERATION_CODE=" +
205
                  "'" + harvestOperationCode + "'";
206
        
207
    try {                           // Query the HARVEST_OPERATION table
208
      stmt = harvester.conn.createStatement();
209
      rs = stmt.executeQuery(queryString);
210
      warn = rs.getWarnings();
211

  
212
      if (warn != null) {
213
        System.out.println("\n---Warning---\n");
214

  
215
        while (warn != null) {
216
          System.out.println("Message: " + warn.getMessage());
217
          System.out.println("SQLState: " + warn.getSQLState());
218
          System.out.print("Vendor error code: ");
219
          System.out.println(warn.getErrorCode());
220
          System.out.println("");
221
          warn = warn.getNextWarning();
222
        }
223
      }
224
     
225
      while (rs.next()) {
226
        explanation = rs.getString("EXPLANATION");
227
        harvestOperationCodeLevel = rs.getString("HARVEST_OPERATION_CODE_LEVEL");
228
        warn = rs.getWarnings();
229

  
230
        if (warn != null) {
231
          System.out.println("\n---Warning---\n");
232
      
233
          while (warn != null) {
234
            System.out.println("Message: " + warn.getMessage());
235
            System.out.println("SQLState: " + warn.getSQLState());
236
            System.out.print("Vendor error code: ");
237
            System.out.println(warn.getErrorCode());
238
            System.out.println("");
239
            warn = warn.getNextWarning();
240
          }
241
        }
242
      }
243
      
244
      rs.close();
245
      stmt.close();
246
    }
247
    catch (SQLException e) {
248
      System.out.println("SQLException: Database access failed: " + e);
249
    }
250
 
251
    if (fieldName.equals("EXPLANATION")) {
252
      returnString = explanation;
253
    }
254
    else if (fieldName.equals("HARVEST_OPERATION_CODE_LEVEL")) {
255
      returnString = harvestOperationCodeLevel;
256
    }
257
    
258
    return returnString;
259
  }
260

  
261

  
262
  /**
263
   * Prints the contents of this HarvestLog object. Used in generating reports.
264
   */
265
  void printOutput() {
266
    System.out.println("");
267
    System.out.println("harvestLogID:         " + harvestLogID);
268
    System.out.println("harvestDate:          " + harvestDate);
269
    System.out.println("status:               " + status);
270
    System.out.println("message:              " + message);
271
    System.out.println("harvestOperationCode: " + harvestOperationCode);
272
    
273
    if (siteScheduleID != 0) {
274
      harvester.printHarvestSiteSchedule(siteScheduleID);
275
    }
276
    
277
    if (harvestDetailLog != null) {
278
      harvestDetailLog.printOutput();
279
    }
280
  }
281

  
66 282
}
src/edu/ucsb/nceas/metacat/harvesterClient/HarvestSiteSchedule.java
6 6

  
7 7
package edu.ucsb.nceas.metacat.harvesterClient;
8 8

  
9
import java.io.*;
9
import java.io.FileNotFoundException;
10
import java.io.IOException;
11
import java.io.InputStream;
12
import java.io.InputStreamReader;
13
import java.io.Reader;
14
import java.net.MalformedURLException;
15
import java.net.URL;
10 16
import java.sql.Connection;
11 17
import java.sql.SQLException;
12 18
import java.sql.Statement;
13
import java.text.*;
14
import java.util.*;
15
import javax.xml.parsers.*;
16
import org.xml.sax.*;
17
import org.xml.sax.helpers.*;
19
import java.text.DateFormat;
20
import java.text.ParseException;
21
import java.text.SimpleDateFormat;
22
import java.util.ArrayList;
23
import java.util.Date;
24
import javax.xml.parsers.ParserConfigurationException;
25
import org.xml.sax.Attributes;
26
import org.xml.sax.ContentHandler;
27
import org.xml.sax.ErrorHandler;
28
import org.xml.sax.InputSource;
29
import org.xml.sax.SAXException;
30
import org.xml.sax.SAXParseException;
31
import org.xml.sax.XMLReader;
32
import org.xml.sax.helpers.DefaultHandler;
33
import org.xml.sax.helpers.XMLReaderFactory;
18 34

  
19
import edu.ucsb.nceas.metacat.client.*;
35
import edu.ucsb.nceas.metacat.client.Metacat;
36
import edu.ucsb.nceas.metacat.client.MetacatException;
37
import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;
20 38

  
21 39

  
22 40
/**
......
33 51
  private long delta;
34 52
  private String documentListURL;
35 53
  private Harvester harvester;
36
  private int harvestDocumentIndex = 0;
37
  private HarvestDocument[] harvestDocumentList = new HarvestDocument[30];
54
  private ArrayList harvestDocumentList = new ArrayList();
38 55
  private String harvestSiteEndTime;
39 56
  private String harvestSiteStartTime;
40 57
  private String ldapDN;
41
  private String ldapPassword;
58
  private String ldapPwd;
42 59
  final private long millisecondsPerDay = (1000 * 60 * 60 * 24);
43
  private int siteScheduleID;
60
  int siteScheduleID;
44 61
  private String unit;
45 62
  private int updateFrequency;
46 63
    
......
52 69
   * @param siteScheduleID  the value of the SITE_SCHEDULE_ID field
53 70
   * @param documentListURL the value of the DOCUMENTLISTURL field
54 71
   * @param ldapDN          the value of the LDAPDN field
55
   * @param ldapPassword    the value of the LDAPPASSWORD field
72
   * @param ldapPwd    the value of the LDAPPASSWORD field
56 73
   * @param dateNextHarvest the value of the DATENEXTHARVEST field
57 74
   * @param dateLastHarvest the value of the DATELASTHARVEST field
58 75
   * @param updateFrequency the value of the UPDATEFREQUENCY field
......
64 81
                              int    siteScheduleID,
65 82
                              String documentListURL,
66 83
                              String ldapDN,
67
                              String ldapPassword,
84
                              String ldapPwd,
68 85
                              String dateNextHarvest,
69 86
                              String dateLastHarvest,
70 87
                              int    updateFrequency,
......
76 93
    this.siteScheduleID = siteScheduleID;
77 94
    this.documentListURL = documentListURL;
78 95
    this.ldapDN = ldapDN;
79
    this.ldapPassword = ldapPassword;
96
    this.ldapPwd = ldapPwd;
80 97
    this.dateNextHarvest = dateNextHarvest;
81 98
    this.dateLastHarvest = dateLastHarvest;
82 99
    this.updateFrequency = updateFrequency;
......
102 119
   * harvest based on today's date and the update frequency.
103 120
   */
104 121
  private void dbUpdateHarvestSiteSchedule() {
105
		Connection con;
122
		Connection conn;
106 123
    long currentTime;                    // Current time in milliseconds
107 124
    Date dateNextHarvest;                // Date of next harvest
108 125
    String lastHarvest;
......
112 129
		Statement stmt;
113 130
    long timeNextHarvest;
114 131
    
115
    con = harvester.conn;
132
    conn = harvester.conn;
116 133
    now = new Date();
117
    currentTime = now.getTime();         // Current time in milliseconds
134
    currentTime = now.getTime();
118 135
    timeNextHarvest = currentTime + delta;
119 136
    dateNextHarvest = new Date(timeNextHarvest);
120 137
    nextHarvest = "'" + simpleDateFormat.format(dateNextHarvest) + "'";
121 138
    lastHarvest = "'" + simpleDateFormat.format(now) + "'";
122 139
	
123
    System.out.println("Date of next harvest: " + nextHarvest);
124
    System.out.println("Date of last harvest: " + lastHarvest);
125

  
126 140
		try {
127
			stmt = con.createStatement();							
128
			stmt.executeUpdate("UPDATE HARVEST_SITE_SCHEDULE SET DATENEXTHARVEST = " + nextHarvest + " WHERE SITE_SCHEDULE_ID = " + siteScheduleID);
129
			stmt.executeUpdate("UPDATE HARVEST_SITE_SCHEDULE SET DATELASTHARVEST = " + lastHarvest + " WHERE SITE_SCHEDULE_ID = " + siteScheduleID);
141
			stmt = conn.createStatement();							
142
			stmt.executeUpdate("UPDATE HARVEST_SITE_SCHEDULE SET DATENEXTHARVEST = " +
143
                         nextHarvest +
144
                         " WHERE SITE_SCHEDULE_ID = " +
145
                         siteScheduleID);
146
			stmt.executeUpdate("UPDATE HARVEST_SITE_SCHEDULE SET DATELASTHARVEST = " +
147
                         lastHarvest +
148
                         " WHERE SITE_SCHEDULE_ID = " +
149
                         siteScheduleID);
130 150
			stmt.close();
131 151
		}
132 152
    catch(SQLException e) {
133
			System.err.println("SQLException: " + e.getMessage());
153
			System.out.println("SQLException: " + e.getMessage());
134 154
		}
135 155
  }
136 156
    
......
157 177
      
158 178
      if (timeNextHarvest < currentTime) {
159 179
        dueForHarvest = true;
180
        System.out.println("Due for harvest: " + documentListURL);
160 181
      }
161 182
      else {
162
        System.out.println("Next harvest date: " + dnh.toString());
183
        System.out.println("Not due for harvest: " + documentListURL);
163 184
      }
164 185
    }
165 186
    catch (ParseException e) {
166
      System.err.println("Error parsing date: " + e.getMessage());
187
      System.out.println("Error parsing date: " + e.getMessage());
167 188
    }
168 189
    
169
    //return dueForHarvest;
170
    return true;
190
    return dueForHarvest;
171 191
  }
172 192

  
173 193

  
......
186 206
        parseDocumentList();
187 207
        metacatLogin();
188 208
        
189
        for (int i = 0; i < harvestDocumentList.length; i++) {
190
          harvestDocument = harvestDocumentList[i];
209
        for (int i = 0; i < harvestDocumentList.size(); i++) {
210
          harvestDocument = (HarvestDocument) harvestDocumentList.get(i);
191 211
          
192 212
          if (harvestDocument != null) {
193
            harvestDocument.printOutput();
194 213
            harvestDocument.harvestDocument();
195 214
          }
196 215
        }
......
199 218
        dbUpdateHarvestSiteSchedule();
200 219
      }
201 220
      catch (ParserConfigurationException e) {
202
        System.err.println("ParserConfigurationException: " + e.getMessage());
221
        System.out.println("ParserConfigurationException: " + e.getMessage());
203 222
      }
204
      catch (SAXException e) {
205
        System.err.println("SAXException: " + e.getMessage());
206
      }
207
      catch (IOException e) {
208
        System.err.println("IOException: " + e.getMessage());
209
      }
210 223
      
211 224
      reportToSite();
212 225
    }
......
214 227

  
215 228

  
216 229
  /**
217
   * Login to Metacat using the ldapDN and ldapPassword
230
   * Login to Metacat using the ldapDN and ldapPwd
218 231
   */
219 232
  private void metacatLogin() {
220 233
    Metacat metacat = harvester.metacat;
221 234

  
222 235
    if (harvester.connectToMetacat()) {
223

  
224 236
      try {
225 237
        System.out.println("Logging in to Metacat: " + ldapDN);
226
        metacat.login(ldapDN, ldapPassword);
238
        metacat.login(ldapDN, ldapPwd);
227 239
        //System.out.println("Metacat login response: " + response);
228 240
        //sessionId = metacat.getSessionId();
229 241
        //System.out.println("Session ID: " + sessionId);
......
234 246
      catch (Exception e) {
235 247
        System.out.println("Metacat login failed." + e.getMessage());
236 248
      }
237
    }
238
    else {
239
      System.out.println("Not logging in to Metacat");
240
    }
241
    
249
    }    
242 250
  }
243 251
  
244 252
  
......
261 269
        System.out.println("Metacat exception: " + e.getMessage());
262 270
      }
263 271
    }
264
    else {
265
      System.out.println("Not logging out from Metacat");
266
    }
267 272
  }
268 273
  
269 274

  
270 275
  /**
271 276
   * Parse the site document list to find out which documents to harvest.
272
   * 
273
   * @throws SAXException
274
   * @throws IOException
275
   * @throws ParserConfigurationException
276 277
   */
277 278
  private void parseDocumentList() 
278
    throws SAXException, IOException, ParserConfigurationException {
279
    
280
    // Create a parser factory and use it to create a parser
281
    SAXParserFactory parserFactory = SAXParserFactory.newInstance();
282
    SAXParser parser = parserFactory.newSAXParser();
283
	
284
    // Instantiate a DefaultHandler subclass to do your counting for you
285
    DocumentListHandler handler = new DocumentListHandler();
286
	
287
    // Start the parser. It reads the document list and calls methods of the handler.
288
    parser.parse(documentListURL, handler);
279
          throws ParserConfigurationException {
280
    DocumentListHandler documentListHandler = new DocumentListHandler();
281
    InputStream inputStream;
282
    InputStreamReader inputStreamReader;
283
    String schemaLocation = ".";
284
    URL url;
285

  
286
    try {
287
      url = new URL(documentListURL);
288
      inputStream = url.openStream();
289
      harvester.addLogEntry(0, "", "GetDocListSuccess", 
290
                            siteScheduleID, null, "");
291
      inputStreamReader = new InputStreamReader(inputStream);
292
      documentListHandler.runParser(inputStreamReader, schemaLocation);
293
      harvester.addLogEntry(0, "", "ValidateDocListSuccess", 
294
                            siteScheduleID, null, "");
295
    }
296
    catch (MalformedURLException e){
297
      harvester.addLogEntry(1, "MalformedURLException: " + e.getMessage(), 
298
                            "GetDocListError", siteScheduleID, null, "");
299
    }
300
    catch (FileNotFoundException e) {
301
      harvester.addLogEntry(1, "FileNotFoundException: " + e.getMessage(), 
302
                            "GetDocListError", siteScheduleID, null, "");
303
    }
304
    catch (SAXException e) {
305
      harvester.addLogEntry(1, "SAXException: " + e.getMessage(), 
306
                            "ValidateDocListError", siteScheduleID, null, "");
307
    }
308
    catch (ClassNotFoundException e) {
309
      harvester.addLogEntry(1, "ClassNotFoundException: " + e.getMessage(),
310
                            "ValidateDocListError", siteScheduleID, null, "");
311
    }
312
    catch (IOException e) {
313
      harvester.addLogEntry(1, "IOException: " + e.getMessage(), 
314
                            "GetDocListError", siteScheduleID, null, "");
315
    }
289 316
  }
290 317

  
291 318

  
......
293 320
   * Prints the data that is stored in this HarvestSiteSchedule object.
294 321
   */
295 322
  void printOutput() {
296
    System.out.println("");
297
    System.out.println("siteScheduleID: " + siteScheduleID);
298
    System.out.println("documentListURL: " + documentListURL);
299
    System.out.println("ldapDN: " + ldapDN);
300
    System.out.println("ldapPassword: " + ldapPassword);
301
    System.out.println("dateNextHarvest: " + dateNextHarvest);
302
    System.out.println("dateLastHarvest: " + dateLastHarvest);
303
    System.out.println("updateFrequency: " + updateFrequency);
304
    System.out.println("unit: " + unit);
305
    System.out.println("contactEmail: " + contactEmail);
323
    System.out.println("siteScheduleID:       " + siteScheduleID);
324
    System.out.println("documentListURL:      " + documentListURL);
325
    System.out.println("ldapDN:               " + ldapDN);
326
    System.out.println("dateNextHarvest:      " + dateNextHarvest);
327
    System.out.println("dateLastHarvest:      " + dateLastHarvest);
328
    System.out.println("updateFrequency:      " + updateFrequency);
329
    System.out.println("unit:                 " + unit);
330
    System.out.println("contactEmail:         " + contactEmail);
306 331
  }
307 332
  
308 333

  
309 334
  /**
310
   * Pushes a HarvestDocument object onto the harvestDocumentList.
311
   * 
312
   * @param harvestDocument    a new HarvestDocument object to add to the list
313
   */
314
  void pushHarvestDocument(HarvestDocument harvestDocument) {
315
    harvestDocumentList[harvestDocumentIndex] = harvestDocument;
316
    harvestDocumentIndex++;
317
  }
318
  
319

  
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff