/ - Diff - Metacat - Ecoinformatics Redmine

« Previous | Next »

Revision 2031

Added by Duane Costa about 20 years ago

Additional development of Harvester implementation

     package edu.ucsb.nceas.metacat.harvesterClient;
     import java.sql.Connection;
     import java.sql.SQLException;
     import java.sql.Statement;
     /**
      * HarvestDetailLog manages data and operations corresponding to the
      * HARVEST_DETAIL_LOG table. It records errors encountered while attempting
      * to harvest a particular document.
      * to harvest a particular EML document.
+     *
      * @author  costa
      */
     public class HarvestDetailLog {
       private Harvester harvester;              // The parent Harvester object
       private int detailLogID;
       private int harvestLogID;
       private HarvestDocument harvestDocument;  // The associated HarvestDocument
       private String errorMessage;
       private Object harvestDocument;
       private int harvestLogID;
       /**
        * Creates a new instance of HarvestDetailLog.
        * Creates a new instance of HarvestDetailLog and inserts the data into
        * the HARVEST_DETAIL_LOG table.
+       *
        * @param  harvester       the Harvester parent object
        * @param  harvestLogID    foreign key value matching the HARVEST_LOG table
        * @param  harvestDocument HarvestDocument object that generated an error
        * @param  errorMessage    text of the error message
        */
       public HarvestDetailLog() {
       public HarvestDetailLog(Harvester       harvester,
                               int             harvestLogID,
                               HarvestDocument harvestDocument,
                               String          errorMessage
                              ) {
         this.harvester = harvester;
         this.detailLogID = harvester.getDetailLogID();  // Primary key for record
         this.harvestLogID = harvestLogID;
         this.harvestDocument = harvestDocument;
         this.errorMessage = errorMessage;
+      }
       /**
        * Inserts a new entry into the HARVEST_DETAIL_LOG table, based on the
        * contents of this HarvestDetailLog object. Not yet implemented.
        * contents of this HarvestDetailLog object.
        */
       public void dnInsertHarvestDetailLogEntry() {
       void dbInsertHarvestDetailLogEntry() {
         String insertString;
     		Statement stmt;
         // Set the value of the HARVEST_LOG_ID to the current time in UTC seconds
         insertString = "INSERT INTO HARVEST_DETAIL_LOG " +
                        "(DETAIL_LOG_ID, HARVEST_LOG_ID, SCOPE," +
                        " IDENTIFIER, REVISION," +
                        " DOCUMENT_URL, ERROR_MESSAGE, DOCUMENT_TYPE) " +
                        "values(" +
                        detailLogID + ", " +
                        harvestLogID + ", " +
                        "'" + harvestDocument.scope + "', " +
                        harvestDocument.identifier + ", " +
                        harvestDocument.revision + ", " +
                        "'" + harvestDocument.documentURL + "', " +
                        "'" + errorMessage + "'," +
                        "'" + harvestDocument.documentType + "'" +
                        ")";
     		try {
     			stmt = harvester.conn.createStatement();
     			stmt.executeUpdate(insertString);
     			stmt.close();
+    		}
         catch(SQLException e) {
     			System.out.println("SQLException: " + e.getMessage());
+    		}
+      }
       /**
        * Prints the contents of this HarvestLog object. Used in generating reports.
        */
       void printOutput() {
         System.out.println("detailLogID:          " + detailLogID);
         System.out.println("errorMessage:         " + errorMessage);
         harvestDocument.printOutput();
+      }
+    }

     package edu.ucsb.nceas.metacat.harvesterClient;
     import java.io.*;
     import java.sql.*;
     import java.util.*;
     import javax.xml.parsers.*;
     import org.xml.sax.*;
     import org.xml.sax.helpers.*;
     import java.io.File;
     import java.io.FileInputStream;
     import java.io.IOException;
     import java.sql.Connection;
     import java.sql.DriverManager;
     import java.sql.ResultSet;
     import java.sql.SQLException;
     import java.sql.SQLWarning;
     import java.sql.Statement;
     import java.util.ArrayList;
     import java.text.SimpleDateFormat;
     import java.util.Date;
     import java.util.Properties;
     import edu.ucsb.nceas.metacat.client.*;
     import edu.ucsb.nceas.metacat.client.Metacat;
     import edu.ucsb.nceas.metacat.client.MetacatFactory;
     import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;
     /**
      * Harvester is the main class for the Harvester application. The main
-...
        * Class methods
        */
       /**
        * Constructor. Creates a new instance of Harvester.
        */
       public Harvester() {
+      }
       /**
        * Determines whether Harvester is running on a Win32 platform. Used
        * during development to aid in resolving platform dependencies.
        * during development.
+       *
        * @return    true if this in Win32, false otherwise
        */
-...
        * Object fields
        */
       /** Database connection */
       Connection conn;
       private Object harvestEndTime;
       private Object harvestLogList;
       private HarvestSiteSchedule[] harvestSiteScheduleList =
               new HarvestSiteSchedule[20];
       private int harvestSiteScheduleIndex = 0;
       private Object harvestStartTime;
       /** Used during development to determine whether to connect to metacat */
       private boolean connectToMetacat;
       /** Highest DETAIL_LOG_ID primary key in the HARVEST_DETAIL_LOG table */
       private int detailLogID;
       /** Highest HARVEST_LOG_ID primary key in the HARVEST_LOG table */
       private int harvestLogID;
       /** End time of this harvest session */
       private Date harvestEndTime;
       /** List of HarvestLog objects. Stores log entries for report generation. */
       private ArrayList harvestLogList = new ArrayList();
       /** List of HarvestSiteSchedule objects */
       private ArrayList harvestSiteScheduleList = new ArrayList();
       /** Start time of this harvest session */
       private Date harvestStartTime;
       /** Number of days to save log records. Any that are older are purged. */
       int logPeriod;
       /** Metacat client object */
       Metacat metacat;
       /** Loads harvester properties from configuration file */
       Properties properties;
       /*
        * Object methods
        */
       /*
                         Harvester  harvester,
                         Date       harvestDate,
                         int        status,
                         String     message,
                         String     harvestOperationCode,
                         int        siteScheduleID,
                         HarvestDocument harvestDocument,
                         String     errorMessage
     */
       /**
        * Creates a new HarvestLog object and adds it to the harvestLogList.
+       *
        * @param  status          the status of the harvest operation
        * @param  message         the message text of the harvest operation
        * @param  harvestOperationCode  the harvest operation code
        * @param  siteScheduleID  the siteScheduleID for which this operation was
        *                         performed. 0 indicates that the operation did not
        *                         involve a particular harvest site.
        * @param  harvestDocument the associated HarvestDocument object. May be null.
        * @param  errorMessage    additional error message pertaining to document
        *                         error.
        */
       void addLogEntry(int    status,
                        String message,
                        String harvestOperationCode,
                        int    siteScheduleID,
                        HarvestDocument harvestDocument,
                        String errorMessage
                       ) {
         HarvestLog harvestLog;
         /* If there is no associated harvest document, call the basic constructor;
          * else call the extended constructor.
          */
         if (harvestDocument == null) {
           harvestLog = new HarvestLog(this, harvestStartTime, status, message,
                                       harvestOperationCode, siteScheduleID);
+        }
         else {
           harvestLog = new HarvestLog(this, harvestStartTime, status, message,
                                       harvestOperationCode, siteScheduleID,
                                       harvestDocument, errorMessage);
+        }
         harvestLogList.add(harvestLog);
+      }
       /**
        * Determines whether Harvester should attempt to connect to Metacat.
        * Used during development and testing.
-...
        * @return     true if Harvester should connect, otherwise false
        */
       boolean connectToMetacat () {
         boolean connectToMetacat = Harvester.isWin32() ? false : true;
         return connectToMetacat;
+      }
       /**
        * Gets the current value of the detailLogID for storage as a primary key in
        * the DETAIL_LOG_ID field of the HARVEST_DETAIL_LOG table.
+       *
        * @return  the current value of the detailLogID
        */
       int getDetailLogID() {
         int currentValue = detailLogID;
         detailLogID++;
         return currentValue;
+      }
       /**
        * Gets the current value of the harvestLogID for storage as a primary key in
        * the HARVEST_LOG_ID field of the HARVEST_LOG table.
+       *
        * @return  the current value of the detailLogID
        */
       int getHarvestLogID() {
         int currentValue = harvestLogID;
         harvestLogID++;
         return currentValue;
+      }
       /**
        * Gets the maximum value of an integer field from a table.
+       *
        * @param tableName  the database table name
        * @param fieldName  the field name of the integer field in the table
        * @return  the maximum integer stored in the fieldName field of tableName
        */
       private int getMaxValue(String tableName, String fieldName) {
         int maxValue = 100;
         int fieldValue;
     		String query = "SELECT " + fieldName + " FROM " + tableName;
     		Statement stmt;
     		try {
     			stmt = conn.createStatement();
     			ResultSet rs = stmt.executeQuery(query);
     			while (rs.next()) {
     				fieldValue = rs.getInt(fieldName);
             maxValue = Math.max(maxValue, fieldValue);
+    			}
     			stmt.close();
+    		}
         catch(SQLException ex) {
     			System.out.println("SQLException: " + ex.getMessage());
+    		}
         return maxValue;
+      }
       /**
        * Gets the minimum value of an integer field from a table.
+       *
        * @param tableName  the database table name
        * @param fieldName  the field name of the integer field in the table
        * @return  the minimum integer stored in the fieldName field of tableName
        */
       private int getMinValue(String tableName, String fieldName) {
         int minValue = 0;
         int fieldValue;
     		String query = "SELECT " + fieldName + " FROM " + tableName;
     		Statement stmt;
     		try {
     			stmt = conn.createStatement();
     			ResultSet rs = stmt.executeQuery(query);
     			while (rs.next()) {
     				fieldValue = rs.getInt(fieldName);
             if (minValue == 0) {
               minValue = fieldValue;
+            }
             else {
               minValue = Math.min(minValue, fieldValue);
+            }
+    			}
     			stmt.close();
+    		}
         catch(SQLException ex) {
     			System.out.println("SQLException: " + ex.getMessage());
+    		}
         return minValue;
+      }
       /**
        * For every Harvest site schedule in the database, harvest the
        * documents for that site if they are due to be harvested.
+       *
-...
       private void harvest() {
         HarvestSiteSchedule harvestSiteSchedule;
         for (int i = 0; i < harvestSiteScheduleList.length; i++) {
           harvestSiteSchedule = harvestSiteScheduleList[i];
           if (harvestSiteSchedule != null) {
             harvestSiteSchedule.printOutput();
             harvestSiteSchedule.harvestDocumentList();
+          }
         for (int i = 0; i < harvestSiteScheduleList.size(); i++) {
           harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);
           harvestSiteSchedule.harvestDocumentList();
+        }
         reportToAdministrator();
+      }
       /**
        * Loads Harvester properties
        * Initializes the detailLogID and harvestLogID values to their current
        * maximums + 1.
        */
       private void initLogIDs() {
         detailLogID = getMaxValue("HARVEST_DETAIL_LOG", "DETAIL_LOG_ID") + 1;
         harvestLogID = getMaxValue("HARVEST_LOG", "HARVEST_LOG_ID") + 1;
+      }
       /**
        * Loads Harvester properties from a configuration file.
        */
       private void loadProperties() {
         String homedir = System.getProperty("user.home");
         File configfile = new File(homedir, "harvester.properties");
-...
           properties.list(System.out);
+        }
         catch (IOException e) {
           System.err.println("IOException: " + e.getMessage());
           System.out.println("IOException: " + e.getMessage());
           System.exit(1);
+        }
+      }
       void printHarvestSiteSchedule(int siteScheduleID) {
          HarvestSiteSchedule harvestSiteSchedule;
         for (int i = 0; i < harvestSiteScheduleList.size(); i++) {
           harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);
           if (harvestSiteSchedule.siteScheduleID == siteScheduleID) {
             harvestSiteSchedule.printOutput();
+          }
+        }
+      }
       /**
        * Prunes old records from the HARVEST_DETAIL_LOG table. Records are
        * removed if the HARVEST_LOG_ID foreign key is less than the lowest
        * HARVEST_LOG_ID primary key in the HARVEST_LOG table.
        */
       private void pruneHarvestDetailLog() {
     		String deleteString;
         int minHarvestLogID;
         int recordsDeleted;
     		Statement stmt;
         minHarvestLogID = getMinValue("HARVEST_LOG", "HARVEST_LOG_ID");
         deleteString = "DELETE FROM HARVEST_DETAIL_LOG WHERE HARVEST_LOG_ID < " +
                        minHarvestLogID;
     		try {
     			System.out.print("Pruning log entries from HARVEST_DETAIL_LOG: ");
     			stmt = conn.createStatement();
     			recordsDeleted = stmt.executeUpdate(deleteString);
     			System.out.println(recordsDeleted + " records deleted");
     			stmt.close();
+    		}
         catch(SQLException e) {
     			System.out.println("SQLException: " + e.getMessage());
+    		}
+      }
       /**
        * Prunes old records from the HARVEST_LOG table. Records are removed if
        * their HARVEST_DATE is older than a given number of days, as stored in the
        * logPeriod object field.
        */
       private void pruneHarvestLog() {
         long currentTime = harvestStartTime.getTime(); // time in milliseconds
         Date dateLastLog;                    // Prune everything prior to this date
     		String deleteString;
         long delta;
         final long millisecondsPerDay = (1000 * 60 * 60 * 24);
         int recordsDeleted;
         SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MMM-yyyy");
         String dateString;
     		Statement stmt;
         long timeLastLog = 0;
         delta = logPeriod * millisecondsPerDay;
         deleteString = "DELETE FROM HARVEST_LOG WHERE HARVEST_DATE < ";
         timeLastLog = currentTime - delta;
         dateLastLog = new Date(timeLastLog);
         dateString = "'" + simpleDateFormat.format(dateLastLog) + "'";
         deleteString += dateString;
     		try {
     			System.out.print("Pruning log entries from HARVEST_LOG: ");
     			stmt = conn.createStatement();
     			recordsDeleted = stmt.executeUpdate(deleteString);
     			System.out.println(recordsDeleted + " records deleted");
     			stmt.close();
+    		}
         catch (SQLException e) {
     			System.out.println("SQLException: " + e.getMessage());
+    		}
+      }
       /**
        * Reads the HARVEST_SITE_SCHEDULE table in the database, creating
        * a HarvestSiteSchedule object for each row in the table.
        */
-...
         String dateNextHarvest;
         String documentListURL;
         String ldapDN;
         String ldapPassword;
         String ldapPwd;
         int siteScheduleID;
         String unit;
         int updateFrequency;
-...
             siteScheduleID = rs.getInt("SITE_SCHEDULE_ID");
             documentListURL = rs.getString("DOCUMENTLISTURL");
             ldapDN = rs.getString("LDAPDN");
     //        ldapPassword = rs.getString("LDAPPASSWORD");
             ldapPassword = "ntre4dc";
             ldapPwd = rs.getString("LDAPPWD");
             dateNextHarvest = rs.getString("DATENEXTHARVEST");
             dateLastHarvest = rs.getString("DATELASTHARVEST");
             updateFrequency = rs.getInt("UPDATEFREQUENCY");
-...
+              }
+            }
             harvestSiteSchedule = new HarvestSiteSchedule(
                                                           this,
             harvestSiteSchedule = new HarvestSiteSchedule(this,
                                                           siteScheduleID,
                                                           documentListURL,
                                                           ldapDN,
                                                           ldapPassword,
                                                           ldapPwd,
                                                           dateNextHarvest,
                                                           dateLastHarvest,
                                                           updateFrequency,
                                                           unit,
                                                           contactEmail
                                                          );
             harvestSiteScheduleList[harvestSiteScheduleIndex] = harvestSiteSchedule;
             harvestSiteScheduleIndex++;
             harvestSiteScheduleList.add(harvestSiteSchedule);
+          }
+        }
           rs.close();
           stmt.close();
+        }
         catch (SQLException e) {
           System.out.println("Database access failed " + e);
           System.exit(1);
-...
       private void shutdown() {
         // Log shutdown operation
         System.out.println("Shutting Down Harvester");
         addLogEntry(0, "Shutting Down Harvester", "HarvesterShutdown", 0, null, "");
         pruneHarvestLog();
         pruneHarvestDetailLog();
         try {
           // Close the database connection
-...
         catch (SQLException e) {
           System.out.println("Database access failed " + e);
+        }
         writeHarvestLog();
         reportToAdministrator();
+      }
-...
        * Initializes Harvester at startup. Connects to the database and to Metacat.
        */
       private void startup() {
         Boolean ctm;
         String dbDriver;
         Integer lp;
         String metacatURL;
         String osName = Harvester.isWin32() ? "Windows" : "Unix";
         String password;
     //    String response;
         //String response;
         String sessionId;
     		String url;
         String user;
-...
         // Log startup operation
         System.out.println("*****************************************************");
         System.out.println("Starting Up Harvester");
         System.out.println("OS is " + osName);
         loadProperties();
         ctm = Boolean.valueOf(properties.getProperty("connectToMetacat", "true"));
         connectToMetacat = ctm.booleanValue();
         dbDriver = properties.getProperty("dbDriver");
         try {
           lp = Integer.valueOf(properties.getProperty("logPeriod", "90"));
           logPeriod = lp.intValue();
+        }
         catch (NumberFormatException e) {
           System.err.println("NumberFormatException: Error parsing logPeriod " +
                              logPeriod + e.getMessage());
           System.err.println("Defaulting to logPeriod of 90 days");
           logPeriod = 90;
+        }
         metacatURL = properties.getProperty("metacatURL");
         password = properties.getProperty("password");
         url = properties.getProperty("url");
         user = properties.getProperty("user");
         password = properties.getProperty("password");
         metacatURL = properties.getProperty("metacatURL");
         // Load the jdbc driver
         try {
-...
           System.out.println("Database access failed " + e);
           System.exit(1);
+        }
         initLogIDs();
         harvestStartTime = new Date();
         addLogEntry(0, "Starting Up Harvester", "HarvesterStartup", 0, null, "");
         if (connectToMetacat()) {
           try {
-...
             System.out.println("Metacat connection failed." + e.getMessage());
+          }
+        }
         else {
           System.out.println("Not connecting to Metacat");
+        }
+      }
-...
        * Writes one or more log entries to the HARVEST_LOG table.
        */
       private void writeHarvestLog() {
         HarvestLog harvestLog;
         for (int i = 0; i < harvestLogList.size(); i++) {
           harvestLog = (HarvestLog) harvestLogList.get(i);
           harvestLog.printOutput();
+        }
+      }
+    }

     import java.net.MalformedURLException;
     import java.net.URL;
     import edu.ucsb.nceas.metacat.client.*;
     import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;
     import edu.ucsb.nceas.metacat.client.Metacat;
     import edu.ucsb.nceas.metacat.client.MetacatException;
     import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;
     import edu.ucsb.nceas.utilities.IOUtil;
-...
     public class HarvestDocument {
       private String documentName;
       private String documentType;
       private String documentURL;
       String documentType;
       String documentURL;
       private Harvester harvester;
       private HarvestSiteSchedule harvestSiteSchedule;
       private int identifier;
       private int revision;
       private String scope;
       int identifier;
       int revision;
       String scope;
       /**
-...
           inputStreamReader = new InputStreamReader(inputStream);
           documentString = IOUtil.getAsString(inputStreamReader, true);
           stringReader = new StringReader(documentString);
           System.out.println("  Successfully read document: " + documentURL);
           harvester.addLogEntry(0, "", "GetDocSuccess",
                                 harvestSiteSchedule.siteScheduleID, null, "");
+        }
         catch (MalformedURLException e) {
           System.err.println("MalformedURLException: " + e.getMessage());
           harvester.addLogEntry(1, "MalformedURLException", "GetDocError",
                                 harvestSiteSchedule.siteScheduleID, this,
                                 "MalformedURLException: " + e.getMessage());
+        }
         catch (IOException e) {
           System.err.println("IOException: " + e.getMessage());
           harvester.addLogEntry(1, "IOException", "GetDocError",
                                 harvestSiteSchedule.siteScheduleID, this,
                                 "IOException: " + e.getMessage());
+        }
         return stringReader;
-...
          * into metacat.
          */
         if (metacatHasDocument()) {
           System.out.println("  metacat has document");
           highestRevision = metacatHighestRevision();
           System.out.println("  metacatHighestRevision: " + highestRevision);
           harvester.addLogEntry(0,
                                 "Metacat has document: " + documentName +
                                 ", highest revision: " + highestRevision,
                                 "MetacatHasDoc",
                                 harvestSiteSchedule.siteScheduleID, null, "");
+        }
         else {
           stringReader = getSiteDocument();
           if (stringReader != null) {
             if (parseDocument()) {
               metacatReturnString = putMetacatDocument(stringReader);
               System.out.println("  " + metacatReturnString);
               putMetacatDocument(stringReader);
+            }
             else {
               System.out.println("Error parsing document.");
+            }
+          }
           else {
             System.out.print("  Error reading document at URL: ");
             System.out.println(documentURL);
+          }
+        }
+      }
       /**
        * Logs a metacat document error to the harvest detail log.
+       *
        * @param insert               true if insert operation, false is update
        * @param metacatReturnString  string returned from the insert or update
        * @param exceptionName        name of the exception class
        * @param e                    the exception object
        */
       private void logMetacatError (boolean insert,
                                     String metacatReturnString,
                                     String exceptionName,
                                     Exception e
                                    ) {
         if (insert) {
           harvester.addLogEntry(1, metacatReturnString, "InsertDocError",
                                     harvestSiteSchedule.siteScheduleID,
                                     this, exceptionName + ": " + e.getMessage());
+        }
         else {
           harvester.addLogEntry(1, metacatReturnString, "UpdateDocError",
                                     harvestSiteSchedule.siteScheduleID,
                                     this, exceptionName + ": " + e.getMessage());
+        }
+      }
       /**
        * Boolean to determine whether Metacat already has this document.
+       *
        * @return  true if Metacat has the document, otherwise false
-...
       private boolean parseDocument () {
         boolean success = true;
         if (success) {
           harvester.addLogEntry(0, "", "ValidateDocSuccess",
                                 harvestSiteSchedule.siteScheduleID, null, "");
+        }
         else {
           harvester.addLogEntry(1, "Error validating document", "ValidateDocError",
                                 harvestSiteSchedule.siteScheduleID, this, "");
+        }
         return success;
+      }
-...
       /**
        * Print the data fields and values in this HarvestDocument object.
        */
       void printOutput()
+      {
         System.out.println("");
         System.out.println("  scope: " + scope);
         System.out.println("  identifier: " + identifier);
         System.out.println("  revision: " + revision);
         System.out.println("  documentType: " + documentType);
         System.out.println("  documentURL: " + documentURL);
         System.out.println("  documentName: " + documentName);
       void printOutput() {
         System.out.println("scope:                " + scope);
         System.out.println("identifier:           " + identifier);
         System.out.println("revision:             " + revision);
         System.out.println("documentType:         " + documentType);
         System.out.println("documentURL:          " + documentURL);
         System.out.println("documentName:         " + documentName);
+      }
       /**
        * Insert or update this document to Metacat. If revision equals 1, do an
        * insert; otherwise, do an update.
+       *
        * @return  the Metacat return string from the insert or update operation
        */
       private String putMetacatDocument(StringReader stringReader) {
       private void putMetacatDocument(StringReader stringReader) {
         String docid = scope + "." + identifier + "." + revision;
         boolean insert = (revision == 1);
         Metacat metacat = harvester.metacat;
         String metacatReturnString = "";
         if (harvester.connectToMetacat()) {
           try {
             if (revision == 1) {
               System.out.println("  Inserting document to metacat: " + docid);
             if (insert) {
               metacatReturnString = metacat.insert(docid, stringReader, null);
               harvester.addLogEntry(0, docid + " : " + metacatReturnString,
                                     "InsertDocSuccess",
                                     harvestSiteSchedule.siteScheduleID,
                                     null, "");
+            }
             else {
               System.out.println("  Updating document to metacat: " + docid);
               metacatReturnString = metacat.update(docid, stringReader, null);
               harvester.addLogEntry(0, docid + " : " + metacatReturnString,
                                     "UpdateDocSuccess",
                                     harvestSiteSchedule.siteScheduleID,
                                     null, "");
+            }
+          }
           catch (MetacatInaccessibleException e) {
             System.err.println("MetacatInaccessibleException: " + e.getMessage());
             logMetacatError(insert, metacatReturnString,
                             "MetacatInaccessibleException", e);
+          }
           catch (InsufficientKarmaException e) {
             System.err.println("InsufficientKarmaException: " + e.getMessage());
             logMetacatError(insert, metacatReturnString,
                             "InsufficientKarmaException", e);
+          }
           catch (MetacatException e) {
             System.err.println("MetacatException: " + e.getMessage());
             logMetacatError(insert, metacatReturnString, "MetacatException", e);
+          }
           catch (IOException e) {
             System.err.println("IOException: " + e.getMessage());
             logMetacatError(insert, metacatReturnString, "IOException", e);
+          }
+        }
         else {
           metacatReturnString = "Not putting document to metacat";
+        }
         return metacatReturnString;
+      }
+    }

     package edu.ucsb.nceas.metacat.harvesterClient;
     import java.sql.Connection;
     import java.sql.ResultSet;
     import java.sql.SQLException;
     import java.sql.SQLWarning;
     import java.sql.Statement;
     import java.text.SimpleDateFormat;
     import java.util.Date;
     /**
      * Manages log entries to be inserted to the HARVEST_LOG table.
+     *
-...
      */
     public class HarvestLog {
       private Object harvestDate;
       private Harvester harvester;              // The parent Harvester object
       private int harvestLogID;
       private Date harvestDate;
       private int status;
       private String message;
       private String harvestOperationCode;
       private String message;
       private int siteScheduleID;
       private String status;
       private String timestamp;
       private HarvestDetailLog harvestDetailLog;  // Associated detail log, if any
       /**
         * Creates a new instance of HarvestLog.
         */
       public HarvestLog() {
        * Creates a new instance of HarvestLog. This constructor is used when
        * creating log entries that do not involve an error on a harvest document.
        * (For that type of log entry, use the alternate constructor below.)
+       *
        * @param  harvester       the parent Harvester object
        * @param  harvestDate     the date of this harvest
        * @param  status          the status of the harvest operation
        * @param  message         the message text of the harvest operation
        * @param  harvestOperationCode  the harvest operation code
        * @param  siteScheduleID  the siteScheduleID for which this operation was
        *                         performed. 0 indicates that the operation did not
        *                         involve a particular harvest site.
        */
       public HarvestLog(Harvester  harvester,
                         Date       harvestDate,
                         int        status,
                         String     message,
                         String     harvestOperationCode,
                         int        siteScheduleID
                        ) {
         Date now = new Date();
         timestamp = now.toString();
         this.harvester = harvester;
         this.harvestLogID = harvester.getHarvestLogID();
         this.harvestDate = harvestDate;
         this.status = status;
         this.message = message;
         this.harvestOperationCode = harvestOperationCode;
         this.siteScheduleID = siteScheduleID;
         dbInsertHarvestLogEntry();   // Insert this entry to the HARVEST_LOG table
+      }
       /**
        * Creates a new instance of HarvestLog and inserts this entry to the
        * HARVEST_LOG table. This version of the constructor also instantiates a
        * HarvestDetailLog object and inserts it to the HARVEST_DETAIL_LOG table.
+       *
        * @param  harvester       the parent Harvester object
        * @param  harvestDate     the date of this harvest
        * @param  status          the status of the harvest operation
        * @param  message         the message text of the harvest operation
        * @param  harvestOperationCode  the harvest operation code
        * @param  siteScheduleID  the siteScheduleID for which this operation was
        *                         performed. 0 indicates that the operation did not
        *                         involve a particular harvest site.
        */
       public HarvestLog(Harvester  harvester,
                         Date       harvestDate,
                         int        status,
                         String     message,
                         String     harvestOperationCode,
                         int        siteScheduleID,
                         HarvestDocument harvestDocument,
                         String     errorMessage
                        ) {
         Date now = new Date();
         timestamp = now.toString();
         this.harvester = harvester;
         this.harvestLogID = harvester.getHarvestLogID();
         this.harvestDate = harvestDate;
         this.status = status;
         this.message = message;
         this.harvestOperationCode = harvestOperationCode;
         this.siteScheduleID = siteScheduleID;
         this.harvestDetailLog = new HarvestDetailLog(harvester, harvestLogID,
                                                      harvestDocument, errorMessage);
         dbInsertHarvestLogEntry();               // Insert to the HARVEST_LOG table
         harvestDetailLog.dbInsertHarvestDetailLogEntry(); // and HARVEST_DETAIL_LOG
+      }
       /**
        * Retrieves the value of the EXPLANATION field of the HARVEST_OPERATION
        * table based on the value of the HARVEST_OPERATION_CODE field.
        * Not yet implemented.
+       *
        * @param harvestOperationCode  string value of the harvest operation code
        * @return           the explanation for this harvest operation, a String
        * @param  harvestOperationCode  string value of the harvest operation code
        * @return the explanation for this harvest operation, a String
        */
       public String dbGetExplanation(String harvestOperationCode) {
         String explanation = "";
       String dbGetExplanation(String harvestOperationCode) {
         String explanation;
         String fieldName = "EXPLANATION";
         explanation = dbQueryHarvestOperation(fieldName, harvestOperationCode);
         return explanation;
+      }
-...
       /**
        * Retrieves the value of the HARVEST_OPERATION_CODE_LEVEL field of the
        * HARVEST_OPERATION table based on the value of the HARVEST_OPERATION_CODE
        * field. Not yet implemented.
        * field.
+       *
        * @param harvestOperationCode  string value of the harvest operation code
        * @return          the code level value, an int
        * @param  harvestOperationCode  string value of the harvest operation code
        * @return the code level value, a String, one of the following:
        *         "error", "warning", "notice", "info", or "debug"
        */
       public int dbGetHarvestOperationCodeLevel(String harvestOperationCode) {
         int codeLevel = 0;
       String dbGetHarvestOperationCodeLevel(String harvestOperationCode) {
         String harvestOperationCodeLevel;
         String fieldName = "HARVEST_OPERATION_CODE_LEVEL";
         harvestOperationCodeLevel =
                 dbQueryHarvestOperation(fieldName, harvestOperationCode);
         return codeLevel;
         return harvestOperationCodeLevel;
+      }
       /**
        * Inserts a new entry into the HARVEST_LOG table, based on the contents of
        * this HarvestLog object. Not yet implemented.
        */
       public void dbInsertHarvestLogEntry() {
       void dbInsertHarvestLogEntry() {
         String insertString;
         SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MMM-yyyy");
     		Statement stmt;
         insertString = "INSERT INTO HARVEST_LOG " +
                        "(HARVEST_LOG_ID, HARVEST_DATE, STATUS, MESSAGE," +
                        " HARVEST_OPERATION_CODE, SITE_SCHEDULE_ID) " +
                        "values(" +
                        harvestLogID + ", " +
                        "'" + simpleDateFormat.format(harvestDate) + "', " +
                        status + ", " +
                        "'" + timestamp + ": " + message + "', " +
                        "'" + harvestOperationCode + "', " +
                        siteScheduleID +
                        ")";
     		try {
     			stmt = harvester.conn.createStatement();
     			stmt.executeUpdate(insertString);
     			stmt.close();
+    		}
         catch(SQLException e) {
     			System.out.println("SQLException: " + e.getMessage());
+    		}
+      }
       /**
        * Retrieves the value of the either the EXPLANATION field or the
        * HARVEST_OPERATION_CODE_LEVEL field of the HARVEST_OPERATION table based on
        * the value of the HARVEST_OPERATION_CODE field.
+       *
        * @param  fieldName  the field name to match, e.g. "EXPLANATION"
        * @param  harvestOperationCode  string value of the harvest operation code
        * @return the explanation string or the harvestOperationCodeLevel string
        */
       String dbQueryHarvestOperation(String fieldName,
                                      String harvestOperationCode
                                     ) {
         String explanation = "No explanation available";
         String harvestOperationCodeLevel = "debug";
         String queryString;
         String returnString = "";
         ResultSet rs;
         Statement stmt;
         SQLWarning warn;
         queryString = "SELECT EXPLANATION, HARVEST_OPERATION_CODE_LEVEL " +
                       "FROM HARVEST_OPERATION " +
                       "WHERE HARVEST_OPERATION_CODE=" +
                       "'" + harvestOperationCode + "'";
         try {                           // Query the HARVEST_OPERATION table
           stmt = harvester.conn.createStatement();
           rs = stmt.executeQuery(queryString);
           warn = rs.getWarnings();
           if (warn != null) {
             System.out.println("\n---Warning---\n");
             while (warn != null) {
               System.out.println("Message: " + warn.getMessage());
               System.out.println("SQLState: " + warn.getSQLState());
               System.out.print("Vendor error code: ");
               System.out.println(warn.getErrorCode());
               System.out.println("");
               warn = warn.getNextWarning();
+            }
+          }
           while (rs.next()) {
             explanation = rs.getString("EXPLANATION");
             harvestOperationCodeLevel = rs.getString("HARVEST_OPERATION_CODE_LEVEL");
             warn = rs.getWarnings();
             if (warn != null) {
               System.out.println("\n---Warning---\n");
               while (warn != null) {
                 System.out.println("Message: " + warn.getMessage());
                 System.out.println("SQLState: " + warn.getSQLState());
                 System.out.print("Vendor error code: ");
                 System.out.println(warn.getErrorCode());
                 System.out.println("");
                 warn = warn.getNextWarning();
+              }
+            }
+          }
           rs.close();
           stmt.close();
+        }
         catch (SQLException e) {
           System.out.println("SQLException: Database access failed: " + e);
+        }
         if (fieldName.equals("EXPLANATION")) {
           returnString = explanation;
+        }
         else if (fieldName.equals("HARVEST_OPERATION_CODE_LEVEL")) {
           returnString = harvestOperationCodeLevel;
+        }
         return returnString;
+      }
       /**
        * Prints the contents of this HarvestLog object. Used in generating reports.
        */
       void printOutput() {
         System.out.println("");
         System.out.println("harvestLogID:         " + harvestLogID);
         System.out.println("harvestDate:          " + harvestDate);
         System.out.println("status:               " + status);
         System.out.println("message:              " + message);
         System.out.println("harvestOperationCode: " + harvestOperationCode);
         if (siteScheduleID != 0) {
           harvester.printHarvestSiteSchedule(siteScheduleID);
+        }
         if (harvestDetailLog != null) {
           harvestDetailLog.printOutput();
+        }
+      }
+    }

     package edu.ucsb.nceas.metacat.harvesterClient;
     import java.io.*;
     import java.io.FileNotFoundException;
     import java.io.IOException;
     import java.io.InputStream;
     import java.io.InputStreamReader;
     import java.io.Reader;
     import java.net.MalformedURLException;
     import java.net.URL;
     import java.sql.Connection;
     import java.sql.SQLException;
     import java.sql.Statement;
     import java.text.*;
     import java.util.*;
     import javax.xml.parsers.*;
     import org.xml.sax.*;
     import org.xml.sax.helpers.*;
     import java.text.DateFormat;
     import java.text.ParseException;
     import java.text.SimpleDateFormat;
     import java.util.ArrayList;
     import java.util.Date;
     import javax.xml.parsers.ParserConfigurationException;
     import org.xml.sax.Attributes;
     import org.xml.sax.ContentHandler;
     import org.xml.sax.ErrorHandler;
     import org.xml.sax.InputSource;
     import org.xml.sax.SAXException;
     import org.xml.sax.SAXParseException;
     import org.xml.sax.XMLReader;
     import org.xml.sax.helpers.DefaultHandler;
     import org.xml.sax.helpers.XMLReaderFactory;
     import edu.ucsb.nceas.metacat.client.*;
     import edu.ucsb.nceas.metacat.client.Metacat;
     import edu.ucsb.nceas.metacat.client.MetacatException;
     import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;
     /**
-...
       private long delta;
       private String documentListURL;
       private Harvester harvester;
       private int harvestDocumentIndex = 0;
       private HarvestDocument[] harvestDocumentList = new HarvestDocument[30];
       private ArrayList harvestDocumentList = new ArrayList();
       private String harvestSiteEndTime;
       private String harvestSiteStartTime;
       private String ldapDN;
       private String ldapPassword;
       private String ldapPwd;
       final private long millisecondsPerDay = (1000 * 60 * 60 * 24);
       private int siteScheduleID;
       int siteScheduleID;
       private String unit;
       private int updateFrequency;
-...
        * @param siteScheduleID  the value of the SITE_SCHEDULE_ID field
        * @param documentListURL the value of the DOCUMENTLISTURL field
        * @param ldapDN          the value of the LDAPDN field
        * @param ldapPassword    the value of the LDAPPASSWORD field
        * @param ldapPwd    the value of the LDAPPASSWORD field
        * @param dateNextHarvest the value of the DATENEXTHARVEST field
        * @param dateLastHarvest the value of the DATELASTHARVEST field
        * @param updateFrequency the value of the UPDATEFREQUENCY field
-...
                                   int    siteScheduleID,
                                   String documentListURL,
                                   String ldapDN,
                                   String ldapPassword,
                                   String ldapPwd,
                                   String dateNextHarvest,
                                   String dateLastHarvest,
                                   int    updateFrequency,
-...
         this.siteScheduleID = siteScheduleID;
         this.documentListURL = documentListURL;
         this.ldapDN = ldapDN;
         this.ldapPassword = ldapPassword;
         this.ldapPwd = ldapPwd;
         this.dateNextHarvest = dateNextHarvest;
         this.dateLastHarvest = dateLastHarvest;
         this.updateFrequency = updateFrequency;
-...
        * harvest based on today's date and the update frequency.
        */
       private void dbUpdateHarvestSiteSchedule() {
     		Connection con;
     		Connection conn;
         long currentTime;                    // Current time in milliseconds
         Date dateNextHarvest;                // Date of next harvest
         String lastHarvest;
-...
     		Statement stmt;
         long timeNextHarvest;
         con = harvester.conn;
         conn = harvester.conn;
         now = new Date();
         currentTime = now.getTime();         // Current time in milliseconds
         currentTime = now.getTime();
         timeNextHarvest = currentTime + delta;
         dateNextHarvest = new Date(timeNextHarvest);
         nextHarvest = "'" + simpleDateFormat.format(dateNextHarvest) + "'";
         lastHarvest = "'" + simpleDateFormat.format(now) + "'";
         System.out.println("Date of next harvest: " + nextHarvest);
         System.out.println("Date of last harvest: " + lastHarvest);
     		try {
     			stmt = con.createStatement();
     			stmt.executeUpdate("UPDATE HARVEST_SITE_SCHEDULE SET DATENEXTHARVEST = " + nextHarvest + " WHERE SITE_SCHEDULE_ID = " + siteScheduleID);
     			stmt.executeUpdate("UPDATE HARVEST_SITE_SCHEDULE SET DATELASTHARVEST = " + lastHarvest + " WHERE SITE_SCHEDULE_ID = " + siteScheduleID);
     			stmt = conn.createStatement();
     			stmt.executeUpdate("UPDATE HARVEST_SITE_SCHEDULE SET DATENEXTHARVEST = " +
                              nextHarvest +
                              " WHERE SITE_SCHEDULE_ID = " +
                              siteScheduleID);
     			stmt.executeUpdate("UPDATE HARVEST_SITE_SCHEDULE SET DATELASTHARVEST = " +
                              lastHarvest +
                              " WHERE SITE_SCHEDULE_ID = " +
                              siteScheduleID);
     			stmt.close();
+    		}
         catch(SQLException e) {
     			System.err.println("SQLException: " + e.getMessage());
     			System.out.println("SQLException: " + e.getMessage());
+    		}
+      }
-...
           if (timeNextHarvest < currentTime) {
             dueForHarvest = true;
             System.out.println("Due for harvest: " + documentListURL);
+          }
           else {
             System.out.println("Next harvest date: " + dnh.toString());
             System.out.println("Not due for harvest: " + documentListURL);
+          }
+        }
         catch (ParseException e) {
           System.err.println("Error parsing date: " + e.getMessage());
           System.out.println("Error parsing date: " + e.getMessage());
+        }
         //return dueForHarvest;
         return true;
         return dueForHarvest;
+      }
-...
             parseDocumentList();
             metacatLogin();
             for (int i = 0; i < harvestDocumentList.length; i++) {
               harvestDocument = harvestDocumentList[i];
             for (int i = 0; i < harvestDocumentList.size(); i++) {
               harvestDocument = (HarvestDocument) harvestDocumentList.get(i);
               if (harvestDocument != null) {
                 harvestDocument.printOutput();
                 harvestDocument.harvestDocument();
+              }
+            }
-...
             dbUpdateHarvestSiteSchedule();
+          }
           catch (ParserConfigurationException e) {
             System.err.println("ParserConfigurationException: " + e.getMessage());
             System.out.println("ParserConfigurationException: " + e.getMessage());
+          }
           catch (SAXException e) {
             System.err.println("SAXException: " + e.getMessage());
+          }
           catch (IOException e) {
             System.err.println("IOException: " + e.getMessage());
+          }
           reportToSite();
+        }
-...
       /**
        * Login to Metacat using the ldapDN and ldapPassword
        * Login to Metacat using the ldapDN and ldapPwd
        */
       private void metacatLogin() {
         Metacat metacat = harvester.metacat;
         if (harvester.connectToMetacat()) {
           try {
             System.out.println("Logging in to Metacat: " + ldapDN);
             metacat.login(ldapDN, ldapPassword);
             metacat.login(ldapDN, ldapPwd);
             //System.out.println("Metacat login response: " + response);
             //sessionId = metacat.getSessionId();
             //System.out.println("Session ID: " + sessionId);
-...
           catch (Exception e) {
             System.out.println("Metacat login failed." + e.getMessage());
+          }
+        }
         else {
           System.out.println("Not logging in to Metacat");
+        }
+        }
+      }
-...
             System.out.println("Metacat exception: " + e.getMessage());
+          }
+        }
         else {
           System.out.println("Not logging out from Metacat");
+        }
+      }
       /**
        * Parse the site document list to find out which documents to harvest.
+       *
        * @throws SAXException
        * @throws IOException
        * @throws ParserConfigurationException
        */
       private void parseDocumentList()
         throws SAXException, IOException, ParserConfigurationException {
         // Create a parser factory and use it to create a parser
         SAXParserFactory parserFactory = SAXParserFactory.newInstance();
         SAXParser parser = parserFactory.newSAXParser();
         // Instantiate a DefaultHandler subclass to do your counting for you
         DocumentListHandler handler = new DocumentListHandler();
         // Start the parser. It reads the document list and calls methods of the handler.
         parser.parse(documentListURL, handler);
               throws ParserConfigurationException {
         DocumentListHandler documentListHandler = new DocumentListHandler();
         InputStream inputStream;
         InputStreamReader inputStreamReader;
         String schemaLocation = ".";
         URL url;
         try {
           url = new URL(documentListURL);
           inputStream = url.openStream();
           harvester.addLogEntry(0, "", "GetDocListSuccess",
                                 siteScheduleID, null, "");
           inputStreamReader = new InputStreamReader(inputStream);
           documentListHandler.runParser(inputStreamReader, schemaLocation);
           harvester.addLogEntry(0, "", "ValidateDocListSuccess",
                                 siteScheduleID, null, "");
+        }
         catch (MalformedURLException e){
           harvester.addLogEntry(1, "MalformedURLException: " + e.getMessage(),
                                 "GetDocListError", siteScheduleID, null, "");
+        }
         catch (FileNotFoundException e) {
           harvester.addLogEntry(1, "FileNotFoundException: " + e.getMessage(),
                                 "GetDocListError", siteScheduleID, null, "");
+        }
         catch (SAXException e) {
           harvester.addLogEntry(1, "SAXException: " + e.getMessage(),
                                 "ValidateDocListError", siteScheduleID, null, "");
+        }
         catch (ClassNotFoundException e) {
           harvester.addLogEntry(1, "ClassNotFoundException: " + e.getMessage(),
                                 "ValidateDocListError", siteScheduleID, null, "");
+        }
         catch (IOException e) {
           harvester.addLogEntry(1, "IOException: " + e.getMessage(),
                                 "GetDocListError", siteScheduleID, null, "");
+        }
+      }
-...
        * Prints the data that is stored in this HarvestSiteSchedule object.
        */
       void printOutput() {
         System.out.println("");
         System.out.println("siteScheduleID: " + siteScheduleID);
         System.out.println("documentListURL: " + documentListURL);
         System.out.println("ldapDN: " + ldapDN);
         System.out.println("ldapPassword: " + ldapPassword);
         System.out.println("dateNextHarvest: " + dateNextHarvest);
         System.out.println("dateLastHarvest: " + dateLastHarvest);
         System.out.println("updateFrequency: " + updateFrequency);
         System.out.println("unit: " + unit);
         System.out.println("contactEmail: " + contactEmail);
         System.out.println("siteScheduleID:       " + siteScheduleID);
         System.out.println("documentListURL:      " + documentListURL);
         System.out.println("ldapDN:               " + ldapDN);
         System.out.println("dateNextHarvest:      " + dateNextHarvest);
         System.out.println("dateLastHarvest:      " + dateLastHarvest);
         System.out.println("updateFrequency:      " + updateFrequency);
         System.out.println("unit:                 " + unit);
         System.out.println("contactEmail:         " + contactEmail);
+      }
       /**
        * Pushes a HarvestDocument object onto the harvestDocumentList.
+       *
        * @param harvestDocument    a new HarvestDocument object to add to the list
        */
       void pushHarvestDocument(HarvestDocument harvestDocument) {
         harvestDocumentList[harvestDocumentIndex] = harvestDocument;
         harvestDocumentIndex++;
+      }

... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff

Project

General

Profile

Metacat

Revision 2031

Added by Duane Costa about 20 years ago