Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2004 University of New Mexico and the

 *                  Regents of the University of California

 *     '$Date$'

 * '$Revision$'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

package edu.ucsb.nceas.metacat.harvesterClient;

import java.io.FileInputStream;

import java.io.IOException;

import java.sql.DriverManager;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.SQLWarning;

import java.sql.Statement;

import java.util.ArrayList;

import java.text.SimpleDateFormat;

import java.util.Date;

import edu.ucsb.nceas.metacat.client.MetacatFactory;

import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;

/**

 * Harvester is the main class for the Harvester application. The main

 * method creates a single Harvester object which drives the application.

 *

 * @author    costa

 *

 */

public class Harvester {

  /*

   * Class fields

   */

  private static final String CONFIG_DIR_TEST = "./build/war/WEB-INF";

  private static final String CONFIG_NAME = "metacat.properties";

  public static final String filler = "*";

  public static final String marker =

  /*

   * Class methods

   */

   * Constructor. Creates a new instance of Harvester.

   */

  public Harvester() {

  }

    String configDir = test ? CONFIG_DIR_TEST : CONFIG_DIR;

    File propertyFile = new File(configDir, CONFIG_NAME);

    try {

    }

    * Harvester main method.

    *

    * @param args        the command line arguments

    * @throws SAXException

    * @throws IOException

    * @throws ParserConfigurationException

    */

  public static void main(String[] args) {

    int delay = delayDefault.intValue();  // Delay in hours before first harvest

    Integer d;                            // Used for determining delay

    long delta;                           // endTime - startTime

    long endTime;                         // time that a harvest completes

    Harvester harvester;                  // object for a single harvest run

    Integer maxHarvestsDefault = new Integer(30);    // Default max harvests

    int maxHarvests = maxHarvestsDefault.intValue(); // Max number of harvests

    Integer mh;                              // used in determining max harvests

    int nHarvests = 0;                      // counts the number of harvest runs

    int period = periodDefault.intValue();   // Hours between harvests

    Integer p;                               // Used in determining the period

    System.out.println(marker);

    System.out.println("Starting Harvester");

    // Parse the delay property. Use default if necessary.

    try {

    }

    catch (NumberFormatException e) {

      System.out.println("NumberFormatException: Error parsing delay: " +

                         e.getMessage());

      System.out.println("Defaulting to delay=" + delayDefault);

      delay = delayDefault.intValue();

    }

    // Parse the maxHarvests property. Use default if necessary.

    try {

    }

    catch (NumberFormatException e) {

      System.out.println("NumberFormatException: Error parsing maxHarvests: " +

                         e.getMessage());

      System.out.println("Defaulting to maxHarvests=" + maxHarvestsDefault);

      maxHarvests = maxHarvestsDefault.intValue();

    }

    // Parse the period property. Use default if necessary.

    try {

    }

    catch (NumberFormatException e) {

      System.out.println("NumberFormatException: Error parsing period: " +

                         e.getMessage());

      System.out.println("Defaulting to period=" + periodDefault);

      period = periodDefault.intValue();

    }

    if (delay > 0) {

      try {

        System.out.print("First harvest will begin in " + delay);

        if (delay == 1) {

          System.out.println(" hour.");

        }

        else {

          System.out.println(" hours.");

        }

      catch (InterruptedException e) {

          System.err.println("InterruptedException: " + e.getMessage());

          System.exit(1);

      }

    }

    // Repeat a new harvest once every period number of hours, until we reach

    // the maximum number of harvests. Subtract delta from the time period so

    // that each harvest will start at a fixed interval.

    //

    while (nHarvests < maxHarvests) {

      nHarvests++;

      startTime = System.currentTimeMillis();

      harvester = new Harvester();                // New object for this harvest

      harvester.startup(nHarvests, maxHarvests);  // Start up Harvester

      harvester.readHarvestSiteSchedule();        // Read the database table

      harvester.harvest();                        // Harvest the documents

      harvester.shutdown();                       // Shut down Harvester

      endTime = System.currentTimeMillis();

      delta = endTime - startTime;

      if (nHarvests < maxHarvests) {

        try {

                             period + " hours.");

        catch (InterruptedException e) {

          System.err.println("InterruptedException: " + e.getMessage());

          System.exit(1);

        }

      }

    }

  /*

   * Object fields

   */

   *  Sometimes it's useful to test parts of the code without actually

   *  connecting to Metacat.

   */

  /** Highest DETAIL_LOG_ID primary key in the HARVEST_DETAIL_LOG table */

  private int detailLogID;

  private int harvestLogID;

  /** End time of this harvest session */

  private Date harvestEndTime;

  /** List of HarvestLog objects. Stores log entries for report generation. */

  private ArrayList harvestLogList = new ArrayList();

  /** List of HarvestSiteSchedule objects */

  private ArrayList harvestSiteScheduleList = new ArrayList();

  /** Start time of this harvest session */

  private Date harvestStartTime;

  /** Number of days to save log records. Any that are older are purged. */

  int logPeriod;

  /** Metacat client object */

  String smtpServer;

  String timestamp;

  /*

   * Object methods

   */

  /**

   * Creates a new HarvestLog object and adds it to the harvestLogList.

   *

   * @param  status          the status of the harvest operation

   * @param  message         the message text of the harvest operation

   * @param  harvestOperationCode  the harvest operation code

   * @param  siteScheduleID  the siteScheduleID for which this operation was

   *                         performed. 0 indicates that the operation did not

   *                         involve a particular harvest site.

   * @param  harvestDocument the associated HarvestDocument object. May be null.

   * @param  errorMessage    additional error message pertaining to document

   *                         error.

   */

  void addLogEntry(int    status,

                   String message,

                   String harvestOperationCode,

                   int    siteScheduleID,

                   HarvestDocument harvestDocument,

                   String errorMessage

                  ) {

    HarvestLog harvestLog;

    int detailLogID;

     * else call the extended constructor.

     */

    if (harvestDocument == null) {

                                  status, message, harvestOperationCode,

                                  siteScheduleID);

    else {

      harvestLog = new HarvestLog(this, conn, harvestLogID, detailLogID,

                                  harvestStartTime, status, message,

                                  harvestDocument, errorMessage);

    }

    harvestLogList.add(harvestLog);

  }

  public void closeConnection() {

    try {

      // Close the database connection

      System.out.println("Closing the database connection.");

      conn.close();

    }

    catch (SQLException e) {

      System.out.println("Database access failed " + e);

    }

  }

   * Determines whether Harvester should attempt to connect to Metacat.

   * Used during development and testing.

   *

   * @return     true if Harvester should connect, otherwise false

   */

  boolean connectToMetacat () {

    return connectToMetacat;

  }

  /**

   * Normalizes text prior to insertion into the HARVEST_LOG or

   * HARVEST_DETAIL_LOG tables. In particular, replaces the single quote

   * character with the double quote character. This prevents SQL errors

   * involving words that contain single quotes. Also removes \n and \r

   * characters from the text.

   *

   * @param text  the original string

   * @return      a string containing the normalized text

   */

    StringBuffer stringBuffer = new StringBuffer();

      c = text.charAt(i);

      switch (c) {

        case '\'':

          stringBuffer.append('\"');

          break;

        case '\r':

        case '\n':

          break;

        default:

          stringBuffer.append(c);

          break;

      }

    }

    return stringBuffer.toString();

  }

  /**

   * Returns a connection to the database. Opens the connection if a connection

   * has not already been made previously.

   *

   * @return  conn  the database Connection object

   */

  public Connection getConnection() {

    String dbDriver = "";

		String defaultDB;

    String password;

    String user;

    SQLWarning warn;

    if (conn == null) {

      defaultDB = options.getOption("defaultDB");

      password = options.getOption("password");

      user = options.getOption("user");

      try {

        Class.forName(dbDriver);

      }

      catch (ClassNotFoundException e) {

        System.out.println("Can't load driver " + e);

        System.exit(1);

      }

      // Make the database connection

      try {

        System.out.println("Getting connection to Harvester tables");

        conn = DriverManager.getConnection(defaultDB, user, password);

        // If a SQLWarning object is available, print its warning(s).

        // There may be multiple warnings chained.

        warn = conn.getWarnings();

        if (warn != null) {

          while (warn != null) {

            System.out.println("SQLState: " + warn.getSQLState());

            System.out.println("Message:  " + warn.getMessage());

            System.out.println("Vendor: " + warn.getErrorCode());

            System.out.println("");

            warn = warn.getNextWarning();

          }

        }

      }

      catch (SQLException e) {

        System.out.println("Database access failed " + e);

        System.exit(1);

      }

    }

    return conn;

  }

   * the DETAIL_LOG_ID field of the HARVEST_DETAIL_LOG table.

   *

   * @return  the current value of the detailLogID

   */

    detailLogID++;

    return currentValue;

  }

  /**

   * Gets the current value of the harvestLogID for storage as a primary key in

   * the HARVEST_LOG_ID field of the HARVEST_LOG table.

   *

   * @return  the current value of the detailLogID

   */

    harvestLogID++;

    return currentValue;

  }

  /**

   * Gets the maximum value of an integer field from a table.

   *

   * @param tableName  the database table name

   * @param fieldName  the field name of the integer field in the table

   * @return  the maximum integer stored in the fieldName field of tableName

   */

  private int getMaxValue(String tableName, String fieldName) {

		String query = "SELECT " + fieldName + " FROM " + tableName;

		Statement stmt;

		try {

			stmt = conn.createStatement();

			ResultSet rs = stmt.executeQuery(query);

			while (rs.next()) {

				fieldValue = rs.getInt(fieldName);

        maxValue = Math.max(maxValue, fieldValue);

			}

			stmt.close();

		}

    catch(SQLException ex) {

			System.out.println("SQLException: " + ex.getMessage());

		}

    return maxValue;

  }

  /**

   * Gets the minimum value of an integer field from a table.

   *

   * @param tableName  the database table name

   * @param fieldName  the field name of the integer field in the table

   * @return  the minimum integer stored in the fieldName field of tableName

   */

  private int getMinValue(String tableName, String fieldName) {

    int minValue = 0;

    int fieldValue;

		String query = "SELECT " + fieldName + " FROM " + tableName;

		Statement stmt;

		try {

			stmt = conn.createStatement();

			ResultSet rs = stmt.executeQuery(query);

			while (rs.next()) {

				fieldValue = rs.getInt(fieldName);

        if (minValue == 0) {

          minValue = fieldValue;

        }

        else {

          minValue = Math.min(minValue, fieldValue);

        }

			}

			stmt.close();

		}

    catch(SQLException ex) {

			System.out.println("SQLException: " + ex.getMessage());

		}

    return minValue;

  }

  /**

   * documents for that site if they are due to be harvested.

   *

   * @throws SAXException

   * @throws IOException

   * @throws ParserConfigurationException

   */

  private void harvest() {

    HarvestSiteSchedule harvestSiteSchedule;

      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);

      harvestSiteSchedule.harvestDocumentList();

  }

  /**

   * maximums + 1.

    harvestLogID = getMaxValue("HARVEST_LOG", "HARVEST_LOG_ID") + 1;

  }

   * @param siteScheduleID the siteScheduleId of the HarvestSiteSchedule. Will

   *                       have a value of 0 if no particular site is involved,

   *                       which indicates that the report is being prepared

   *                       for the Harvester Administrator rather than for a

   *                       particular Site Contact.

    int nErrors = 0;

    String phrase;

    for (int i = 0; i < harvestLogList.size(); i++) {

      harvestLog = (HarvestLog) harvestLogList.get(i);

      if ((siteScheduleID == 0) || (siteScheduleID == logSiteScheduleID)) {

        if (harvestLog.isErrorEntry()) {

          nErrors++;

        }

      }

    out.println(marker);

    if (nErrors > 0) {

      phrase = (nErrors == 1) ? " ERROR WAS " : " ERRORS WERE ";

      out.println("* A TOTAL OF " + nErrors + phrase + "DETECTED.");

      out.println("* Please see the log entries below for additonal details.");

    }

    else {

      out.println("* NO ERRORS WERE DETECTED DURING THIS HARVEST.");

    }

   * Prints harvest log entries for this harvest run. Entries may be filtered

   * for a particular site, or all entries may be printed.

   *

   * @param out            the PrintStream object to write to

   * @param maxCodeLevel   the maximum code level that should be printed,

   *                       e.g. "warning". Any log entries higher than this

   *                       level will not be printed.

   * @param siteScheduleID if greater than 0, indicates that the log

   *                       entry should only be printed for a particular site

   *                       as identified by its siteScheduleID. if 0, then

   *                       print output for all sites.

   */

  void printHarvestLog(PrintStream out, String maxCodeLevel, int siteScheduleID

                      ) {

    HarvestLog harvestLog;

    int logSiteScheduleID;

    int nErrors = 0;

    String phrase;

    out.println("");

    out.println(marker);

    out.println(filler);

    out.println("*                       LOG ENTRIES");

    out.println(filler);

    out.println(marker);

      harvestLog = (HarvestLog) harvestLogList.get(i);

      logSiteScheduleID = harvestLog.getSiteScheduleID();

      if ((siteScheduleID == 0) || (siteScheduleID == logSiteScheduleID)) {

        harvestLog.printOutput(out, maxCodeLevel);

      }

    }

  /**

   *

    for (int i = 0; i < harvestSiteScheduleList.size(); i++) {

      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);

      if (harvestSiteSchedule.siteScheduleID == siteScheduleID) {

    }

  }

  /**

   * Prunes old records from the HARVEST_DETAIL_LOG table. Records are

   * removed if the HARVEST_LOG_ID foreign key is less than the lowest

   * HARVEST_LOG_ID primary key in the HARVEST_LOG table.

   */

  private void pruneHarvestDetailLog() {

		String deleteString;

    int minHarvestLogID;

    int recordsDeleted;

		Statement stmt;

    deleteString = "DELETE FROM HARVEST_DETAIL_LOG WHERE HARVEST_LOG_ID < " +

                   minHarvestLogID;

			System.out.print("Pruning log entries from HARVEST_DETAIL_LOG: ");

			recordsDeleted = stmt.executeUpdate(deleteString);

			System.out.println(recordsDeleted + " records deleted");

			stmt.close();

		}

    catch(SQLException e) {

			System.out.println("SQLException: " + e.getMessage());

		}

  }

   * their HARVEST_DATE is older than a given number of days, as stored in the

   * logPeriod object field.

   */

  private void pruneHarvestLog() {

    long currentTime = harvestStartTime.getTime(); // time in milliseconds

    Date dateLastLog;                    // Prune everything prior to this date

		String deleteString;

    long delta;

    final long millisecondsPerDay = (1000 * 60 * 60 * 24);

    int recordsDeleted;

    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MMM-yyyy");

    String dateString;

		Statement stmt;

    long timeLastLog = 0;

    delta = logPeriod * millisecondsPerDay;

    deleteString = "DELETE FROM HARVEST_LOG WHERE HARVEST_DATE < ";

    timeLastLog = currentTime - delta;

    dateLastLog = new Date(timeLastLog);

    dateString = "'" + simpleDateFormat.format(dateLastLog) + "'";

    deleteString += dateString;

		try {

			System.out.print("Pruning log entries from HARVEST_LOG: ");

			recordsDeleted = stmt.executeUpdate(deleteString);

			System.out.println(recordsDeleted + " records deleted");

			stmt.close();

		}

    catch (SQLException e) {

			System.out.println("SQLException: " + e.getMessage());

		}

  }

  /**

   * a HarvestSiteSchedule object for each row in the table.

   */

  private void readHarvestSiteSchedule() {

    HarvestSiteSchedule harvestSiteSchedule;

    ResultSet rs;

    SQLWarning warn;

    Statement stmt;

    String contactEmail;

    String dateLastHarvest;

    String dateNextHarvest;

    String documentListURL;

    String ldapDN;

    String unit;

    int updateFrequency;

    try {

      // Read the HARVEST_SITE_SCHEDULE table

      stmt = conn.createStatement();

      rs = stmt.executeQuery("SELECT * FROM HARVEST_SITE_SCHEDULE");

      warn = rs.getWarnings();

      if (warn != null) {

        System.out.println("\n---Warning---\n");

        while (warn != null) {

          System.out.println("Message: " + warn.getMessage());

          System.out.println("SQLState: " + warn.getSQLState());

          System.out.print("Vendor error code: ");

          System.out.println(warn.getErrorCode());

          System.out.println("");

          warn = warn.getNextWarning();

        }

      }

      while (rs.next()) {

        siteScheduleID = rs.getInt("SITE_SCHEDULE_ID");

        documentListURL = rs.getString("DOCUMENTLISTURL");