Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2004 University of New Mexico and the

 *                  Regents of the University of California

 *     '$Date$'

 * '$Revision$'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

package edu.ucsb.nceas.metacat.harvesterClient;

import java.sql.DriverManager;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.SQLWarning;

import java.sql.Statement;

import java.util.ArrayList;

import java.text.SimpleDateFormat;

import java.util.Date;

import edu.ucsb.nceas.metacat.client.MetacatFactory;

import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;

/**

 * Harvester is the main class for the Harvester application. The main

 * method creates a single Harvester object which drives the application.

 *

 * @author    costa

 *

 */

public class Harvester {

  /*

   * Class fields

   */

  /*

   * Class methods

   */

   * Constructor. Creates a new instance of Harvester.

   */

  public Harvester() {

  }

    try {

	 * Harvester main method.

	 *

	 *

	 *   args[0] if "false", then this is not command-line mode,

	 *           Command-line mode is true by default.

	 *

	 *   args[1] if present, represents the path to the harvest list schema file.

	 *           Specifying it overrides the default path to the schema file.

	 *

	 * @throws IOException

	 * @throws ParserConfigurationException

	 */

	public static void main(String[] args) {

	    Integer delayDefault = new Integer(0); // Default number of hours delay

												// harvest

		Integer d; // Used for determining delay

		long delta; // endTime - startTime

		long endTime; // time that a harvest completes

		Harvester harvester; // object for a single harvest run

		Integer maxHarvestsDefault = new Integer(0); // Default max harvests

		int maxHarvests = maxHarvestsDefault.intValue(); // Max number of

															// harvests

		Integer mh; // used in determining max harvests

		int nHarvests = 0; // counts the number of harvest runs

		final long oneHour = (60 * 60 * 1000); // milliseconds in one hour

		Integer periodDefault = new Integer(24); // Default hours between

													// harvests

		int period = periodDefault.intValue(); // Hours between harvests

		Integer p; // Used in determining the period

		long startTime; // time that a harvest run starts

		 * If there is a second argument, it is the schemaLocation value

		 */

		if (args.length > 1) {

			schemaLocation = args[1];

				Thread.sleep(10000);

			} catch (InterruptedException e) {

				e.printStackTrace();

			}

		}

		System.out.println("Starting Harvester");

		try {

		} catch (NumberFormatException e) {

			System.out.println("NumberFormatException: Error parsing delay: "

					+ e.getMessage());

			System.out.println("Defaulting to delay=" + delayDefault);

			delay = delayDefault.intValue();

		} catch (PropertyNotFoundException pnfe) {

			System.out.println("PropertyNotFoundException: Error finding delay: "

					+ pnfe.getMessage());

			System.out.println("Defaulting to delay=" + delayDefault);

			delay = delayDefault.intValue();

		}

		try {

		} catch (NumberFormatException e) {

			System.out.println("NumberFormatException: Error parsing maxHarvests: "

					+ e.getMessage());

			System.out.println("Defaulting to maxHarvests=" + maxHarvestsDefault);

			maxHarvests = maxHarvestsDefault.intValue();

		} catch (PropertyNotFoundException pnfe) {

			System.out.println("PropertyNotFoundException: Error finding maxHarvests: "

					+ pnfe.getMessage());

			System.out.println("Defaulting to maxHarvests=" + maxHarvestsDefault);

			maxHarvests = maxHarvestsDefault.intValue();

		}

		// Parse the period property. Use default if necessary.

		try {

		} catch (NumberFormatException e) {

			System.out.println("NumberFormatException: Error parsing period: "

					+ e.getMessage());

			System.out.println("Defaulting to period=" + periodDefault);

			period = periodDefault.intValue();

		} catch (PropertyNotFoundException pnfe) {

			System.out.println("PropertyNotFoundException: Error finding period: "

					+ pnfe.getMessage());

			System.out.println("Defaulting to period=" + periodDefault);

			period = periodDefault.intValue();

		}

		// Sleep for delay number of hours prior to starting first harvest

		if (delay > 0) {

			try {

				System.out.print("First harvest will begin in " + delay);

				if (delay == 1) {

					System.out.println(" hour.");

				} else {

					System.out.println(" hours.");

				}

				Thread.sleep(delay * oneHour);

			} catch (InterruptedException e) {

				System.err.println("InterruptedException: " + e.getMessage());

				System.exit(1);

			}

		}

    //

      startTime = System.currentTimeMillis();

													// harvest

      harvester.readHarvestSiteSchedule();        // Read the database table

      harvester.harvest();                        // Harvest the documents

      harvester.shutdown();                       // Shut down Harvester

      endTime = System.currentTimeMillis();

      delta = endTime - startTime;

                             period + " hours.");

        catch (InterruptedException e) {

          System.err.println("InterruptedException: " + e.getMessage());

          System.exit(1);

        }

      }

    }

  /**

   * Set the keepRunning flag. If set to false, the main program will end

   * the while loop that keeps harvester running every period number of hours.

   * The static method is intended to be called from the HarvesterServlet class

   * which creates a thread to run Harvester. When the thread is destroyed, the

   * thread's destroy() method calls Harvester.setKeepRunning(false).

   *

   * @param keepRunning

   */

  static void setKeepRunning(boolean keepRunning) {

    Harvester.keepRunning = keepRunning;

  }

   * Object fields

   */

   *  Sometimes it's useful to test parts of the code without actually

   *  connecting to Metacat.

   */

  /** Highest DETAIL_LOG_ID primary key in the HARVEST_DETAIL_LOG table */

  private int detailLogID;

  private int harvestLogID;

  /** End time of this harvest session */

  private Date harvestEndTime;

  /** List of HarvestLog objects. Stores log entries for report generation. */

  private ArrayList harvestLogList = new ArrayList();

  /** List of HarvestSiteSchedule objects */

  private ArrayList harvestSiteScheduleList = new ArrayList();

  /** Start time of this harvest session */

  private Date harvestStartTime;

  /** Number of days to save log records. Any that are older are purged. */

  int logPeriod;

  /** Metacat client object */

  String smtpServer;

  String timestamp;

  /*

   * Object methods

   */

  /**

   * Creates a new HarvestLog object and adds it to the harvestLogList.

   *

   * @param  status          the status of the harvest operation

   * @param  message         the message text of the harvest operation

   * @param  harvestOperationCode  the harvest operation code

   * @param  siteScheduleID  the siteScheduleID for which this operation was

   *                         performed. 0 indicates that the operation did not

   *                         involve a particular harvest site.

   * @param  harvestDocument the associated HarvestDocument object. May be null.

   * @param  errorMessage    additional error message pertaining to document

   *                         error.

   */

  void addLogEntry(int    status,

                   String message,

                   String harvestOperationCode,

                   int    siteScheduleID,

                   HarvestDocument harvestDocument,

                   String errorMessage

                  ) {

    HarvestLog harvestLog;

    int detailLogID;

     * else call the extended constructor.

     */

    if (harvestDocument == null) {

                                  status, message, harvestOperationCode,

                                  siteScheduleID);

    else {

      harvestLog = new HarvestLog(this, conn, harvestLogID, detailLogID,

                                  harvestStartTime, status, message,

                                  harvestDocument, errorMessage);

    }

    harvestLogList.add(harvestLog);

  }

  public void closeConnection() {

    try {

      // Close the database connection

      System.out.println("Closing the database connection.");

      conn.close();

    }

    catch (SQLException e) {

      System.out.println("Database access failed " + e);

    }

  }

   * Determines whether Harvester should attempt to connect to Metacat.

   * Used during development and testing.

   *

   * @return     true if Harvester should connect, otherwise false

   */

  boolean connectToMetacat () {

    return connectToMetacat;

  }

  /**

   * Normalizes text prior to insertion into the HARVEST_LOG or

   * HARVEST_DETAIL_LOG tables. In particular, replaces the single quote

   * character with the double quote character. This prevents SQL errors

   * involving words that contain single quotes. Also removes \n and \r

   * characters from the text.

   *

   * @param text  the original string

   * @return      a string containing the normalized text

   */

    StringBuffer stringBuffer = new StringBuffer();

      c = text.charAt(i);

      switch (c) {

        case '\'':

          stringBuffer.append('\"');

          break;

        case '\r':

        case '\n':

          break;

        default:

          stringBuffer.append(c);

          break;

      }

    }

    return stringBuffer.toString();

  }

  /**

   * Returns a connection to the database. Opens the connection if a connection

   * has not already been made previously.

   *

   * @return  conn  the database Connection object

   */

  public Connection getConnection() {

    String dbDriver = "";

    String password = null;

    String user = null;

    if (conn == null) {

			dbDriver = PropertyService.getProperty("database.driver");

			defaultDB = PropertyService.getProperty("database.connectionURI");

			password = PropertyService.getProperty("database.password");

			user = PropertyService.getProperty("database.user");

		} catch (PropertyNotFoundException pnfe) {

			System.out.println("Can't find property " + pnfe);

	        System.exit(1);

		}

      try {

        Class.forName(dbDriver);

      }

      catch (ClassNotFoundException e) {

        System.out.println("Can't load driver " + e);

        System.exit(1);

      }

      // Make the database connection

      try {

        System.out.println("Getting connection to Harvester tables");

        conn = DriverManager.getConnection(defaultDB, user, password);

        // If a SQLWarning object is available, print its warning(s).

        // There may be multiple warnings chained.

        warn = conn.getWarnings();

        if (warn != null) {

          while (warn != null) {

            System.out.println("SQLState: " + warn.getSQLState());

            System.out.println("Message:  " + warn.getMessage());

            System.out.println("Vendor: " + warn.getErrorCode());

            System.out.println("");

            warn = warn.getNextWarning();

          }

        }

      }

      catch (SQLException e) {

        System.out.println("Database access failed " + e);

        System.exit(1);

      }

    }

    return conn;

  }

   * the DETAIL_LOG_ID field of the HARVEST_DETAIL_LOG table.

   *

   * @return  the current value of the detailLogID

   */

    detailLogID++;

    return currentValue;

  }

  /**

   * Gets the current value of the harvestLogID for storage as a primary key in

   * the HARVEST_LOG_ID field of the HARVEST_LOG table.

   *

   * @return  the current value of the detailLogID

   */

    harvestLogID++;

    return currentValue;

  }

  /**

   * Gets the maximum value of an integer field from a table.

   *

   * @param tableName  the database table name

   * @param fieldName  the field name of the integer field in the table

   * @return  the maximum integer stored in the fieldName field of tableName

   */

  private int getMaxValue(String tableName, String fieldName) {

    Statement stmt;

      stmt = conn.createStatement();

      ResultSet rs = stmt.executeQuery(query);

        fieldValue = rs.getInt(fieldName);

      stmt.close();

    }

    }

    return maxValue;

  }

  /**

   * Gets the minimum value of an integer field from a table.

   *

   * @param tableName  the database table name

   * @param fieldName  the field name of the integer field in the table

   * @return  the minimum integer stored in the fieldName field of tableName

   */

  private int getMinValue(String tableName, String fieldName) {

    int minValue = 0;

    int fieldValue;

    Statement stmt;

      stmt = conn.createStatement();

      ResultSet rs = stmt.executeQuery(query);

        fieldValue = rs.getInt(fieldName);

        if (minValue == 0) {

          minValue = fieldValue;

        }

        else {

          minValue = Math.min(minValue, fieldValue);

        }

      stmt.close();

    }

    }

  }

  /**

   * documents for that site if they are due to be harvested.

   *

   * @throws SAXException

   * @throws IOException

   * @throws ParserConfigurationException

   */

  private void harvest() {

    HarvestSiteSchedule harvestSiteSchedule;

      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);

      if (Harvester.schemaLocation != null) {

        harvestSiteSchedule.setSchemaLocation(Harvester.schemaLocation);

      }

  }

  /**

   * maximums + 1.

    harvestLogID = getMaxValue("HARVEST_LOG", "HARVEST_LOG_ID") + 1;

  }

   * @param siteScheduleID the siteScheduleId of the HarvestSiteSchedule. Will

   *                       have a value of 0 if no particular site is involved,

   *                       which indicates that the report is being prepared

   *                       for the Harvester Administrator rather than for a

   *                       particular Site Contact.

    int nErrors = 0;

    String phrase;

    for (int i = 0; i < harvestLogList.size(); i++) {

      harvestLog = (HarvestLog) harvestLogList.get(i);

      if ((siteScheduleID == 0) || (siteScheduleID == logSiteScheduleID)) {

        if (harvestLog.isErrorEntry()) {

          nErrors++;

        }

      }

    out.println(marker);

    if (nErrors > 0) {

      phrase = (nErrors == 1) ? " ERROR WAS " : " ERRORS WERE ";

      out.println("* A TOTAL OF " + nErrors + phrase + "DETECTED.");

      out.println("* Please see the log entries below for additonal details.");

    }

    else {

      out.println("* NO ERRORS WERE DETECTED DURING THIS HARVEST.");

    }

   * Prints harvest log entries for this harvest run. Entries may be filtered

   * for a particular site, or all entries may be printed.

   *

   * @param out            the PrintStream object to write to

   * @param maxCodeLevel   the maximum code level that should be printed,

   *                       e.g. "warning". Any log entries higher than this

   *                       level will not be printed.

   * @param siteScheduleID if greater than 0, indicates that the log

   *                       entry should only be printed for a particular site

   *                       as identified by its siteScheduleID. if 0, then

   *                       print output for all sites.

   */

  void printHarvestLog(PrintStream out, String maxCodeLevel, int siteScheduleID

                      ) {

    HarvestLog harvestLog;

    int logSiteScheduleID;

    int nErrors = 0;

    String phrase;

    out.println("");

    out.println(marker);

    out.println(filler);

    out.println("*                       LOG ENTRIES");

    out.println(filler);

    out.println(marker);

      harvestLog = (HarvestLog) harvestLogList.get(i);

      logSiteScheduleID = harvestLog.getSiteScheduleID();

      if ((siteScheduleID == 0) || (siteScheduleID == logSiteScheduleID)) {

        harvestLog.printOutput(out, maxCodeLevel);

      }

    }

  /**

   *

    for (int i = 0; i < harvestSiteScheduleList.size(); i++) {

      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);

      if (harvestSiteSchedule.siteScheduleID == siteScheduleID) {

    }

  }

  /**

   * Prunes old records from the HARVEST_LOG table. Records are removed if

   * their HARVEST_DATE is older than a given number of days, as stored in the

   * table that reference the to-be-pruned entries in the HARVEST_LOG table.

  private void pruneHarvestLog() {

    long currentTime = harvestStartTime.getTime(); // time in milliseconds

    Date dateLastLog;                    // Prune everything prior to this date

    String deleteStringDetailLog;

    final long millisecondsPerDay = (1000 * 60 * 60 * 24);

    int recordsDeleted;

    String dateString;

    String selectString;

    Statement stmt;

    deleteString = "DELETE FROM HARVEST_LOG WHERE HARVEST_DATE < ";

    deleteStringDetailLog =

                       "DELETE FROM HARVEST_DETAIL_LOG WHERE HARVEST_LOG_ID = ";

    dateLastLog = new Date(timeLastLog);

    dateString = "'" + simpleDateFormat.format(dateLastLog) + "'";

    deleteString += dateString;

      System.out.println(

                "Pruning log entries from HARVEST_DETAIL_LOG and HARVEST_LOG:");

      /* Get the list of entries that need to be pruned from the HARVEST_LOG

       * table.

       */

      stmt = conn.createStatement();