Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2004 University of New Mexico and the

 *                  Regents of the University of California

 *     '$Date$'

 * '$Revision$'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

package edu.ucsb.nceas.metacat.harvesterClient;

import java.io.FileInputStream;

import java.io.IOException;

import java.sql.DriverManager;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.SQLWarning;

import java.sql.Statement;

import java.util.ArrayList;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Properties;

import edu.ucsb.nceas.metacat.client.MetacatFactory;

import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;

/**

 * Harvester is the main class for the Harvester application. The main

 * method creates a single Harvester object which drives the application.

 *

 * @author    costa

 *

 */

public class Harvester {

  /*

   * Class fields

   */

"*****************************************************************************";

  static Properties properties;

  /*

   * Class methods

   */

   * Constructor. Creates a new instance of Harvester.

   */

  public Harvester() {

  }

    try {

      properties.load(new FileInputStream(configfile));

    }

    catch (IOException e) {

      System.out.println("IOException: " + e.getMessage());

      System.exit(1);

    }

    * Harvester main method.

    *

    * @param args        the command line arguments

    * @throws SAXException

    * @throws IOException

    * @throws ParserConfigurationException

    */

  public static void main(String[] args) {

    int delay = delayDefault.intValue();  // Delay in hours before first harvest

    Integer d;                            // Used for determining delay

    long delta;                           // endTime - startTime

    long endTime;                         // time that a harvest completes

    Harvester harvester;                  // object for a single harvest run

    Integer maxHarvestsDefault = new Integer(30);    // Default max harvests

    int maxHarvests = maxHarvestsDefault.intValue(); // Max number of harvests

    Integer mh;                              // used in determining max harvests

    int nHarvests = 0;                      // counts the number of harvest runs

    int period = periodDefault.intValue();   // Hours between harvests

    Integer p;                               // Used in determining the period

    System.out.println(marker);

    System.out.println("Starting Harvester");

    //properties.list(System.out);

    // Parse the delay property. Use default if necessary.

    try {

      d = Integer.valueOf(properties.getProperty(

                                                 "delay",

                                                 delayDefault.toString()

                                                ));

      delay = d.intValue();

    }

    catch (NumberFormatException e) {

      System.out.println("NumberFormatException: Error parsing delay: " +

                         e.getMessage());

      System.out.println("Defaulting to delay=" + delayDefault);

      delay = delayDefault.intValue();

    }

    // Parse the maxHarvests property. Use default if necessary.

    try {

      mh = Integer.valueOf(properties.getProperty(

                                                  "maxHarvests",

                                                  maxHarvestsDefault.toString()

                                                 ));

      maxHarvests = mh.intValue();

    }

    catch (NumberFormatException e) {

      System.out.println("NumberFormatException: Error parsing maxHarvests: " +

                         e.getMessage());

      System.out.println("Defaulting to maxHarvests=" + maxHarvestsDefault);

      maxHarvests = maxHarvestsDefault.intValue();

    }

    // Parse the period property. Use default if necessary.

    try {

      p = Integer.valueOf(properties.getProperty("period",

                                                 periodDefault.toString()

                                                ));

      period = p.intValue();

    }

    catch (NumberFormatException e) {

      System.out.println("NumberFormatException: Error parsing period: " +

                         e.getMessage());

      System.out.println("Defaulting to period=" + periodDefault);

      period = periodDefault.intValue();

    }

    if (delay > 0) {

      try {

        System.out.print("First harvest will begin in " + delay);

        if (delay == 1) {

          System.out.println(" hour.");

        }

        else {

          System.out.println(" hours.");

        }

      catch (InterruptedException e) {

          System.err.println("InterruptedException: " + e.getMessage());

          System.exit(1);

      }

    }

    // Repeat a new harvest once every period number of hours, until we reach

    // the maximum number of harvests. Subtract delta from the time period so

    // that each harvest will start at a fixed interval.

    //

    while (nHarvests < maxHarvests) {

      nHarvests++;

      startTime = System.currentTimeMillis();

      harvester = new Harvester();                // New object for this harvest

      harvester.startup(nHarvests, maxHarvests);  // Start up Harvester

      harvester.readHarvestSiteSchedule();        // Read the database table

      harvester.harvest();                        // Harvest the documents

      harvester.shutdown();                       // Shut down Harvester

      endTime = System.currentTimeMillis();

      delta = endTime - startTime;

      if (nHarvests < maxHarvests) {

        try {

        catch (InterruptedException e) {

          System.err.println("InterruptedException: " + e.getMessage());

          System.exit(1);

        }

      }

    }

  /*

   * Object fields

   */

   *  Sometimes it's useful to test parts of the code without actually

   *  connecting to Metacat.

   */

  /** Highest DETAIL_LOG_ID primary key in the HARVEST_DETAIL_LOG table */

  private int detailLogID;

  private int harvestLogID;

  /** End time of this harvest session */

  private Date harvestEndTime;

  /** List of HarvestLog objects. Stores log entries for report generation. */

  private ArrayList harvestLogList = new ArrayList();

  /** List of HarvestSiteSchedule objects */

  private ArrayList harvestSiteScheduleList = new ArrayList();

  /** Start time of this harvest session */

  private Date harvestStartTime;

  /** Number of days to save log records. Any that are older are purged. */

  int logPeriod;

  /** Metacat client object */

  String smtpServer;

  String timestamp;

  /*

   * Object methods

   */

  /**

   * Creates a new HarvestLog object and adds it to the harvestLogList.

   *

   * @param  status          the status of the harvest operation

   * @param  message         the message text of the harvest operation

   * @param  harvestOperationCode  the harvest operation code

   * @param  siteScheduleID  the siteScheduleID for which this operation was

   *                         performed. 0 indicates that the operation did not

   *                         involve a particular harvest site.

   * @param  harvestDocument the associated HarvestDocument object. May be null.

   * @param  errorMessage    additional error message pertaining to document

   *                         error.

   */

  void addLogEntry(int    status,

                   String message,

                   String harvestOperationCode,

                   int    siteScheduleID,

                   HarvestDocument harvestDocument,

                   String errorMessage

                  ) {

    HarvestLog harvestLog;

     * else call the extended constructor.

     */

    if (harvestDocument == null) {

      harvestLog = new HarvestLog(this, harvestStartTime, status, message,

                                  harvestOperationCode, siteScheduleID);

    }

    else {

      harvestLog = new HarvestLog(this, harvestStartTime, status, message,

                                  harvestOperationCode, siteScheduleID,

                                  harvestDocument, errorMessage);

    }

    harvestLogList.add(harvestLog);

  }

   * Determines whether Harvester should attempt to connect to Metacat.

   * Used during development and testing.

   *

   * @return     true if Harvester should connect, otherwise false

   */

  boolean connectToMetacat () {

    return connectToMetacat;

  }

  /**

   * Normalizes text prior to insertion into the HARVEST_LOG or

   * HARVEST_DETAIL_LOG tables. In particular, replaces the single quote

   * character with the double quote character. This prevents SQL errors

   * involving words that contain single quotes. Also removes \n and \r

   * characters from the text.

   *

   * @param text  the original string

   * @return      a string containing the normalized text

   */

  String dequoteText(String text) {

    char c;

    StringBuffer stringBuffer = new StringBuffer();

      c = text.charAt(i);

      switch (c) {

        case '\'':

          stringBuffer.append('\"');

          break;

        case '\r':

        case '\n':

          break;

        default:

          stringBuffer.append(c);

          break;

      }

    }

    return stringBuffer.toString();

  }

   * the DETAIL_LOG_ID field of the HARVEST_DETAIL_LOG table.

   *

   * @return  the current value of the detailLogID

   */

  int getDetailLogID() {

    int currentValue = detailLogID;

    detailLogID++;

    return currentValue;

  }

  /**

   * Gets the current value of the harvestLogID for storage as a primary key in

   * the HARVEST_LOG_ID field of the HARVEST_LOG table.

   *

   * @return  the current value of the detailLogID

   */

  int getHarvestLogID() {

    int currentValue = harvestLogID;

    harvestLogID++;

    return currentValue;

  }

  /**

   * Gets the maximum value of an integer field from a table.

   *

   * @param tableName  the database table name

   * @param fieldName  the field name of the integer field in the table

   * @return  the maximum integer stored in the fieldName field of tableName

   */

  private int getMaxValue(String tableName, String fieldName) {

    int maxValue = 100;

    int fieldValue;

		String query = "SELECT " + fieldName + " FROM " + tableName;

		Statement stmt;

		try {

			stmt = conn.createStatement();

			ResultSet rs = stmt.executeQuery(query);

			while (rs.next()) {

				fieldValue = rs.getInt(fieldName);

        maxValue = Math.max(maxValue, fieldValue);

			}

			stmt.close();

		}

    catch(SQLException ex) {

			System.out.println("SQLException: " + ex.getMessage());

		}

    return maxValue;

  }

  /**

   * Gets the minimum value of an integer field from a table.

   *

   * @param tableName  the database table name

   * @param fieldName  the field name of the integer field in the table

   * @return  the minimum integer stored in the fieldName field of tableName

   */

  private int getMinValue(String tableName, String fieldName) {

    int minValue = 0;

    int fieldValue;

		String query = "SELECT " + fieldName + " FROM " + tableName;

		Statement stmt;

		try {

			stmt = conn.createStatement();

			ResultSet rs = stmt.executeQuery(query);

			while (rs.next()) {

				fieldValue = rs.getInt(fieldName);

        if (minValue == 0) {

          minValue = fieldValue;

        }

        else {

          minValue = Math.min(minValue, fieldValue);

        }

			}

			stmt.close();

		}

    catch(SQLException ex) {

			System.out.println("SQLException: " + ex.getMessage());

		}

    return minValue;

  }

  /**

   * documents for that site if they are due to be harvested.

   *

   * @throws SAXException

   * @throws IOException

   * @throws ParserConfigurationException

   */

  private void harvest() {

    HarvestSiteSchedule harvestSiteSchedule;

      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);

      harvestSiteSchedule.harvestDocumentList();

  }

  /**

   * maximums + 1.

    detailLogID = getMaxValue("HARVEST_DETAIL_LOG", "DETAIL_LOG_ID") + 1;

    harvestLogID = getMaxValue("HARVEST_LOG", "HARVEST_LOG_ID") + 1;

  }

   * for a particular site, or all entries may be printed.

   * @param maxCodeLevel   the maximum code level that should be printed,

   *                       e.g. "warning". Any log entries higher than this

   *                       level will not be printed.

   * @param siteScheduleID if greater than 0, indicates that the log

   *                       entry should only be printed for a particular site

   *                       as identified by its siteScheduleID. if 0, then

   *                       print output for all sites.

                       String maxCodeLevel,

                       int siteScheduleID

                      ) {

    int nErrors = 0;

    String phrase;

    for (int i = 0; i < harvestLogList.size(); i++) {

      harvestLog = (HarvestLog) harvestLogList.get(i);

      if ((siteScheduleID == 0) || (siteScheduleID == logSiteScheduleID)) {

        if (harvestLog.isErrorEntry()) {

          nErrors++;

        }

      }

    out.println(marker);

    out.println("*");

    if (nErrors > 0) {

      phrase = (nErrors == 1) ? " ERROR WAS " : " ERRORS WERE ";

      out.println("* A TOTAL OF " + nErrors + phrase + "DETECTED.");

      out.println("* Please see the log entries below for additonal details.");

    }

    else {

      out.println("* NO ERRORS WERE DETECTED DURING THIS HARVEST.");

    }

    out.println("*");

    out.println(marker);

    for (int i = 0; i < harvestLogList.size(); i++) {

      harvestLog = (HarvestLog) harvestLogList.get(i);

      logSiteScheduleID = harvestLog.getSiteScheduleID();

      if ((siteScheduleID == 0) || (siteScheduleID == logSiteScheduleID)) {

        harvestLog.printOutput(out, maxCodeLevel);

      }

    }

  /**

   *

    for (int i = 0; i < harvestSiteScheduleList.size(); i++) {

      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);

      if (harvestSiteSchedule.siteScheduleID == siteScheduleID) {

    }

  }

  /**

   * Prunes old records from the HARVEST_DETAIL_LOG table. Records are

   * removed if the HARVEST_LOG_ID foreign key is less than the lowest

   * HARVEST_LOG_ID primary key in the HARVEST_LOG table.

   */

  private void pruneHarvestDetailLog() {

		String deleteString;

    int minHarvestLogID;

    int recordsDeleted;

		Statement stmt;

    deleteString = "DELETE FROM HARVEST_DETAIL_LOG WHERE HARVEST_LOG_ID < " +

                   minHarvestLogID;

			System.out.print("Pruning log entries from HARVEST_DETAIL_LOG: ");

			stmt = conn.createStatement();

			recordsDeleted = stmt.executeUpdate(deleteString);

			System.out.println(recordsDeleted + " records deleted");

			stmt.close();

		}

    catch(SQLException e) {

			System.out.println("SQLException: " + e.getMessage());

		}

  }

   * their HARVEST_DATE is older than a given number of days, as stored in the

   * logPeriod object field.

   */

  private void pruneHarvestLog() {

    long currentTime = harvestStartTime.getTime(); // time in milliseconds

    Date dateLastLog;                    // Prune everything prior to this date

		String deleteString;

    long delta;

    final long millisecondsPerDay = (1000 * 60 * 60 * 24);

    int recordsDeleted;

    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MMM-yyyy");

    String dateString;

		Statement stmt;

    long timeLastLog = 0;

    delta = logPeriod * millisecondsPerDay;

    deleteString = "DELETE FROM HARVEST_LOG WHERE HARVEST_DATE < ";

    timeLastLog = currentTime - delta;

    dateLastLog = new Date(timeLastLog);

    dateString = "'" + simpleDateFormat.format(dateLastLog) + "'";

    deleteString += dateString;

		try {

			System.out.print("Pruning log entries from HARVEST_LOG: ");

			stmt = conn.createStatement();

			recordsDeleted = stmt.executeUpdate(deleteString);

			System.out.println(recordsDeleted + " records deleted");

			stmt.close();

		}

    catch (SQLException e) {

			System.out.println("SQLException: " + e.getMessage());

		}

  }

  /**

   * a HarvestSiteSchedule object for each row in the table.

   */

  private void readHarvestSiteSchedule() {

    HarvestSiteSchedule harvestSiteSchedule;

    ResultSet rs;

    SQLWarning warn;

    Statement stmt;

    String contactEmail;

    String dateLastHarvest;

    String dateNextHarvest;

    String documentListURL;

    String ldapDN;

    String unit;

    int updateFrequency;

    try {

      // Read the HARVEST_SITE_SCHEDULE table

      stmt = conn.createStatement();

      rs = stmt.executeQuery("SELECT * FROM HARVEST_SITE_SCHEDULE");

      warn = rs.getWarnings();

      if (warn != null) {

        System.out.println("\n---Warning---\n");

        while (warn != null) {

          System.out.println("Message: " + warn.getMessage());

          System.out.println("SQLState: " + warn.getSQLState());

          System.out.print("Vendor error code: ");

          System.out.println(warn.getErrorCode());

          System.out.println("");

          warn = warn.getNextWarning();

        }

      }

      while (rs.next()) {

        siteScheduleID = rs.getInt("SITE_SCHEDULE_ID");

        documentListURL = rs.getString("DOCUMENTLISTURL");

        ldapDN = rs.getString("LDAPDN");

        dateLastHarvest = rs.getString("DATELASTHARVEST");

        updateFrequency = rs.getInt("UPDATEFREQUENCY");

        unit = rs.getString("UNIT");

        contactEmail = rs.getString("CONTACT_EMAIL");

        warn = rs.getWarnings();

        if (warn != null) {

          System.out.println("\n---Warning---\n");

          while (warn != null) {

            System.out.println("Message: " + warn.getMessage());

            System.out.println("SQLState: " + warn.getSQLState());

            System.out.print("Vendor error code: ");

            System.out.println(warn.getErrorCode());

            System.out.println("");

            warn = warn.getNextWarning();

          }

        }

                                                      documentListURL,

                                                      ldapDN,

                                                      dateLastHarvest,

                                                      updateFrequency,

                                                      unit,

                                                      contactEmail

                                                     );

      rs.close();

      stmt.close();

    }

      System.out.println("Database access failed " + e);

      System.exit(1);

    }

  }

  /**

   * entry pertaining to this harvest run.

   * @param maxCodeLevel  the maximum code level that should be printed,

   *                      e.g. "warning". Any log entries higher than this

   *                      level will not be printed.

    String from = harvesterAdministrator;

    MailMessage msg;

    if (!to.equals("")) {

      System.out.println("Sending report to Harvester Administrator at address "

                         + harvesterAdministrator);

      try {

        msg = new MailMessage(smtpServer);

        msg.from(from);

        msg.to(to);

        msg.setSubject(subject);

        body = msg.getPrintStream();

      }

      catch (IOException e) {

        System.out.println("There was a problem sending email to " + to);

        System.out.println("IOException: " + e.getMessage());

      }

  /**

   * Shuts down Harvester. Performs cleanup operations such as logging out

   * of Metacat and disconnecting from the database.

   */

  private void shutdown() {

                                    // ("error") to level 5 ("debug")

    int siteScheduleID = 0;

    System.out.println("Shutting Down Harvester");

    pruneHarvestLog();

    pruneHarvestDetailLog();