Metacat

/*

 * Harvester.java

 *

 * Created on January 14, 2004, 4:44 PM

 */

package edu.ucsb.nceas.metacat.harvesterClient;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.SQLWarning;

import java.sql.Statement;

import java.util.ArrayList;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Properties;

import edu.ucsb.nceas.metacat.client.Metacat;

import edu.ucsb.nceas.metacat.client.MetacatFactory;

import edu.ucsb.nceas.metacat.client.MetacatInaccessibleException;

/**

 * Harvester is the main class for the Harvester application. The main

 * method creates a single Harvester object which drives the application.

 * 

 * @author    costa

 * 

 */

public class Harvester {

  /*

   * Class fields

   */

  /* 

   * Class methods

   */

  /**

   * Constructor. Creates a new instance of Harvester.

   */

  public Harvester() {

  }

  /**

   * Determines whether Harvester is running on a Win32 platform. Used

   * during development.

   * 

   * @return    true if this in Win32, false otherwise

   */

  public static boolean isWin32 () {

    boolean isWin32;

    String osName = System.getProperty("os.name");

    isWin32 = (osName.startsWith("Windows"));

    return isWin32;

  }

  /**

    * Harvester main method.

    * 

    * @param args        the command line arguments

    * @throws SAXException

    * @throws IOException

    * @throws ParserConfigurationException

    */

  public static void main(String[] args) {

    Harvester harvester = new Harvester();

    harvester.startup();                  // Start up Harvester

    harvester.readHarvestSiteSchedule();  // Read the database table

    harvester.harvest();                  // Harvest the documents

    harvester.shutdown();                 // Shut down Harvester

  }

  /*

   * Object fields

   */

  /** Database connection */

  Connection conn;

  /** Used during development to determine whether to connect to metacat */

  private boolean connectToMetacat;

  /** Highest DETAIL_LOG_ID primary key in the HARVEST_DETAIL_LOG table */

  private int detailLogID;

  /** Highest HARVEST_LOG_ID primary key in the HARVEST_LOG table */

  private int harvestLogID;

  /** End time of this harvest session */

  private Date harvestEndTime;

  /** List of HarvestLog objects. Stores log entries for report generation. */

  private ArrayList harvestLogList = new ArrayList();

  /** List of HarvestSiteSchedule objects */

  private ArrayList harvestSiteScheduleList = new ArrayList();

  /** Start time of this harvest session */

  private Date harvestStartTime;

  /** Number of days to save log records. Any that are older are purged. */

  int logPeriod;

  /** Metacat client object */

  Metacat metacat;

  /** Loads harvester properties from configuration file */

  Properties properties;

  /*

   * Object methods

   */

  /*                

                    Harvester  harvester,

                    Date       harvestDate,

                    int        status,

                    String     message, 

                    String     harvestOperationCode,

                    int        siteScheduleID,

                    HarvestDocument harvestDocument,

                    String     errorMessage 

*/

  /**

   * Creates a new HarvestLog object and adds it to the harvestLogList.

   * 

   * @param  status          the status of the harvest operation

   * @param  message         the message text of the harvest operation

   * @param  harvestOperationCode  the harvest operation code

   * @param  siteScheduleID  the siteScheduleID for which this operation was

   *                         performed. 0 indicates that the operation did not

   *                         involve a particular harvest site.

   * @param  harvestDocument the associated HarvestDocument object. May be null.

   * @param  errorMessage    additional error message pertaining to document

   *                         error.

   */

  void addLogEntry(int    status,

                   String message,

                   String harvestOperationCode,

                   int    siteScheduleID,

                   HarvestDocument harvestDocument,

                   String errorMessage

                  ) {

    HarvestLog harvestLog;

    /* If there is no associated harvest document, call the basic constructor;

     * else call the extended constructor.

     */

    if (harvestDocument == null) {    

      harvestLog = new HarvestLog(this, harvestStartTime, status, message,

                                  harvestOperationCode, siteScheduleID);

    }

    else {

      harvestLog = new HarvestLog(this, harvestStartTime, status, message,

                                  harvestOperationCode, siteScheduleID,

                                  harvestDocument, errorMessage);

    }

    harvestLogList.add(harvestLog);

  }

  /**

   * Determines whether Harvester should attempt to connect to Metacat.

   * Used during development and testing.

   * 

   * @return     true if Harvester should connect, otherwise false

   */

  boolean connectToMetacat () {

    return connectToMetacat;

  }

  /**

   * Normalizes text prior to insertion into the HARVEST_LOG or

   * HARVEST_DETAIL_LOG tables. In particular, replaces the single quote

   * character with the double quote character. This prevents SQL errors

   * involving words that contain single quotes. Also removes \n and \r

   * characters from the text.

   * 

   * @param text  the original string

   * @return      a string containing the normalized text

   */

  String dequoteText(String text) {

    char c;

    StringBuffer stringBuffer = new StringBuffer();

    for (int i = 0; i < text.length(); i++) {

      c = text.charAt(i);

      switch (c) {

        case '\'':

          stringBuffer.append('\"');

          break;

        case '\r':

        case '\n':

          break;

        default:

          stringBuffer.append(c);

          break;

      }

    }

    return stringBuffer.toString();

  }

  /**

   * Gets the current value of the detailLogID for storage as a primary key in

   * the DETAIL_LOG_ID field of the HARVEST_DETAIL_LOG table.

   * 

   * @return  the current value of the detailLogID

   */

  int getDetailLogID() {

    int currentValue = detailLogID;

    detailLogID++;

    return currentValue;

  }

  /**

   * Gets the current value of the harvestLogID for storage as a primary key in

   * the HARVEST_LOG_ID field of the HARVEST_LOG table.

   * 

   * @return  the current value of the detailLogID

   */

  int getHarvestLogID() {

    int currentValue = harvestLogID;

    harvestLogID++;

    return currentValue;

  }

  /** 

   * Gets the maximum value of an integer field from a table.

   * 

   * @param tableName  the database table name

   * @param fieldName  the field name of the integer field in the table

   * @return  the maximum integer stored in the fieldName field of tableName

   */

  private int getMaxValue(String tableName, String fieldName) {

    int maxValue = 100;

    int fieldValue;

		String query = "SELECT " + fieldName + " FROM " + tableName;

		Statement stmt;

		try {

			stmt = conn.createStatement();							

			ResultSet rs = stmt.executeQuery(query);

			while (rs.next()) {

				fieldValue = rs.getInt(fieldName);

        maxValue = Math.max(maxValue, fieldValue);

			}

			stmt.close();	

		} 

    catch(SQLException ex) {

			System.out.println("SQLException: " + ex.getMessage());

		}

    return maxValue;

  }

  /** 

   * Gets the minimum value of an integer field from a table.

   * 

   * @param tableName  the database table name

   * @param fieldName  the field name of the integer field in the table

   * @return  the minimum integer stored in the fieldName field of tableName

   */

  private int getMinValue(String tableName, String fieldName) {

    int minValue = 0;

    int fieldValue;

		String query = "SELECT " + fieldName + " FROM " + tableName;

		Statement stmt;

		try {

			stmt = conn.createStatement();							

			ResultSet rs = stmt.executeQuery(query);

			while (rs.next()) {

				fieldValue = rs.getInt(fieldName);

        if (minValue == 0) {

          minValue = fieldValue;

        }

        else {

          minValue = Math.min(minValue, fieldValue);

        }

			}

			stmt.close();	

		} 

    catch(SQLException ex) {

			System.out.println("SQLException: " + ex.getMessage());

		}

    return minValue;

  }

  /**

   * For every Harvest site schedule in the database, harvest the

   * documents for that site if they are due to be harvested.

   * 

   * @throws SAXException

   * @throws IOException

   * @throws ParserConfigurationException

   */

  private void harvest() {

    HarvestSiteSchedule harvestSiteSchedule;

    for (int i = 0; i < harvestSiteScheduleList.size(); i++) {

      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);

      harvestSiteSchedule.harvestDocumentList();

    }

  }

  /**

   * Initializes the detailLogID and harvestLogID values to their current

   * maximums + 1.

   */

  private void initLogIDs() {

    detailLogID = getMaxValue("HARVEST_DETAIL_LOG", "DETAIL_LOG_ID") + 1;

    harvestLogID = getMaxValue("HARVEST_LOG", "HARVEST_LOG_ID") + 1;

  }

  /**

   * Loads Harvester properties from a configuration file.

   */

  private void loadProperties() {

    String homedir = System.getProperty("user.home");

    File configfile = new File(homedir, "harvester.properties");

    properties = new Properties();

    try {

      properties.load(new FileInputStream(configfile));

      properties.list(System.out);

    }

    catch (IOException e) {

      System.out.println("IOException: " + e.getMessage());

      System.exit(1);

    }

  }

  void printHarvestSiteSchedule(int siteScheduleID) {

     HarvestSiteSchedule harvestSiteSchedule;

    for (int i = 0; i < harvestSiteScheduleList.size(); i++) {

      harvestSiteSchedule = (HarvestSiteSchedule)harvestSiteScheduleList.get(i);

      if (harvestSiteSchedule.siteScheduleID == siteScheduleID) {

        harvestSiteSchedule.printOutput();

      }

    }

  }

  /**

   * Prunes old records from the HARVEST_DETAIL_LOG table. Records are

   * removed if the HARVEST_LOG_ID foreign key is less than the lowest

   * HARVEST_LOG_ID primary key in the HARVEST_LOG table.

   */

  private void pruneHarvestDetailLog() {

		String deleteString;

    int minHarvestLogID;

    int recordsDeleted;

		Statement stmt;

    minHarvestLogID = getMinValue("HARVEST_LOG", "HARVEST_LOG_ID");

    deleteString = "DELETE FROM HARVEST_DETAIL_LOG WHERE HARVEST_LOG_ID < " +

                   minHarvestLogID;

		try {

			System.out.print("Pruning log entries from HARVEST_DETAIL_LOG: ");

			stmt = conn.createStatement();							

			recordsDeleted = stmt.executeUpdate(deleteString);

			System.out.println(recordsDeleted + " records deleted");

			stmt.close();

		}

    catch(SQLException e) {

			System.out.println("SQLException: " + e.getMessage());

		}

  }

  /**

   * Prunes old records from the HARVEST_LOG table. Records are removed if

   * their HARVEST_DATE is older than a given number of days, as stored in the

   * logPeriod object field.

   */

  private void pruneHarvestLog() {

    long currentTime = harvestStartTime.getTime(); // time in milliseconds

    Date dateLastLog;                    // Prune everything prior to this date

		String deleteString;

    long delta;

    final long millisecondsPerDay = (1000 * 60 * 60 * 24);

    int recordsDeleted;

    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MMM-yyyy");

    String dateString;

		Statement stmt;

    long timeLastLog = 0;

    delta = logPeriod * millisecondsPerDay;

    deleteString = "DELETE FROM HARVEST_LOG WHERE HARVEST_DATE < ";

    timeLastLog = currentTime - delta;

    dateLastLog = new Date(timeLastLog);

    dateString = "'" + simpleDateFormat.format(dateLastLog) + "'";

    deleteString += dateString;

		try {

			System.out.print("Pruning log entries from HARVEST_LOG: ");

			stmt = conn.createStatement();							

			recordsDeleted = stmt.executeUpdate(deleteString);

			System.out.println(recordsDeleted + " records deleted");

			stmt.close();

		}

    catch (SQLException e) {

			System.out.println("SQLException: " + e.getMessage());

		}

  }

  /**

   * Reads the HARVEST_SITE_SCHEDULE table in the database, creating

   * a HarvestSiteSchedule object for each row in the table.

   */

  private void readHarvestSiteSchedule() {

    HarvestSiteSchedule harvestSiteSchedule;

    ResultSet rs;

    SQLWarning warn;

    Statement stmt;

    String contactEmail;

    String dateLastHarvest;

    String dateNextHarvest;

    String documentListURL;

    String ldapDN;

    String ldapPwd;

    int siteScheduleID;

    String unit;

    int updateFrequency;

    try {

      // Read the HARVEST_SITE_SCHEDULE table

      stmt = conn.createStatement();

      rs = stmt.executeQuery("SELECT * FROM HARVEST_SITE_SCHEDULE");

      warn = rs.getWarnings();

      if (warn != null) {

        System.out.println("\n---Warning---\n");

        while (warn != null) {

          System.out.println("Message: " + warn.getMessage());

          System.out.println("SQLState: " + warn.getSQLState());

          System.out.print("Vendor error code: ");

          System.out.println(warn.getErrorCode());

          System.out.println("");

          warn = warn.getNextWarning();

        }

      }

      while (rs.next()) {

        siteScheduleID = rs.getInt("SITE_SCHEDULE_ID");

        documentListURL = rs.getString("DOCUMENTLISTURL");

        ldapDN = rs.getString("LDAPDN");

        ldapPwd = rs.getString("LDAPPWD");

        dateNextHarvest = rs.getString("DATENEXTHARVEST");

        dateLastHarvest = rs.getString("DATELASTHARVEST");

        updateFrequency = rs.getInt("UPDATEFREQUENCY");

        unit = rs.getString("UNIT");

        contactEmail = rs.getString("CONTACT_EMAIL");

        warn = rs.getWarnings();

        if (warn != null) {

          System.out.println("\n---Warning---\n");

          while (warn != null) {

            System.out.println("Message: " + warn.getMessage());

            System.out.println("SQLState: " + warn.getSQLState());

            System.out.print("Vendor error code: ");

            System.out.println(warn.getErrorCode());

            System.out.println("");

            warn = warn.getNextWarning();

          }

        }

        harvestSiteSchedule = new HarvestSiteSchedule(this,

                                                      siteScheduleID,

                                                      documentListURL,

                                                      ldapDN,

                                                      ldapPwd,

                                                      dateNextHarvest,

                                                      dateLastHarvest,

                                                      updateFrequency,

                                                      unit,

                                                      contactEmail

                                                     );

        harvestSiteScheduleList.add(harvestSiteSchedule);

      }

      rs.close();

      stmt.close();

    }

    catch (SQLException e) {

      System.out.println("Database access failed " + e);

      System.exit(1);

    }

  }

  /**

   * Sends a report to the Harvester administrator.

   */

  void reportToAdministrator() {

    System.out.println("\nSending report to administrator.");

  }

  /**

   * Shuts down Harvester. Performs cleanup operations such as logging out

   * of Metacat and disconnecting from the database.

   */

  private void shutdown() {

    // Log shutdown operation

    System.out.println("Shutting Down Harvester");

    addLogEntry(0, "Shutting Down Harvester", "HarvesterShutdown", 0, null, "");

    pruneHarvestLog();

    pruneHarvestDetailLog();

    try {

      // Close the database connection

      System.out.println("Closing the database connection");

      conn.close();

    }

    catch (SQLException e) {

      System.out.println("Database access failed " + e);

    }

    writeHarvestLog();

    reportToAdministrator();

  }

  /**

   * Initializes Harvester at startup. Connects to the database and to Metacat.

   */

  private void startup() {

    Boolean ctm;

    String dbDriver;

    Integer lp;

    String metacatURL;

    String osName = Harvester.isWin32() ? "Windows" : "Unix";

    String password;

    //String response;

    String sessionId;

		String url;

    String user;

    String userName = System.getProperty("user.name");

    SQLWarning warn;

    // Log startup operation

    System.out.println("*****************************************************");

    System.out.println("Starting Up Harvester");

    loadProperties();

    ctm = Boolean.valueOf(properties.getProperty("connectToMetacat", "true"));

    connectToMetacat = ctm.booleanValue();

    dbDriver = properties.getProperty("dbDriver");

    try {

      lp = Integer.valueOf(properties.getProperty("logPeriod", "90"));

      logPeriod = lp.intValue();

    }

    catch (NumberFormatException e) {

      System.err.println("NumberFormatException: Error parsing logPeriod " +

                         logPeriod + e.getMessage());

      System.err.println("Defaulting to logPeriod of 90 days");

      logPeriod = 90;

    }

    metacatURL = properties.getProperty("metacatURL");

    password = properties.getProperty("password");

    url = properties.getProperty("url");

    user = properties.getProperty("user");

    // Load the jdbc driver

    try {

      Class.forName(dbDriver);

    }

    catch (ClassNotFoundException e) {

      System.out.println("Can't load driver " + e);

      System.exit(1);

    } 

    // Make the database connection

    try {

      System.out.println("Getting connection to Harvester tables");

      conn = DriverManager.getConnection(url, user, password);

      // If a SQLWarning object is available, print its warning(s).

      // There may be multiple warnings chained.

      warn = conn.getWarnings();

      if (warn != null) {

        while (warn != null) {

          System.out.println("SQLState: " + warn.getSQLState());

          System.out.println("Message:  " + warn.getMessage());

          System.out.println("Vendor: " + warn.getErrorCode());

          System.out.println("");

          warn = warn.getNextWarning();

        }

      }

    }

    catch (SQLException e) {

      System.out.println("Database access failed " + e);

      System.exit(1);

    }

    initLogIDs();

    harvestStartTime = new Date();

    addLogEntry(0, "Starting Up Harvester", "HarvesterStartup", 0, null, "");

    if (connectToMetacat()) {      

      try {

        System.out.println("Connecting to Metacat: " + metacatURL);

        metacat = MetacatFactory.createMetacatConnection(metacatURL);

      } 

      catch (MetacatInaccessibleException e) {

        System.out.println("Metacat connection failed." + e.getMessage());

      } 

      catch (Exception e) {

        System.out.println("Metacat connection failed." + e.getMessage());

      }

    }

  }

  /**

   * Writes one or more log entries to the HARVEST_LOG table.

   */

  private void writeHarvestLog() {

    HarvestLog harvestLog;

    for (int i = 0; i < harvestLogList.size(); i++) {

      harvestLog = (HarvestLog) harvestLogList.get(i);

      harvestLog.printOutput();

    }

  }

}