Project

General

Profile

« Previous | Next » 

Revision 4999

Added by Duane Costa over 15 years ago

Implement fix for Bug #4245: Harvester command line scripts don't execute.

View differences:

test/edu/ucsb/nceas/metacattest/harvesterClient/HarvestDetailLogTest.java
79 79 
   */
80 80 
  protected void setUp() {
81 81 
    Connection conn;
82 
    boolean commandLineMode = true;
82 83 
    String contactEmail = "jdoe@institution.edu";
83 84 
    String dateLastHarvest = "2004-04-01 00:00:00.0";
84 85 
    String dateNextHarvest = "2004-05-01 00:00:00.0";
......
101 102 
    int updateFrequency = 1;
102 103
103 104 
    harvester = new Harvester();
104 
    Harvester.loadProperties(test);
105 
    Harvester.loadProperties(commandLineMode, test);
105 106 
    conn = harvester.getConnection();  // initializes the database connection
106 107 
    harvester.initLogIDs();
107 108 
    harvestLogID = harvester.getHarvestLogID();
test/edu/ucsb/nceas/metacattest/harvesterClient/HarvesterTest.java
107 107 
	 * Tests loading of Harvester properties from a configuration file.
108 108 
	 */
109 109 
	public void testLoadProperties() {
110 
	    boolean commandLineMode = true;
110 111 
		String ctm = null;
111 112 
		boolean test = true;
112 113 

  		




  113
  
  
    
		Harvester.loadProperties(test);


  		




  
  114
  
    
		Harvester.loadProperties(commandLineMode, test);


  		




  114
  115
  
    
		try {


  		




  115
  116
  
    
			ctm = PropertyService.getProperty("harvester.connectToMetacat");


  		




  116
  117
  
    
		} catch (PropertyNotFoundException pnfe) {


  		












  

    
      test/edu/ucsb/nceas/metacattest/harvesterClient/HarvestDocumentTest.java
    
  





  77
  77
  
    
   * objects.


  		




  78
  78
  
    
   */


  		




  79
  79
  
    
  protected void setUp() {


  		




  
  80
  
    
    boolean commandLineMode = true;


  		




  80
  81
  
    
    String contactEmail = "jdoe@institution.edu";


  		




  81
  82
  
    
    String dateLastHarvest = "2004-04-01 00:00:00.0";


  		




  82
  83
  
    
    String dateNextHarvest = "2004-05-01 00:00:00.0";


  		




  ......




  103
  104
  
    
    int updateFrequency = 1;


  		




  104
  105
  
    

  		




  105
  106
  
    
    harvester = new Harvester();


  		




  106
  
  
    
    Harvester.loadProperties(test);


  		




  
  107
  
    
    Harvester.loadProperties(commandLineMode, test);


  		




  107
  108
  
    
    harvester.getConnection();  // initializes the database connection


  		




  108
  109
  
    
    harvester.initLogIDs();


  		




  109
  110
  
    
    harvester.setHarvestStartTime(new Date());


  		












  

    
      test/edu/ucsb/nceas/metacattest/harvesterClient/HarvestLogTest.java
    
  





  83
  83
  
    
   * HarvestLog that does have an associated HarvestDetailLog.


  		




  84
  84
  
    
   */


  		




  85
  85
  
    
  protected void setUp() {


  		




  
  86
  
    
    boolean commandLineMode = true;


  		




  86
  87
  
    
    String contactEmail = "jdoe@institution.edu";


  		




  87
  88
  
    
    String dateLastHarvest = "2004-04-01 00:00:00.0";


  		




  88
  89
  
    
    String dateNextHarvest = "2004-05-01 00:00:00.0";


  		




  ......




  110
  111
  
    
    int updateFrequency = 1;


  		




  111
  112
  
    

  		




  112
  113
  
    
    harvester = new Harvester();


  		




  113
  
  
    
    Harvester.loadProperties(test);


  		




  
  114
  
    
    Harvester.loadProperties(commandLineMode, test);


  		




  114
  115
  
    
    conn = harvester.getConnection();  // initializes the database connection


  		




  115
  116
  
    
    harvester.initLogIDs();


  		




  116
  117
  
    
    harvestLogID = harvester.getHarvestLogID();


  		












  

    
      test/edu/ucsb/nceas/metacattest/harvesterClient/HarvestSiteScheduleTest.java
    
  





  79
  79
  
    
   * with a past date of next harvest (is due for harvest now).


  		




  80
  80
  
    
   */


  		




  81
  81
  
    
  protected void setUp() {


  		




  
  82
  
    
    boolean commandLineMode = true;


  		




  82
  83
  
    
    String contactEmail = "jdoe@institution.edu";


  		




  83
  84
  
    
    String dateLastHarvest = "2004-04-01 00:00:00.0";


  		




  84
  85
  
    
    String dateNextHarvestFuture = "2056-01-01 00:00:00.0";


  		




  ......




  94
  95
  
    
    int updateFrequency = 1;


  		




  95
  96
  
    

  		




  96
  97
  
    
    harvester = new Harvester();


  		




  97
  
  
    
    Harvester.loadProperties(test);


  		




  
  98
  
    
    Harvester.loadProperties(commandLineMode, test);


  		




  98
  99
  
    
    harvester.getConnection();  // initializes the database connection


  		




  99
  100
  
    
    harvester.initLogIDs();


  		




  100
  101
  
    
    harvester.setHarvestStartTime(new Date());


  		












  

    
      lib/harvester/runHarvester.sh
    
  





  1
  1
  
    
METACAT_CLASSES=$METACAT_HOME/build/classes


  		




  2
  2
  
    
METACAT_LIB=$METACAT_HOME/lib


  		




  3
  
  
    
JDBC=$METACAT_HOME/build/war/lib/jdbc.jar


  		




  4
  
  
    
LIB_JARS=$METACAT_LIB/cos.jar:$METACAT_LIB/xercesImpl.jar:$METACAT_LIB/utilities.jar


  		




  
  3
  
    
JDBC=$METACAT_HOME/build/war/lib/jdbc.jar:$METACAT_LIB/postgresql-8.0-312.jdbc3.jar


  		




  
  4
  
    
LIB_JARS=$METACAT_LIB/cos.jar:$METACAT_LIB/xercesImpl.jar:$METACAT_LIB/utilities.jar:$METACAT_LIB/log4j-1.2.12.jar:$METACAT_LIB/xalan.jar


  		




  5
  5
  
    
export CLASSPATH=$METACAT_CLASSES:$JDBC:$LIB_JARS


  		




  6
  6
  
    
cd $METACAT_CLASSES


  		




  7
  7
  
    
java edu.ucsb.nceas.metacat.harvesterClient.Harvester


  		












  

    
      lib/harvester/runHarvester.bat
    
  





  1
  1
  
    
echo off


  		




  2
  2
  
    
set METACAT_CLASSES=%METACAT_HOME%\build\classes


  		




  3
  3
  
    
set METACAT_LIB=%METACAT_HOME%\lib


  		




  4
  
  
    
set JDBC=%METACAT_HOME%\build\war\lib\jdbc.jar


  		




  5
  
  
    
set LIB_JARS=%METACAT_LIB%\cos.jar;%METACAT_LIB%\xercesImpl.jar;%METACAT_LIB%\utilities.jar


  		




  
  4
  
    
set LOG4J_PATH=%METACAT_HOME%\build\war\WEB-INF\log4j.properties


  		




  
  5
  
    
set JDBC=%METACAT_HOME%\build\war\lib\jdbc.jar;%METACAT_LIB%\postgresql-8.0-312.jdbc3.jar


  		




  
  6
  
    
set LIB_JARS=%METACAT_LIB%\cos.jar;%METACAT_LIB%\xercesImpl.jar;%METACAT_LIB%\utilities.jar;%METACAT_LIB%\log4j-1.2.12.jar;%METACAT_LIB%\xalan.jar


  		




  6
  7
  
    
set CLASSPATH=%METACAT_CLASSES%;%JDBC%;%LIB_JARS%


  		




  7
  8
  
    
cd %METACAT_CLASSES%


  		




  8
  
  
    
java edu.ucsb.nceas.metacat.harvesterClient.Harvester


  		




  
  9
  
    
java -Dlog4j.configuration=%LOG4J_PATH% edu.ucsb.nceas.metacat.harvesterClient.Harvester


  		












  

    
      src/edu/ucsb/nceas/metacat/harvesterClient/Harvester.java
    
  





  37
  37
  
    
import java.text.SimpleDateFormat;


  		




  38
  38
  
    
import java.util.Date;


  		




  39
  39
  
    

  		




  
  40
  
    
import org.apache.log4j.Logger;


  		




  
  41
  
    
import org.apache.log4j.PropertyConfigurator;


  		




  40
  42
  
    

  		




  41
  43
  
    
import edu.ucsb.nceas.metacat.client.Metacat;


  		




  42
  44
  
    
import edu.ucsb.nceas.metacat.client.MetacatFactory;


  		




  ......




  61
  63
  
    
  private static final String CONFIG_DIR = "../../build/war/WEB-INF";


  		




  62
  64
  
    
  private static final String CONFIG_DIR_TEST = "./build/war/WEB-INF";


  		




  63
  65
  
    
//  private static final String CONFIG_NAME = "metacat.properties";


  		




  
  66
  
    
  private static final String LOG_CONFIG_NAME = "../../build/war/WEB-INF/log4j.properties";


  		




  64
  67
  
    
  public static final String filler = "*";


  		




  65
  68
  
    
  private static boolean keepRunning = true;


  		




  66
  69
  
    
  public static final String marker =


  		




  ......




  84
  87
  
    
  /**


  		




  85
  88
  
    
   * Loads Harvester options from a configuration file.


  		




  86
  89
  
    
   */


  		




  87
  
  
    
  public static void loadProperties(boolean test) {


  		




  
  90
  
    
  public static void loadProperties(boolean commandLineMode, boolean test) {


  		




  88
  91
  
    
    String configDir = test ? CONFIG_DIR_TEST : CONFIG_DIR;    


  		




  89
  92
  
    

  		




  90
  93
  
    
    try {


  		




  91
  
  
    
    	PropertyService.getInstance();


  		




  
  94
  
    
      if (commandLineMode) {


  		




  
  95
  
    
        PropertyService.getTestInstance(configDir);


  		




  
  96
  
    
      }


  		




  
  97
  
    
      else {


  		




  
  98
  
    
        PropertyService.getInstance();


  		




  
  99
  
    
      }


  		




  92
  100
  
    
    } 


  		




  93
  101
  
    
    catch (ServiceException e) {


  		




  94
  102
  
    
      System.out.println("Error in loading properties: " + e.getMessage());


  		




  ......




  99
  107
  
    
    /**


  		




  100
  108
  
    
	 * Harvester main method.


  		




  101
  109
  
    
	 * 


  		




  102
  
  
    
	 * @param args


  		




  103
  
  
    
	 *            the command line arguments


  		




  
  110
  
    
	 * @param args               the command line arguments


  		




  
  111
  
    
	 * 


  		




  
  112
  
    
	 *   args[0] if "false", then this is not command-line mode,


  		




  
  113
  
    
	 *           Command-line mode is true by default.


  		




  
  114
  
    
	 *           


  		




  
  115
  
    
	 *   args[1] if present, represents the path to the harvest list schema file.


  		




  
  116
  
    
	 *           Specifying it overrides the default path to the schema file.


  		




  
  117
  
    
	 *   


  		




  104
  118
  
    
	 * @throws SAXException


  		




  105
  119
  
    
	 * @throws IOException


  		




  106
  120
  
    
	 * @throws ParserConfigurationException


  		




  107
  121
  
    
	 */


  		




  108
  122
  
    
	public static void main(String[] args) {


  		




  109
  
  
    
		Integer delayDefault = new Integer(0); // Default number of hours delay


  		




  
  123
  
    
	    boolean commandLineMode = true;


  		




  
  124
  
    
	    boolean test = false;   // set to true for JUnit testing


  		




  
  125
  
    

  		




  
  126
  
    
	    Integer delayDefault = new Integer(0); // Default number of hours delay


  		




  110
  127
  
    
		int delay = delayDefault.intValue(); // Delay in hours before first


  		




  111
  128
  
    
												// harvest


  		




  112
  129
  
    
		Integer d; // Used for determining delay


  		




  ......




  124
  141
  
    
		int period = periodDefault.intValue(); // Hours between harvests


  		




  125
  142
  
    
		Integer p; // Used in determining the period


  		




  126
  143
  
    
		long startTime; // time that a harvest run starts


  		




  127
  
  
    
		boolean test = false; // Passed to loadOption()


  		




  
  144
  
    
		


  		




  
  145
  
    
		if ((args.length > 0) && (args[0] != null)) {


  		




  
  146
  
    
		  if (args[0].equals("false")) {


  		




  
  147
  
    
		    commandLineMode = false;


  		




  
  148
  
    
		  }


  		




  
  149
  
    
		  else {


  		




  
  150
  
    
		    // If commandLineMode is true, initialize log4j properties


  		




  
  151
  
    
	        PropertyConfigurator.configureAndWatch(LOG_CONFIG_NAME);


  		




  
  152
  
    
		  }


  		




  
  153
  
    
		}


  		




  128
  154
  
    

  		




  129
  
  
    
		if (args.length > 0) {


  		




  130
  
  
    
			schemaLocation = args[0];


  		




  
  155
  
    
		/*


  		




  
  156
  
    
		 * If there is a second argument, it is the schemaLocation value


  		




  
  157
  
    
		 */


  		




  
  158
  
    
		if (args.length > 1) {


  		




  
  159
  
    
			schemaLocation = args[1];


  		




  131
  160
  
    
			System.err.println("schemaLocation: " + schemaLocation);


  		




  132
  161
  
    

  		




  133
  162
  
    
			try {


  		




  ......




  139
  168
  
    

  		




  140
  169
  
    
		System.out.println(marker);


  		




  141
  170
  
    
		System.out.println("Starting Harvester");


  		




  142
  
  
    
		Harvester.loadProperties(test);


  		




  
  171
  
    
		Harvester.loadProperties(commandLineMode, test);


  		




  143
  172
  
    

  		




  144
  173
  
    
		// Parse the delay property. Use default if necessary.


  		




  145
  174
  
    
		try {


  		












  

    
      src/edu/ucsb/nceas/metacat/harvesterClient/HarvesterServlet.java
    
  





  83
  83
  
    
   * 30 seconds to give Metacat a chance to fully initialize.


  		




  84
  84
  
    
   */


  		




  85
  85
  
    
  public void run() {


  		




  86
  
  
    
      String[] args = new String[1];


  		




  87
  
  
    
      args[0] = SCHEMA_LOCATION;


  		




  
  86
  
    
      String[] args = new String[2];


  		




  
  87
  
    
      args[0] = "false";     // Set to true if in command line mode or test mode


  		




  
  88
  
    
      args[1] = SCHEMA_LOCATION;


  		




  88
  89
  
    

  		




  89
  90
  
    
      try {


  		




  90
  91
  
    
        Thread.sleep(30000);    // Sleep 30 seconds before starting Harvester


  		












  

    
      src/edu/ucsb/nceas/metacat/harvesterClient/HarvestSiteSchedule.java
    
  





  337
  337
  
    
                            null,


  		




  338
  338
  
    
                            "");


  		




  339
  339
  
    
      inputStreamReader = new InputStreamReader(inputStream);


  		




  
  340
  
    
      char[] harvestListChars = new char[1024];


  		




  
  341
  
    
      inputStreamReader.read(harvestListChars, 0, 1024);


  		




  
  342
  
    
      System.out.println("documentListURL: " + documentListURL);


  		




  
  343
  
    
      String encoding = inputStreamReader.getEncoding();


  		




  
  344
  
    
      System.out.println("encoding: " + encoding);


  		




  
  345
  
    
      String harvestListStr = new String(harvestListChars);


  		




  
  346
  
    
      System.out.println("harvestListStr:\n" + harvestListStr);


  		




  340
  347
  
    
      documentListHandler.runParser(inputStreamReader, schemaLocation);


  		




  341
  348
  
    
      harvester.addLogEntry(0,


  		




  342
  349
  
    
                            "Validated: " + documentListURL,


  		












Also available in:  Unified diff