Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2004 Regents of the University of California and the

 *              National Center for Ecological Analysis and Synthesis

 *  Purpose: To test the Access Controls in metacat by JUnit

 *

 *   '$Author$'

 *     '$Date$'

 * '$Revision$'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.admin.upgrade.dataone;

import junit.framework.Test;

import junit.framework.TestSuite;

import edu.ucsb.nceas.MCTestCase;

import edu.ucsb.nceas.metacat.admin.upgrade.dataone.GenerateSystemMetadata;

/**

 * A JUnit test for testing system metadata generation

 */

public class GenerateSystemMetadataTest

    extends MCTestCase {

	/**

     * Constructor to build the test

     *

     * @param name the name of the test method

     */

    public GenerateSystemMetadataTest(String name) {

        super(name);

    }

    /**

     * Establish a testing framework by initializing appropriate objects

     */

    public void setUp() {

    }

    /**

     * Release any objects after tests are complete

     */

    public void tearDown() {

    }

    /**

     * Create a suite of tests to be run together

     */

    public static Test suite() {

        TestSuite suite = new TestSuite();

        suite.addTest(new GenerateSystemMetadataTest("initialize"));

        // Test basic functions

        suite.addTest(new GenerateSystemMetadataTest("upgrade"));

        return suite;

    }

    /**

     * Run an initial test that always passes to check that the test

     * harness is working.

     */

    public void initialize() {

        assertTrue(1 == 1);

    }

    public void upgrade() throws Exception {

    	GenerateSystemMetadata upgrader = new GenerateSystemMetadata();

        upgrader.upgrade();

    }

}

package edu.ucsb.nceas.metacat.admin.upgrade.dataone;

/**

 *  '$RCSfile$'

 *    Purpose: A Class for upgrading the database to version 1.5

 *  Copyright: 2000 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Saurabh Garg

 *

 *   '$Author$'

 *     '$Date$'

 * '$Revision$'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

import java.util.Collections;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import edu.ucsb.nceas.metacat.DBUtil;

import edu.ucsb.nceas.metacat.admin.AdminException;

import edu.ucsb.nceas.metacat.admin.upgrade.UpgradeUtilityInterface;

import edu.ucsb.nceas.metacat.dataone.SystemMetadataFactory;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.utilities.SortedProperties;

public class GenerateORE implements UpgradeUtilityInterface {

	private static Log log = LogFactory.getLog(GenerateORE.class);

    public boolean upgrade() throws AdminException {

        boolean success = true;

        // include ORE, data, for this server only

        boolean includeOre = true;

        boolean downloadData = true;

        int serverLocation = 1;

        try {

        	// get only local ids for this server

            List<String> idList = DBUtil.getAllDocidsByType(null, true, serverLocation);

            Collections.sort(idList);

            // generate based on this list

            SystemMetadataFactory.generateSystemMetadata(idList, includeOre, downloadData);

		} catch (Exception e) {

			String msg = "Problem generating missing system metadata: " + e.getMessage();

			log.error(msg, e);

			success = false;

			throw new AdminException(msg);

		}

    	return success;

    }

    public static void main(String [] ags){

        try {

        	// set up the properties based on the test/deployed configuration of the workspace

        	SortedProperties testProperties = 

				new SortedProperties("test/test.properties");

			testProperties.load();

			String metacatContextDir = testProperties.getProperty("metacat.contextDir");

			PropertyService.getInstance(metacatContextDir + "/WEB-INF");

			// now run it

            GenerateORE upgrader = new GenerateORE();

	        upgrader.upgrade();

        } catch (Exception ex) {

            System.out.println("Exception:" + ex.getMessage());

            ex.printStackTrace();

        }

    }

}

package edu.ucsb.nceas.metacat.admin.upgrade.dataone;

/**

 *  '$RCSfile$'

 *    Purpose: A Class for upgrading the database to version 1.5

 *  Copyright: 2000 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Saurabh Garg

 *

 *   '$Author: leinfelder $'

 *     '$Date: 2011-03-29 18:23:38 +0000 (Tue, 29 Mar 2011) $'

 * '$Revision: 6025 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

import java.util.Collections;

import java.util.List;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.TimeUnit;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import edu.ucsb.nceas.metacat.DBUtil;

import edu.ucsb.nceas.metacat.admin.AdminException;

import edu.ucsb.nceas.metacat.admin.upgrade.UpgradeUtilityInterface;

import edu.ucsb.nceas.metacat.dataone.SystemMetadataFactory;

import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.utilities.SortedProperties;

public class GenerateSystemMetadata implements UpgradeUtilityInterface {

	private static Log log = LogFactory.getLog(GenerateSystemMetadata.class);

    public boolean upgrade() throws AdminException {

    	// do this in a thread too so that we don't have to hang the UI (web)

    	ExecutorService executor = Executors.newSingleThreadExecutor();

    	Runnable command = new Runnable() {

			@Override

			public void run() {

				// just run it

				try {

					boolean success = multiThreadUpgrade();

				} catch (AdminException e) {

					throw new RuntimeException(e);

				}

			}

    	};

		executor.execute(command);

		executor.shutdown();

		// wait for it to finish before returning?

        boolean wait = false;

        if (wait) {

            log.debug("Waiting for upgrade to complete");

            try {

				executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS);

			} catch (InterruptedException e) {

				AdminException ae = new AdminException(e.getMessage());

				ae.initCause(e);

				throw ae;

			}

            log.debug("Done waiting for upgrade thread");

        }

    	return true;

        //return singleThreadUpgrade();

    }

    /**

     * Use multiple threads to process parts of the complete ID list concurrently

     * @return

     * @throws AdminException

     */

    public boolean multiThreadUpgrade() throws AdminException {

        boolean success = true;

        // do not include ORE or data, but can generate SystemMetadata for ALL records

        final boolean includeOre = false;

        final boolean downloadData = false;

        int serverLocation = -1;

        try {

        	// get list of ALL docids at ALL server locations

            List<String> idList = DBUtil.getAllDocidsByType(null, true, serverLocation);

            Collections.sort(idList);

            // executor

            int availableProcessors = Runtime.getRuntime().availableProcessors();

            int nThreads = availableProcessors * 1;

            //availableProcessors++;

        	log.debug("Using nThreads: " + nThreads);

            ExecutorService executor = Executors.newFixedThreadPool(nThreads);

            int taskCount = 0;

            // init HZ

            log.debug("Making sure Hazelcast is up");

            HazelcastService.getInstance();

            // chunk into groups

			int fromIndex = 0;

            int toIndex = 0;

            String prefix = null;

            for (String docid: idList) {

            	// increment the next entry, exclusive

            	toIndex++;

            	// use scope.docid (without revision) to determine groups

            	if (prefix == null || !docid.startsWith(prefix)) {

            		// construct a sublist for this previous group of docids

					final List<String> subList = idList.subList(fromIndex, toIndex);

	            	log.debug("Grouping docid prefix: " + prefix);

					log.debug("subList.size: " + subList.size());

					// add the task for this sublist

					Runnable command = new Runnable() {

						@Override

						public void run() {

							// generate based on this list

				            try {

				            	log.debug("Processing subList.size: " + subList.size());

								SystemMetadataFactory.generateSystemMetadata(subList, includeOre, downloadData);

								log.debug("Done processing subList.size: " + subList.size());

							} catch (Exception e) {

								throw new RuntimeException(e);

							}

						}

					};

					// execute the task 

					executor.execute(command);

					taskCount++;

					// start at the end of this sublist

					fromIndex = toIndex;

            	}

            	log.debug("docid: " + docid);

            	// get the previous docid prefix

            	String previousId = docid;

            	prefix = previousId.substring(0, previousId.lastIndexOf("."));

            }

            log.info("done launching threaded tasks, count: " + taskCount);

            // wait for executor to finish

            executor.shutdown();

			// wait a long time

            log.debug("Waiting for all threads to complete");

            executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS);

            log.debug("Done waiting for all threads to complete");

            // now we are ready to be a data one node

            PropertyService.setProperty("dataone.systemmetadata.generated", Boolean.TRUE.toString());

		} catch (Exception e) {

			String msg = "Problem generating missing system metadata: " + e.getMessage();

			log.error(msg, e);

			success = false;

			throw new AdminException(msg);

		}

    	return success;

    }

    public static void main(String [] ags){

        try {

        	// set up the properties based on the test/deployed configuration of the workspace

        	SortedProperties testProperties = 

				new SortedProperties("test/test.properties");

			testProperties.load();

			String metacatContextDir = testProperties.getProperty("metacat.contextDir");

			PropertyService.getInstance(metacatContextDir + "/WEB-INF");

			// now run it

            GenerateSystemMetadata upgrader = new GenerateSystemMetadata();

	        upgrader.upgrade();

        } catch (Exception ex) {

            System.out.println("Exception:" + ex.getMessage());

            ex.printStackTrace();

        }

    }

}

import edu.ucsb.nceas.metacat.admin.upgrade.dataone.GenerateORE;

import edu.ucsb.nceas.metacat.admin.upgrade.dataone.GenerateSystemMetadata;

		<property name="classtorun" value="edu.ucsb.nceas.metacat.admin.upgrade.dataone.GenerateSystemMetadata" />