Metacat

/**

 *  '$RCSfile$'

 *    Purpose: Implements a service for managing a Hazelcast cluster member

 *  Copyright: 2011 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Christopher Jones

 *

 *   '$Author$'

 *     '$Date$'

 * '$Revision$'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

import java.util.concurrent.Executors;

import org.dataone.service.types.v1.Node;

import org.dataone.service.types.v1.NodeReference;

import org.dataone.service.types.v1.SystemMetadata;

import com.hazelcast.core.EntryListener;

import com.hazelcast.core.LifecycleListener;

import com.hazelcast.core.MembershipListener;

import com.hazelcast.partition.PartitionService;

import edu.ucsb.nceas.metacat.McdbDocNotFoundException;

import edu.ucsb.nceas.metacat.shared.ServiceException;

 * The Hazelcast service enables Metacat as a Hazelcast cluster member

 */

public class HazelcastService extends BaseService

  /* The singleton instance of the hazelcast service */

  private Config hzConfig;

  private String systemMetadataMap;

  /* The Hazelcast distributed system metadata map */

  private IMap<Identifier, SystemMetadata> systemMetadata;

  private String identifiersSet;

  /* The Hazelcast distributed identifiers set */

  private ISet<Identifier> identifiers;

  /* The Hazelcast distributed missing identifiers set */

  private ISet<Identifier> missingIdentifiers;

   * Constructor: Creates an instance of the hazelcast service. Since

   * this uses a singleton pattern, use getInstance() to gain the instance.

   */

  private HazelcastService() {

    super();

    _serviceName="HazelcastService";

    try {

    } catch (ServiceException se) {

    }

  }

  /**

   *  Get the instance of the HazelcastService that has been instantiated,

   *  or instantiate one if it has not been already.

   *

   * @return hazelcastService - The instance of the hazelcast service

   */

  public static HazelcastService getInstance(){

    }

  /**

	try {

		configFileName = PropertyService.getProperty("dataone.hazelcast.configFilePath");

		configFileName = PropertyService.CONFIG_FILE_DIR + FileUtil.getFS() + "hazelcast.xml";

		try {

			String msg = e.getMessage();

			logMetacat.error(msg);

			throw new ServiceException(msg);

		}

	}

    try {

      systemMetadataMap =

        PropertyService.getProperty("dataone.hazelcast.storageCluster.systemMetadataMap");

      missingIdentifiers.addItemListener(this, true);

      systemMetadata.addEntryListener(this, true);

      hzInstance.getCluster().addMembershipListener(this);

      hzInstance.getLifecycleService().addLifecycleListener(this);

      String msg = "Couldn't find Hazelcast properties for the DataONE clusters. " +

        "The error message was: " + e.getMessage();

      logMetacat.error(msg);

    // make sure we have all metadata locally

    try {

	}

   * Get the system metadata map

   *

   * @return systemMetadata - the hazelcast map of system metadata

	  return systemMetadata;

  }

  /**

   * @return identifiers - the set of unique DataONE identifiers in the cluster

   */

  public ISet<Identifier> getIdentifiers() {

      return identifiers;

  }

  /**

   * in-memory representation of it.

   * @param guid

   */

  public void refreshSystemMetadataEntry(String guid) {

	Identifier identifier = new Identifier();

	identifier.setValue(guid);

	// force hazelcast to update system metadata in memory from the store

	HazelcastService.getInstance().getSystemMetadataMap().evict(identifier);

    try {

    } catch (RuntimeException e) {

        logMetacat.info("Couldn't get a lock for identifier " +

    return lock;

  }

   * Get the DataONE hazelcast node map

   * @return nodes - the hazelcast map of nodes

   */

//	  return nodes;

//  }

   * Indicate whether or not this service is refreshable.

   *

   * @return refreshable - the boolean refreshable status

   */

  public boolean refreshable() {

    // and start back up again

    return false;

  }

  /**

   * Stop the HazelcastService. When stopped, the service will no longer

   * respond to requests.

   */

  public void stop() throws ServiceException {

  }

      return this.hzInstance;

  }

  /**

   * Refresh the Hazelcast service by restarting it

   */

  /**

	 * Implement the EntryListener interface for Hazelcast, reponding to entry

	 * added events in the hzSystemMetadata map. Evaluate the entry and create

	 * CNReplicationTasks as appropriate (for DATA, METADATA, RESOURCE)

	 *

	 * @param event - The EntryEvent that occurred

	 */

	  logMetacat.info("SystemMetadata entry added event on identifier " +

	      event.getKey().getValue());

		entryUpdated(event);

	 * Implement the EntryListener interface for Hazelcast, reponding to entry

	 * evicted events in the hzSystemMetadata map.  Evaluate the entry and create

	 * CNReplicationTasks as appropriate (for DATA, METADATA, RESOURCE)

	 *

	 * @param event - The EntryEvent that occurred

	 */

      logMetacat.info("SystemMetadata entry evicted event on identifier " +

          event.getKey().getValue());

      if ( !identifiers.contains(event.getKey()) ) {

          identifiers.add(event.getKey());

      }

	}

	/**

	 * Implement the EntryListener interface for Hazelcast, reponding to entry

	 * removed events in the hzSystemMetadata map.  Evaluate the entry and create

	 * CNReplicationTasks as appropriate (for DATA, METADATA, RESOURCE)

	 *

	 * @param event - The EntryEvent that occurred

	 */

    logMetacat.info("SystemMetadata entry removed event on identifier " +

        event.getKey().getValue());

	  // we typically don't remove objects in Metacat, but can remove System Metadata

    // keep the hzIdentifiers set in sync with the systemmetadata table

    if ( identifiers.contains(event.getKey()) ) {

        identifiers.remove(event.getKey());

    }

	/**

	 * Implement the EntryListener interface for Hazelcast, reponding to entry

	 * updated events in the hzSystemMetadata map.  Evaluate the entry and create

	 * CNReplicationTasks as appropriate (for DATA, METADATA, RESOURCE)

	 *

	 * @param event - The EntryEvent that occurred

	 */

		logMetacat.debug("Entry added/updated to System Metadata map: " + event.getKey().getValue());

		Member ownerMember = partition.getOwner();

		SystemMetadata sysmeta = event.getValue();

		if (!ownerMember.localMember()) {

			if (sysmeta == null) {

				logMetacat.warn("No SystemMetadata provided in the event, getting from shared map: " + event.getKey().getValue());

				sysmeta = getSystemMetadataMap().get(event.getKey());

					logMetacat.error("Could not find SystemMetadata in shared map for: " + event.getKey().getValue());

					// TODO: should probably return at this point since the save will fail

			saveLocally(event.getValue());

		}

		if (!identifiers.contains(event.getKey())) {

			identifiers.add(event.getKey());

		}

	/**

	 * Save SystemMetadata to local store if needed

	 * @param sm

	 */

	private void saveLocally(SystemMetadata sm) {

		logMetacat.debug("Saving entry locally: " + sm.getIdentifier().getValue());

		try {

			IdentifierManager.getInstance().insertOrUpdateSystemMetadata(sm);

	        logMetacat.error("Could not save System Metadata to local store.", e);

	    }

	 * Checks the local backing store for missing SystemMetadata,

	 * retrieves those entries from the shared map if they exist,

	 * and saves them locally.

	 */

	private void synchronizeLocalStore() {

			logMetacat.debug("Member missing SystemMetadata entries, count = " + localIds.size());

			for (String localId: localIds) {

				logMetacat.debug("Processing system metadata for localId: " + localId);

				try {

					String docid = DocumentUtil.getSmartDocId(localId);

					int rev = DocumentUtil.getRevisionFromAccessionNumber(localId);

					String guid = IdentifierManager.getInstance().getGUID(docid, rev);

					logMetacat.debug("Found mapped guid: " + guid);

					Identifier pid = new Identifier();

					pid.setValue(guid);

					SystemMetadata sm = systemMetadata.get(pid);

					logMetacat.debug("Found shared system metadata for guid: " + guid);

					saveLocally(sm);

					logMetacat.debug("Saved shared system metadata locally for guid: " + guid);

				} catch (Exception e) {

					logMetacat.error("Could not save shared SystemMetadata entry locally, localId: " + localId, e);

				}

			}

		}

	}

	 * Make sure we have a copy of every entry in the shared map.

	 * We use lazy loading and therefore the CNs may not all be in sync when one

	 * comes back online after an extended period of being offline

	 * This method loops through the entries that a FULLY UP-TO-DATE CN has

	 * and makes sure each one is present on the shared map.

	 * It is meant to overcome a HZ weakness wherein ownership of a key results in

	 * null values where the owner does not have a complete backing store.

	 * This will be an expensive routine and should be run in a background process so that

	 * the server can continue to service other requests during the synch

	 * @throws Exception

	 */

		Set<Identifier> localIdKeys = loadAllKeys();

		//  the PIDs missing locally

		Set<Identifier> missingIdKeys = new HashSet<Identifier>();

			Identifier pid = idIter.next();

			} else {

				// we don't have this locally, so we should try to get it

				missingIdKeys.add(pid);

				logMetacat.debug("Adding missing hzIdentifiers key: " + key.getValue());

				identifiers.add(key);

			}

		}

		Iterator<Identifier> missingPids = missingIdKeys.iterator();

		while (missingPids.hasNext()) {

			Identifier pid = missingPids.next();

			logMetacat.debug("Publishing missing pid to wanted list: " + pid.getValue());

			missingIdentifiers.add(pid);

		executor.execute(new Runnable() {

			@Override

			public void run() {

				try {

					resynchToRemote();

					logMetacat.error("Error in resynchInThread: " + e.getMessage(), e);

				}

			}

		});

		executor.shutdown();

	}

	 * When there is missing SystemMetadata on the local member,

	 * we retrieve it from the shared map and add it to the local

	 * backing store for safe keeping.

	 */

	@Override

	public void memberAdded(MembershipEvent event) {

		Member member = event.getMember();

		logMetacat.debug("Member added to cluster: " + member.getInetSocketAddress());

		boolean isLocal = member.localMember();

		if (isLocal) {

			logMetacat.debug("Member islocal: " + member.getInetSocketAddress());

	}

	@Override

	public void memberRemoved(MembershipEvent event) {

		// TODO Auto-generated method stub

	}

	 * In cases where this cluster is paused, we want to

	 * check that the local store accurately reflects the shared

	 * SystemMetadata map

	 * @param event

	 */

	@Override

	public void stateChanged(LifecycleEvent event) {

		logMetacat.debug("HZ LifecycleEvent.state: " + event.getState());

		if (event.getState().equals(LifecycleEvent.LifecycleState.RESUMED)) {

			logMetacat.debug("HZ LifecycleEvent.state is RESUMED, calling synchronizeLocalStore()");

			synchronizeLocalStore();

		}

	}

	 * Load all System Metadata keys from the backing store

	 * @return set of pids

	 */

	private Set<Identifier> loadAllKeys() {

		Set<Identifier> pids = new HashSet<Identifier>();

		try {

			// ALTERNATIVE 1: this has more overhead than just looking at the GUIDs

//			ObjectList ol = IdentifierManager.getInstance().querySystemMetadata(

//					null, //startTime,

//					null, //endTime,

//					null, //objectFormatId,

//					false, //replicaStatus,

//					0, //start,

//					-1 //count

//					);

//			for (ObjectInfo o: ol.getObjectInfoList()) {

//				Identifier pid = o.getIdentifier();

//				if ( !pids.contains(pid) ) {

//					pids.add(pid);

//				}

//			}

			// ALTERNATIVE method: look up all the Identifiers from the table

			List<String> guids = IdentifierManager.getInstance().getAllSystemMetadataGUIDs();

				Identifier pid = new Identifier();

				pid.setValue(guid);

				pids.add(pid);

			}

		} catch (Exception e) {

			throw new RuntimeException(e.getMessage(), e);

		}

		return pids;

	}

	 * Respond to itemAdded events on the hzMissingIdentifiers Set.  Uses a

	 * distributed ILock to try to prevent multiple put calls on hzSystemMetadata

	 *

	 * @param pid   the identifier of the event

	 */

		Identifier pid = (Identifier) event.getItem();

		logMetacat.debug("Responding to itemAdded for pid: " + pid.getValue());

		// lock this event, only if we have a local copy to contribute

			SystemMetadata sm = IdentifierManager.getInstance().getSystemMetadata(pid.getValue());

			if (sm != null) {

				lock = hzInstance.getLock(MISSING_PID_PREFIX + pid.getValue());

				if ( lock.tryLock() ) {

			        // "publish" the system metadata to the shared map since it showed up on the missing queue

			        logMetacat.debug("Adding SystemMetadata to shared map for pid: " + pid.getValue());

			        systemMetadata.put(pid, sm);

			        // remove the entry since we processed it

			        missingIdentifiers.remove(pid);

				  } else {

				      logMetacat.debug(MISSING_PID_PREFIX + pid.getValue() + " was already locked. Skipping.");

				  }

			} else {

				// can't help here

				logMetacat.warn("Local system metadata not found for pid: " + pid.getValue());

			}

			logMetacat.error("Error looking up missing system metadata for pid: " + pid.getValue());

				lock.unlock();

			}

   * Respond to itemRemoved events on the hzMissingIdentifiers Set

   *

   * @param pid   the identifier of the event

   */

	}