/ - Diff - Metacat - Ecoinformatics Redmine

« Previous | Next »

Revision 7187

Added by ben leinfelder over 12 years ago

Do not loadAllKeys() for SystemMetadataMap when Metacat first starts up. hzIdentifiers will be populated with a simple SQL statement rather than the serial loading of every single SystemMetadata object. It will remain in synch using the usual entryXXX() methods as before.
This should save us resources where we were previously attempting to load ALL SystemMetadata into memory on startup.

     import java.io.FileNotFoundException;
     import java.sql.SQLException;
     import java.util.ArrayList;
     import java.util.Collection;
     import java.util.Date;
     import java.util.HashSet;
     import java.util.List;
     import java.util.Set;
     import java.util.concurrent.locks.Lock;
     import org.apache.log4j.Logger;
-...
     import org.dataone.service.types.v1.Identifier;
     import org.dataone.service.types.v1.Node;
     import org.dataone.service.types.v1.NodeReference;
     import org.dataone.service.types.v1.ObjectInfo;
     import org.dataone.service.types.v1.ObjectList;
     import org.dataone.service.types.v1.SystemMetadata;
     import com.hazelcast.config.Config;
-...
     import edu.ucsb.nceas.metacat.IdentifierManager;
     import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
     import edu.ucsb.nceas.metacat.dataone.D1NodeService;
     import edu.ucsb.nceas.metacat.properties.PropertyService;
     import edu.ucsb.nceas.metacat.shared.BaseService;
     import edu.ucsb.nceas.metacat.shared.ServiceException;
-...
           // Get a reference to the shared identifiers set as a cluster member
           identifiers = Hazelcast.getSet(identifiersSet);
           identifiers.addAll(loadAllKeys());
           // Listen for changes to the system metadata map
           systemMetadata.addEntryListener(this, true);
-...
+    		}
+    	}
     	/**
     	 * Load all System Metadata keys from the backing store
     	 * @return set of pids
     	 */
     	private Set<Identifier> loadAllKeys() {
     		Set<Identifier> pids = new HashSet<Identifier>();
     		try {
     			// ALTERNATIVE 1: this has more overhead than just looking at the GUIDs
     //			ObjectList ol = IdentifierManager.getInstance().querySystemMetadata(
     //					null, //startTime,
     //					null, //endTime,
     //					null, //objectFormatId,
     //					false, //replicaStatus,
     //					0, //start,
     //					-1 //count
     //					);
     //			for (ObjectInfo o: ol.getObjectInfoList()) {
     //				Identifier pid = o.getIdentifier();
     //				if ( !pids.contains(pid) ) {
     //					pids.add(pid);
     //				}
     //			}
     			// ALTERNATIVE method: look up all the Identifiers from the table
     			List<String> guids = IdentifierManager.getInstance().getAllSystemMetadataGUIDs();
     			for (String guid: guids){
     				Identifier pid = new Identifier();
     				pid.setValue(guid);
     				pids.add(pid);
+    			}
     		} catch (Exception e) {
     			throw new RuntimeException(e.getMessage(), e);
+    		}
     		return pids;
+    	}
+    }

     import java.sql.SQLException;
     import java.util.Collection;
     import java.util.HashMap;
     import java.util.List;
     import java.util.Map;
     import java.util.Set;
     import org.apache.log4j.Logger;
     import org.dataone.service.exceptions.InvalidSystemMetadata;
     import org.dataone.service.types.v1.Identifier;
     import org.dataone.service.types.v1.ObjectInfo;
     import org.dataone.service.types.v1.ObjectList;
     import org.dataone.service.types.v1.SystemMetadata;
     import com.hazelcast.core.Hazelcast;
     import com.hazelcast.core.MapLoader;
     import com.hazelcast.core.MapStore;
     import edu.ucsb.nceas.metacat.IdentifierManager;
     import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
     import edu.ucsb.nceas.metacat.properties.PropertyService;
     import edu.ucsb.nceas.utilities.PropertyNotFoundException;
     /**
      * Storage implementation for Hazelcast System Metadata
-...
     		return map;
+    	}
     	/**
     	 * Returning null so that no entries are loaded on map initialization
+    	 *
     	 * @see http://www.hazelcast.com/docs/1.9.4/manual/single_html/#Map
+    	 *
     	 * As of 1.9.3 MapLoader has the new MapLoader.loadAllKeys API.
     	 * It is used for pre-populating the in-memory map when the map is first touched/used.
     	 * If MapLoader.loadAllKeys returns NULL then nothing will be loaded.
     	 * Your MapLoader.loadAllKeys implementation can return all or some of the keys.
     	 * You may select and return only the hot keys, for instance.
     	 * Also note that this is the fastest way of pre-populating the map as
     	 * Hazelcast will optimize the loading process by having each node loading owned portion of the entries.
     	 */
     	@Override
     	public Set<Identifier> loadAllKeys() {
     		String identifiersSet;
     		try {
     			identifiersSet =
     				PropertyService.getProperty("dataone.hazelcast.storageCluster.identifiersSet");
     		} catch (PropertyNotFoundException e1) {
     		    identifiersSet = "hzIdentifiers";
+    		}
     		Set<Identifier> pids = Hazelcast.getSet(identifiersSet);
     		try {
     			// this has more overhead than just looking at the GUIDs
     			ObjectList ol = IdentifierManager.getInstance().querySystemMetadata(
     					null, //startTime,
     					null, //endTime,
     					null, //objectFormatId,
     					false, //replicaStatus,
 , //start,
     					-1 //count
     					);
     			for (ObjectInfo o: ol.getObjectInfoList()) {
     				Identifier pid = o.getIdentifier();
     				if ( !pids.contains(pid) ) {
     					pids.add(pid);
+    				}
+    			}
     			// ALTERNATIVE method: look up all the Identifiers from the table
     //			List<String> guids = IdentifierManager.getInstance().getAllGUIDs();
     //			for (String guid: guids){
     //				Identifier pid = new Identifier();
     //				pid.setValue(guid);
     //				pids.add(pid);
     //			}
     		} catch (Exception e) {
     			throw new RuntimeException(e.getMessage(), e);
+    		}
     		return pids;
     		return null;
+    	}
+    }

          * return a listing of all guids in the object store
          * @return a list of all GUIDs in metacat
          */
         public List<String> getAllGUIDs()
         public List<String> getAllSystemMetadataGUIDs()
+        {
             Vector<String> guids = new Vector<String>();
             String sql = "select guid from identifier";
             String sql = "select guid from systemmetadata";
             DBConnection dbConn = null;
             int serialNumber = -1;
             try

Also available in: Unified diff

Project

General

Profile

Metacat

Revision 7187

Added by ben leinfelder over 12 years ago