Project

General

Profile

« Previous | Next » 

Revision 7187

Added by ben leinfelder almost 12 years ago

Do not loadAllKeys() for SystemMetadataMap when Metacat first starts up. hzIdentifiers will be populated with a simple SQL statement rather than the serial loading of every single SystemMetadata object. It will remain in synch using the usual entryXXX() methods as before.
This should save us resources where we were previously attempting to load ALL SystemMetadata into memory on startup.

View differences:

SystemMetadataMap.java
3 3 
import java.sql.SQLException;
4 4 
import java.util.Collection;
5 5 
import java.util.HashMap;
6 
import java.util.List;
7 6 
import java.util.Map;
8 7 
import java.util.Set;
9 8 

  		




  10
  9
  
    
import org.apache.log4j.Logger;


  		




  11
  10
  
    
import org.dataone.service.exceptions.InvalidSystemMetadata;


  		




  12
  11
  
    
import org.dataone.service.types.v1.Identifier;


  		




  13
  
  
    
import org.dataone.service.types.v1.ObjectInfo;


  		




  14
  
  
    
import org.dataone.service.types.v1.ObjectList;


  		




  15
  12
  
    
import org.dataone.service.types.v1.SystemMetadata;


  		




  16
  13
  
    

  		




  17
  
  
    
import com.hazelcast.core.Hazelcast;


  		




  18
  14
  
    
import com.hazelcast.core.MapLoader;


  		




  19
  15
  
    
import com.hazelcast.core.MapStore;


  		




  20
  16
  
    

  		




  21
  17
  
    
import edu.ucsb.nceas.metacat.IdentifierManager;


  		




  22
  18
  
    
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;


  		




  23
  
  
    
import edu.ucsb.nceas.metacat.properties.PropertyService;


  		




  24
  
  
    
import edu.ucsb.nceas.utilities.PropertyNotFoundException;


  		




  25
  19
  
    

  		




  26
  20
  
    
/**


  		




  27
  21
  
    
 * Storage implementation for Hazelcast System Metadata


  		




  ......




  102
  96
  
    
		return map;


  		




  103
  97
  
    
	}


  		




  104
  98
  
    

  		




  
  99
  
    
	/**


  		




  
  100
  
    
	 * Returning null so that no entries are loaded on map initialization


  		




  
  101
  
    
	 * 


  		




  
  102
  
    
	 * @see http://www.hazelcast.com/docs/1.9.4/manual/single_html/#Map


  		




  
  103
  
    
	 * 


  		




  
  104
  
    
	 * As of 1.9.3 MapLoader has the new MapLoader.loadAllKeys API. 


  		




  
  105
  
    
	 * It is used for pre-populating the in-memory map when the map is first touched/used. 


  		




  
  106
  
    
	 * If MapLoader.loadAllKeys returns NULL then nothing will be loaded. 


  		




  
  107
  
    
	 * Your MapLoader.loadAllKeys implementation can return all or some of the keys. 


  		




  
  108
  
    
	 * You may select and return only the hot keys, for instance. 


  		




  
  109
  
    
	 * Also note that this is the fastest way of pre-populating the map as 


  		




  
  110
  
    
	 * Hazelcast will optimize the loading process by having each node loading owned portion of the entries.


  		




  
  111
  
    
	 */


  		




  105
  112
  
    
	@Override


  		




  106
  113
  
    
	public Set<Identifier> loadAllKeys() {


  		




  107
  
  
    
	  


  		




  108
  
  
    
		String identifiersSet;


  		




  109
  
  
    
		try {


  		




  110
  
  
    
			identifiersSet = 


  		




  111
  
  
    
				PropertyService.getProperty("dataone.hazelcast.storageCluster.identifiersSet");


  		




  112
  
  
    
		} catch (PropertyNotFoundException e1) {


  		




  113
  
  
    
		    identifiersSet = "hzIdentifiers";


  		




  114
  
  
    
		}


  		




  115
  
  
    
		


  		




  116
  
  
    
		Set<Identifier> pids = Hazelcast.getSet(identifiersSet);


  		




  117
  
  
    
		


  		




  118
  
  
    
		try {


  		




  119
  
  
    
			


  		




  120
  
  
    
			// this has more overhead than just looking at the GUIDs


  		




  121
  
  
    
			ObjectList ol = IdentifierManager.getInstance().querySystemMetadata(


  		




  122
  
  
    
					null, //startTime, 


  		




  123
  
  
    
					null, //endTime, 


  		




  124
  
  
    
					null, //objectFormatId, 


  		




  125
  
  
    
					false, //replicaStatus, 


  		




  126
  
  
    
					0, //start, 


  		




  127
  
  
    
					-1 //count


  		




  128
  
  
    
					);


  		




  129
  
  
    
			for (ObjectInfo o: ol.getObjectInfoList()) {


  		




  130
  
  
    
				Identifier pid = o.getIdentifier();


  		




  131
  
  
    
				if ( !pids.contains(pid) ) {


  		




  132
  
  
    
					pids.add(pid);


  		




  133
  
  
    
				}				


  		




  134
  
  
    
			}


  		




  135
  
  
    
			


  		




  136
  
  
    
			// ALTERNATIVE method: look up all the Identifiers from the table


  		




  137
  
  
    
//			List<String> guids = IdentifierManager.getInstance().getAllGUIDs();


  		




  138
  
  
    
//			for (String guid: guids){


  		




  139
  
  
    
//				Identifier pid = new Identifier();


  		




  140
  
  
    
//				pid.setValue(guid);


  		




  141
  
  
    
//				pids.add(pid);


  		




  142
  
  
    
//			}


  		




  143
  
  
    
			


  		




  144
  
  
    
		} catch (Exception e) {


  		




  145
  
  
    
			throw new RuntimeException(e.getMessage(), e);


  		




  146
  
  
    
			


  		




  147
  
  
    
		}


  		




  148
  
  
    
		


  		




  149
  
  
    
		return pids;


  		




  
  114
  
    
		return null;


  		




  150
  115
  
    
	}


  		




  151
  116
  
    

  		




  152
  117
  
    
}


  		












Also available in:  Unified diff