Project

General

Profile

Revision 6420

Added by rnahf over 9 years ago

further refactoring and start of unit tests for hazelcast elements

View differences:

src/edu/ucsb/nceas/metacat/dataone/hazelcast/ObjectPathMapLoader.java
1
package edu.ucsb.nceas.metacat.dataone.hazelcast;
2

  
3
import java.util.Collection;
4
import java.util.Collections;
5
import java.util.HashSet;
6
import java.util.Hashtable;
7
import java.util.List;
8
import java.util.Map;
9
import java.util.Set;
10

  
11
import org.dataone.service.types.v1.Identifier;
12

  
13
import com.hazelcast.core.MapLoader;
14

  
15
import edu.ucsb.nceas.metacat.IdentifierManager;
16
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
17
import edu.ucsb.nceas.metacat.properties.PropertyService;
18
import edu.ucsb.nceas.utilities.FileUtil;
19
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
20

  
21

  
22
/**
23
 * MapLoader implementation for a hazelcast ObjectPath.  This class is called
24
 * when the ObjectPathMap needs to refresh against the persistent data-store.
25
 * 
26
 * @author rnahf
27
 *
28
 */
29
public class ObjectPathMapLoader implements MapLoader<Identifier, String> {
30
	private static IdentifierManager im;
31
	private static String dataPath;
32
	private static String metadataPath;
33

  
34

  
35
	
36
	/**
37
	 * creates an ObjectPathMapLoader
38
	 */
39
	public ObjectPathMapLoader() {
40
		try {
41
			dataPath = PropertyService.getProperty("application.datafilepath");
42
			metadataPath = PropertyService.getProperty("application.documentfilepath");
43
		} catch (PropertyNotFoundException e) {
44
			// TODO Auto-generated catch block
45
			e.printStackTrace();
46
		}
47
		im = IdentifierManager.getInstance();
48
	}
49

  
50
	
51
	/*
52
	 * Metadata is stored in a different place on the filesystem than
53
	 * the data.  The doctype value for metadata can vary, but for data
54
	 * is always 'BIN', so using a simple if-then-else to separate
55
	 */
56
	private String pathToDocid(String localid) throws McdbDocNotFoundException  {
57
		
58
		Hashtable<String, Object> ht = im.getDocumentInfo(localid);
59
		if (ht.get("doctype").equals("BIN")) {
60
			return dataPath + FileUtil.getFS() + localid;
61
		} else {
62
			return metadataPath + FileUtil.getFS() + localid;
63
		}		
64
	}
65

  
66
	
67
	/**
68
	 *  Implementation of hazelcast MapLoader interface method.
69
	 *  For the provided Identifier (as key), returns the path to the
70
	 *  document on the local filesystem.  Returns null if it can't 
71
	 *  create the path. 
72
	 */
73
	@Override
74
	public String load(Identifier key) 
75
	{
76
		String docid = null;
77
		String path = null;
78
		try {
79
			docid = im.getLocalId(key.getValue());
80
			path = pathToDocid(docid);
81
		} catch (McdbDocNotFoundException e) {
82
			// TODO Auto-generated catch block
83
			e.printStackTrace();
84
			return null;
85
		}
86
		return path;
87
	}
88
	
89
	
90
	/**
91
	 *  Implementation of hazelcast MapLoader interface method.  This method loads
92
	 *  mappings for all Identifiers in the parameters.  Any Identifier not found
93
	 *  is not included in the resulting map.
94
	 */
95
	@Override
96
	public Map<Identifier, String> loadAll(Collection<Identifier> identifiers) {
97
		
98
		Hashtable<Identifier,String> map = new Hashtable<Identifier,String>();
99
		for (Identifier id : identifiers) {
100
			try {
101
				String docid = im.getLocalId(id.getValue());
102
				map.put(id, pathToDocid(docid));
103
				
104
			} catch (McdbDocNotFoundException e) {
105
				// TODO should the map load an empty path instead of
106
				// leaving out the entire entry?
107
				e.printStackTrace();
108
			}
109
		}
110
		return map;
111
	}
112

  
113
	
114
	/**
115
	 * Return the full set of guids in the local metacat repository as
116
	 * dataone Identifiers.
117
	 * 
118
	 * (Hazelcast allows avoiding pre-loading by returning NULL, so consider
119
	 * re-implementing if that serves the purpose of this class better)
120
	 */
121
	@Override
122
	public Set<Identifier> loadAllKeys() {
123
		
124
		List<String> guids = im.getAllGUIDs();
125
		
126
		Set<Identifier> set = Collections.synchronizedSet(new HashSet<Identifier>());
127
		for (String guid : guids) {
128
			Identifier id = new Identifier();
129
			id.setValue(guid);
130
			set.add(id);
131
		}
132
		return set;
133
	}
134
}
test/edu/ucsb/nceas/metacat/dataone/HzObjectPathMapTest.java
1
package edu.ucsb.nceas.metacat.dataone;
2

  
3
import java.util.Map;
4

  
5
import org.junit.Test;
6

  
7
import com.hazelcast.core.Hazelcast;
8
import com.hazelcast.core.HazelcastInstance;
9

  
10
import edu.ucsb.nceas.MCTestCase;
11

  
12

  
13
public class HzObjectPathMapTest extends MCTestCase {
14

  
15
	
16
	@Test
17
    public void testTwoMemberMapSizes() {
18
		
19
		System.getProperties().setProperty("hazelcast.config",
20
				"/Users/rnahf/projects/nceas/metacat/" +
21
				"test/edu/ucsb/nceas/metacat/dataone/hazelcast.test.properties.xml");
22
        // start the first member
23
        HazelcastInstance h1 = Hazelcast.newHazelcastInstance(null);
24
        // get the map and put 1000 entries
25
        Map map1 = h1.getMap("hzObjectPath");
26
 
27
        for (int i = 0; i < 1000; i++) {
28
            map1.put(i, "value" + i);
29
        }
30
        // check the map size
31
        assertEquals(1000, map1.size());
32
        // start the second member
33
        HazelcastInstance h2 = Hazelcast.newHazelcastInstance(null);
34
        // get the same map from the second member
35
        Map map2 = h2.getMap("hzObjectPath");
36
        // check the size of map2
37
        assertEquals(1000, map2.size());
38
        // check the size of map1 again
39
        assertEquals(1000, map1.size());
40
    }
41
	
42
	
43
}
test/edu/ucsb/nceas/metacat/dataone/hazelcast.test.properties.xml
1
<hazelcast>
2
    ...
3
    <map name="hzObjectPath">
4
        ...
5
        <map-store enabled="true">
6
            <!--
7
               Name of the class implementing MapLoader and/or MapStore. 
8
               The class should implement at least of these interfaces and
9
               contain no-argument constructor. Note that the inner classes are not supported.
10
            -->
11
            <class-name>edu.ucsb.nceas.metacat.dataone.hazelcast.ObjectPathMap</class-name>
12
            <!--
13
               Number of seconds to delay to call the MapStore.store(key, value).
14
               If the value is zero then it is write-through so MapStore.store(key, value)
15
               will be called as soon as the entry is updated.
16
               Otherwise it is write-behind so updates will be stored after write-delay-seconds
17
               value by calling Hazelcast.storeAll(map). Default value is 0.
18
            -->
19
            <write-delay-seconds>0</write-delay-seconds>
20
        </map-store>
21
    </map>
22
</hazelcast>
src/edu/ucsb/nceas/metacat/dataone/hazelcast/ObjectPathMap.java
1
package edu.ucsb.nceas.metacat.dataone.hazelcast;
2

  
3
import java.util.Collection;
4
import java.util.Collections;
5
import java.util.HashSet;
6
import java.util.Hashtable;
7
import java.util.List;
8
import java.util.Map;
9
import java.util.Set;
10

  
11
import org.dataone.service.types.v1.Identifier;
12

  
13
import com.hazelcast.core.MapLoader;
14

  
15
import edu.ucsb.nceas.metacat.IdentifierManager;
16
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
17
import edu.ucsb.nceas.metacat.properties.PropertyService;
18
import edu.ucsb.nceas.metacat.shared.ServiceException;
19
import edu.ucsb.nceas.utilities.FileUtil;
20
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
21

  
22

  
23
/**
24
 * MapLoader implementation for a hazelcast hzObjectPath.  This class is called
25
 * when the ObjectPathMap needs to refresh against the persistent data-store.
26
 * The use case for this class is to communicate the filepath between JVMs on
27
 * the same machine, specifically between the metacat instance and the d1_indexer.
28
 * 
29
 * d1_indexer will get Identifiers from elsewhere, but use this class to get
30
 * the paths to their associated files.  The getAllKeys() method can (and should)
31
 * return null in a live setting.  For unit testing, it may be useful to use
32
 * it to get some 
33
 * 
34
 * @author rnahf
35
 *
36
 */
37
public class ObjectPathMap implements MapLoader<Identifier, String> {
38
	private static IdentifierManager im;
39
	private static String dataPath;
40
	private static String metadataPath;
41

  
42

  
43
	
44
	/**
45
	 * creates an ObjectPathMapLoader
46
	 */
47
	public ObjectPathMap() {
48
//		try {
49
//			PropertyService ps = PropertyService.getInstance();
50
//			dataPath = PropertyService.getProperty("application.datafilepath");
51
//			metadataPath = PropertyService.getProperty("application.documentfilepath");
52
//		} catch (PropertyNotFoundException e) {
53
//			// TODO Auto-generated catch block
54
//			e.printStackTrace();
55
//		} catch (ServiceException e) {
56
//			// TODO Auto-generated catch block
57
//			e.printStackTrace();
58
//		}
59
		dataPath = "/data/";
60
		metadataPath = "/metadata/";
61
		im = IdentifierManager.getInstance();
62
	}
63

  
64
	
65
	/*
66
	 * Metadata is stored in a different place on the filesystem than
67
	 * the data.  The doctype value for metadata can vary, but for data
68
	 * is always 'BIN', so using a simple if-then-else to separate
69
	 */
70
	private String pathToDocid(String localid) throws McdbDocNotFoundException  {
71
		
72
		Hashtable<String, Object> ht = im.getDocumentInfo(localid);
73
		if (ht.get("doctype").equals("BIN")) {
74
			return dataPath + FileUtil.getFS() + localid;
75
		} else {
76
			return metadataPath + FileUtil.getFS() + localid;
77
		}		
78
	}
79

  
80
	
81
	/**
82
	 *  Implementation of hazelcast MapLoader interface method.
83
	 *  For the provided Identifier (as key), returns the path to the
84
	 *  document on the local filesystem.  Returns null if it can't 
85
	 *  create the path. 
86
	 */
87
	@Override
88
	public String load(Identifier key) 
89
	{
90
		String docid = null;
91
		String path = null;
92
		try {
93
			docid = im.getLocalId(key.getValue());
94
			path = pathToDocid(docid);
95
		} catch (McdbDocNotFoundException e) {
96
			// TODO Auto-generated catch block
97
			e.printStackTrace();
98
			return null;
99
		}
100
		return path;
101
	}
102
	
103
	
104
	/**
105
	 *  Implementation of hazelcast MapLoader interface method.  This method loads
106
	 *  mappings for all Identifiers in the parameters.  Any Identifier not found
107
	 *  is not included in the resulting map.
108
	 */
109
	@Override
110
	public Map<Identifier, String> loadAll(Collection<Identifier> identifiers) {
111
		
112
		Hashtable<Identifier,String> map = new Hashtable<Identifier,String>();
113
		for (Identifier id : identifiers) {
114
			try {
115
				String docid = im.getLocalId(id.getValue());
116
				map.put(id, pathToDocid(docid));
117
				
118
			} catch (McdbDocNotFoundException e) {
119
				// TODO should the map load an empty path instead of
120
				// leaving out the entire entry?
121
				e.printStackTrace();
122
			}
123
		}
124
		return map;
125
	}
126

  
127
	
128
	/**
129
	 * Return the full set of guids in the local metacat repository as
130
	 * dataone Identifiers.
131
	 * 
132
	 * (Hazelcast allows avoiding pre-loading by returning NULL, so consider
133
	 * re-implementing if that serves the purpose of this class better)
134
	 */
135
	@Override
136
	public Set<Identifier> loadAllKeys() {
137
		
138
		List<String> guids = im.getAllGUIDs();
139
		
140
		Set<Identifier> set = Collections.synchronizedSet(new HashSet<Identifier>());
141
		for (String guid : guids) {
142
			Identifier id = new Identifier();
143
			id.setValue(guid);
144
			set.add(id);
145
		}
146
		return set;
147
	}
148
}

Also available in: Unified diff