1
|
package edu.ucsb.nceas.metacat.index;
|
2
|
|
3
|
import static org.junit.Assert.assertFalse;
|
4
|
import static org.junit.Assert.assertTrue;
|
5
|
|
6
|
import edu.ucsb.nceas.metacat.common.SolrServerFactory;
|
7
|
import java.io.StringWriter;
|
8
|
import java.io.Writer;
|
9
|
import java.util.ArrayList;
|
10
|
import java.util.HashMap;
|
11
|
import java.util.List;
|
12
|
import java.util.Map;
|
13
|
|
14
|
import org.apache.solr.client.solrj.SolrServer;
|
15
|
import org.apache.solr.client.solrj.SolrServerException;
|
16
|
import org.apache.solr.client.solrj.response.QueryResponse;
|
17
|
import org.apache.solr.common.params.SolrParams;
|
18
|
import org.apache.solr.core.CoreContainer;
|
19
|
import org.apache.solr.core.SolrCore;
|
20
|
import org.apache.solr.request.LocalSolrQueryRequest;
|
21
|
import org.apache.solr.response.SolrQueryResponse;
|
22
|
import org.apache.solr.response.XMLResponseWriter;
|
23
|
import org.apache.solr.servlet.SolrRequestParsers;
|
24
|
import org.dataone.service.types.v1.Identifier;
|
25
|
import org.dataone.service.types.v1.SystemMetadata;
|
26
|
import org.dataone.service.util.TypeMarshaller;
|
27
|
import org.junit.Before;
|
28
|
import org.junit.Test;
|
29
|
|
30
|
public class SolrIndexIT {
|
31
|
|
32
|
private static final String SYSTEMMETAFILEPATH = "src/test/resources/eml-system-meta-example.xml";
|
33
|
private static final String EMLFILEPATH = "src/test/resources/eml-example.xml";
|
34
|
private static final String SYSTEMMETAUPDATEFILEPATH = "src/test/resources/eml-updating-system-meta-example.xml";
|
35
|
private static final String EMLUPDATEFILEPATH = "src/test/resources/eml-updating-example.xml";
|
36
|
private static final String SYSTEMMETAARCHIVEFILEPATH = "src/test/resources/eml-archive-system-meta-example.xml";
|
37
|
private static final String id = "urn:uuid:606a19dd-b531-4bf4-b5a5-6d06c3d39098";
|
38
|
private static final String newId = "urn:uuid:606a19dd-b531-4bf4-b5a5-6d06c3d39099";
|
39
|
|
40
|
private String annotation_id = "http://doi.org/annotation.1.1";
|
41
|
private static final String ANNOTATION_SYSTEM_META_FILE_PATH = "src/test/resources/annotation-system-meta-example.xml";
|
42
|
private static final String AO_FILE_PATH = "src/test/resources/ao-example.rdf";;
|
43
|
private static final String OA_FILE_PATH = "src/test/resources/oa-example.rdf";;
|
44
|
|
45
|
private SolrIndex solrIndex = null;
|
46
|
|
47
|
@Before
|
48
|
public void setUp() throws Exception {
|
49
|
solrIndex = generateSolrIndex();
|
50
|
}
|
51
|
|
52
|
public static SolrIndex generateSolrIndex() throws Exception {
|
53
|
String springConfigFile = "/index-processor-context.xml";
|
54
|
String metacatPropertyFile = null; //in this test, we use the test.properties file rather than metacat.properties file. so set it to be null.
|
55
|
ApplicationController controller = new ApplicationController(springConfigFile, metacatPropertyFile);
|
56
|
controller.initialize();
|
57
|
List<SolrIndex> list = controller.getSolrIndexes();
|
58
|
SolrIndex[] solrIndexesarray = list.toArray(new SolrIndex[list.size()]);
|
59
|
SolrIndex index = solrIndexesarray[0];
|
60
|
//SolrServer solrServer = SolrServerFactory.createSolrServer();
|
61
|
//index.setSolrServer(solrServer);
|
62
|
return index;
|
63
|
}
|
64
|
|
65
|
/**
|
66
|
* Test building index for an insert.
|
67
|
*/
|
68
|
@Test
|
69
|
public void testInsert() throws Exception {
|
70
|
|
71
|
|
72
|
//InputStream systemInputStream = new FileInputStream(new File(SYSTEMMETAFILEPATH));
|
73
|
SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAFILEPATH);
|
74
|
//List<String> chain = null;
|
75
|
Identifier pid = new Identifier();
|
76
|
pid.setValue(id);
|
77
|
solrIndex.update(pid, systemMetadata, EMLFILEPATH);
|
78
|
String result = doQuery(solrIndex.getSolrServer());
|
79
|
List<String> ids = solrIndex.getSolrIds();
|
80
|
//assertTrue(ids.size() == 1);
|
81
|
boolean foundId = false;
|
82
|
for(String identifiers :ids) {
|
83
|
if(id.equals(identifiers)) {
|
84
|
foundId = true;
|
85
|
}
|
86
|
}
|
87
|
assertTrue(foundId);
|
88
|
assertTrue(result.contains("version1"));
|
89
|
|
90
|
}
|
91
|
|
92
|
/**
|
93
|
* Test building index for an insert.
|
94
|
*/
|
95
|
@Test
|
96
|
public void testUpdate() throws Exception {
|
97
|
//InputStream systemInputStream = new FileInputStream(new File(SYSTEMMETAFILEPATH));
|
98
|
SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAUPDATEFILEPATH);
|
99
|
/*obsoletes.add(id);
|
100
|
obsoletes.add("tao");*/
|
101
|
Identifier pid = new Identifier();
|
102
|
pid.setValue(newId);
|
103
|
solrIndex.update(pid, systemMetadata, EMLFILEPATH);
|
104
|
String result = doQuery(solrIndex.getSolrServer());
|
105
|
assertTrue(result.contains("version1"));
|
106
|
assertTrue(result.contains("version2"));
|
107
|
|
108
|
// have to re-index the older version
|
109
|
Identifier obsoletedPid = systemMetadata.getObsoletes();
|
110
|
SystemMetadata obsoletedSystemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAFILEPATH);
|
111
|
assertTrue(obsoletedSystemMetadata.getIdentifier().getValue().equals(obsoletedPid.getValue()));
|
112
|
obsoletedSystemMetadata.setObsoletedBy(pid);
|
113
|
solrIndex.update(obsoletedPid, obsoletedSystemMetadata, EMLFILEPATH);
|
114
|
|
115
|
// old version should be marked as obsoleted and not returned
|
116
|
result = doQuery(solrIndex.getSolrServer(), "&fq=-obsoletedBy:*");
|
117
|
assertTrue(!result.contains("version1"));
|
118
|
assertTrue(result.contains("version2"));
|
119
|
}
|
120
|
|
121
|
/**
|
122
|
* Test building index for an insert.
|
123
|
*/
|
124
|
@Test
|
125
|
public void testArchive() throws Exception {
|
126
|
SolrIndex solrIndex = generateSolrIndex();
|
127
|
//InputStream systemInputStream = new FileInputStream(new File(SYSTEMMETAFILEPATH));
|
128
|
//System metadata's archive is true.
|
129
|
SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAARCHIVEFILEPATH);
|
130
|
/*ArrayList<String> obsoletes = new ArrayList<String>();
|
131
|
obsoletes.add(id);
|
132
|
obsoletes.add("tao");*/
|
133
|
Identifier pid = new Identifier();
|
134
|
pid.setValue(newId);
|
135
|
solrIndex.update(pid, systemMetadata, EMLUPDATEFILEPATH);
|
136
|
String result = doQuery(solrIndex.getSolrServer());
|
137
|
assertTrue(result.contains("version1"));
|
138
|
assertTrue(!result.contains("version2"));
|
139
|
}
|
140
|
|
141
|
|
142
|
/**
|
143
|
* Test building index for dynamic fields.
|
144
|
*/
|
145
|
@Test
|
146
|
public void testDynamicFields() throws Exception {
|
147
|
|
148
|
SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAFILEPATH);
|
149
|
Identifier pid = new Identifier();
|
150
|
pid.setValue(id);
|
151
|
solrIndex.update(pid, systemMetadata, EMLFILEPATH);
|
152
|
String result = doQuery(solrIndex.getSolrServer());
|
153
|
List<String> ids = solrIndex.getSolrIds();
|
154
|
boolean foundId = false;
|
155
|
for(String identifiers :ids) {
|
156
|
if (id.equals(identifiers)) {
|
157
|
foundId = true;
|
158
|
}
|
159
|
}
|
160
|
assertTrue(foundId);
|
161
|
assertTrue(result.contains("version1"));
|
162
|
|
163
|
// augment with the dynamic field
|
164
|
String fieldName = "test_count_i";
|
165
|
Map<String, List<Object>> fields = new HashMap<String, List<Object>>();
|
166
|
List<Object> values = new ArrayList<Object>();
|
167
|
values.add(6);
|
168
|
fields.put(fieldName, values);
|
169
|
solrIndex.insertFields(pid, fields);
|
170
|
result = doQuery(solrIndex.getSolrServer(), "&fq=" + fieldName + ":[0 TO 5]");
|
171
|
assertFalse(result.contains(id));
|
172
|
result = doQuery(solrIndex.getSolrServer(), "&fq=" + fieldName + ":[6 TO 6]");
|
173
|
assertTrue(result.contains(id));
|
174
|
|
175
|
// now update the value
|
176
|
values.clear();
|
177
|
values.add(7);
|
178
|
fields.put(fieldName, values);
|
179
|
solrIndex.insertFields(pid, fields);
|
180
|
result = doQuery(solrIndex.getSolrServer(), "&fq=" + fieldName + ":[7 TO 7]");
|
181
|
assertTrue(result.contains(id));
|
182
|
|
183
|
}
|
184
|
|
185
|
/**
|
186
|
* Test building index for annotation using OpenAnnotation.
|
187
|
*/
|
188
|
@Test
|
189
|
public void testOpenAnnotation() throws Exception {
|
190
|
|
191
|
SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAFILEPATH);
|
192
|
Identifier pid = new Identifier();
|
193
|
pid.setValue(id);
|
194
|
solrIndex.update(pid, systemMetadata, EMLFILEPATH);
|
195
|
String result = doQuery(solrIndex.getSolrServer());
|
196
|
List<String> ids = solrIndex.getSolrIds();
|
197
|
boolean foundId = false;
|
198
|
for(String identifiers :ids) {
|
199
|
if (id.equals(identifiers)) {
|
200
|
foundId = true;
|
201
|
}
|
202
|
}
|
203
|
assertTrue(foundId);
|
204
|
assertTrue(result.contains("version1"));
|
205
|
|
206
|
// augment with the dynamic field
|
207
|
SystemMetadata annotationSystemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, ANNOTATION_SYSTEM_META_FILE_PATH);
|
208
|
Identifier annotationPid = new Identifier();
|
209
|
annotationPid.setValue(annotation_id);
|
210
|
solrIndex.update(annotationPid, annotationSystemMetadata, OA_FILE_PATH);
|
211
|
String annotationResult = doQuery(solrIndex.getSolrServer(), "&fq=standard_sm:\"http://ecoinformatics.org/oboe/oboe.1.0/oboe-standards.owl#Gram\"");
|
212
|
assertTrue(annotationResult.contains(pid.getValue()));
|
213
|
assertTrue(annotationResult.contains("http://ecoinformatics.org/oboe/oboe.1.0/oboe-standards.owl#Gram"));
|
214
|
|
215
|
// check that it contains the creator annotation as well
|
216
|
assertTrue(annotationResult.contains("creator_sm"));
|
217
|
assertTrue(annotationResult.contains("http://sandbox-1.orcid.org/0000-0003-2141-4459"));
|
218
|
|
219
|
}
|
220
|
|
221
|
/**
|
222
|
* Do query - with no additional params
|
223
|
*/
|
224
|
public static String doQuery(SolrServer server)
|
225
|
throws SolrServerException {
|
226
|
return doQuery(server, null);
|
227
|
}
|
228
|
|
229
|
/**
|
230
|
* Do query, allowing additional parameters
|
231
|
*/
|
232
|
public static String doQuery(SolrServer server, String moreParams)
|
233
|
throws SolrServerException {
|
234
|
StringBuffer request = new StringBuffer();
|
235
|
request.append("q=" + "*:*");
|
236
|
if (moreParams != null) {
|
237
|
request.append(moreParams);
|
238
|
}
|
239
|
SolrParams solrParams = SolrRequestParsers.parseQueryString(request
|
240
|
.toString());
|
241
|
QueryResponse reponse = server.query(solrParams);
|
242
|
String result = toXML(solrParams, reponse);
|
243
|
System.out.println("**************************************************************************");
|
244
|
System.out.println("The query result:\n");
|
245
|
System.out.println(result);
|
246
|
System.out.println("**************************************************************************");
|
247
|
return result;
|
248
|
}
|
249
|
|
250
|
/**
|
251
|
* Transform the query response to the xml format.
|
252
|
*/
|
253
|
private static String toXML(SolrParams request, QueryResponse response) {
|
254
|
XMLResponseWriter xmlWriter = new XMLResponseWriter();
|
255
|
Writer w = new StringWriter();
|
256
|
SolrQueryResponse sResponse = new SolrQueryResponse();
|
257
|
sResponse.setAllValues(response.getResponse());
|
258
|
try {
|
259
|
SolrCore core = null;
|
260
|
CoreContainer container = SolrServerFactory.getCoreContainer();
|
261
|
core = container.getCore("collection1");
|
262
|
xmlWriter.write(w, new LocalSolrQueryRequest(core, request), sResponse);
|
263
|
} catch (Exception e) {
|
264
|
throw new RuntimeException("Unable to convert Solr response into XML", e);
|
265
|
}
|
266
|
return w.toString();
|
267
|
}
|
268
|
|
269
|
|
270
|
|
271
|
|
272
|
}
|
273
|
|