Revision 7815
Added by ben leinfelder about 11 years ago
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/event/IndexEvent.java | ||
---|---|---|
23 | 23 |
|
24 | 24 |
import java.util.Date; |
25 | 25 |
|
26 |
import org.dataone.service.types.v1.Event; |
|
26 | 27 |
import org.dataone.service.types.v1.Identifier; |
27 | 28 |
|
28 | 29 |
|
... | ... | |
33 | 34 |
*/ |
34 | 35 |
public class IndexEvent { |
35 | 36 |
|
36 |
public static final int SUCCESSINSERT = 0; |
|
37 |
public static final int FAILUREINSERT = 6; |
|
38 |
public static final int SUCCESSDELETE = 12; |
|
39 |
public static final int FAILUREDELETE = 18; |
|
40 |
public static final int STARTTIMEDINDEX = 24; |
|
41 |
public static final int FINISHTIMEDINDEX = 30; |
|
42 |
private int type = -1; |
|
37 |
private Event action = null; |
|
43 | 38 |
private Date date = null; |
44 |
private Identifier pid = null;
|
|
39 |
private Identifier identifier = null;
|
|
45 | 40 |
private String description = null; |
46 | 41 |
|
47 |
|
|
48 |
|
|
49 |
|
|
50 | 42 |
/** |
51 |
* Get the type of the event.
|
|
52 |
* @return the type of the event
|
|
43 |
* Get the action of the event.
|
|
44 |
* @return the action of the event
|
|
53 | 45 |
*/ |
54 |
public int getType() {
|
|
55 |
return this.type;
|
|
46 |
public Event getAction() {
|
|
47 |
return this.action;
|
|
56 | 48 |
} |
57 | 49 |
|
58 | 50 |
/** |
59 |
* Set the type of the event
|
|
60 |
* @param type
|
|
51 |
* Set the action of the event
|
|
52 |
* @param action
|
|
61 | 53 |
*/ |
62 |
public void setType(int type) {
|
|
63 |
this.type = type;
|
|
54 |
public void setAction(Event action) {
|
|
55 |
this.action = action;
|
|
64 | 56 |
} |
65 | 57 |
|
66 | 58 |
/** |
... | ... | |
83 | 75 |
* Get the identifier of the data object involving the event |
84 | 76 |
* @return |
85 | 77 |
*/ |
86 |
public Identifier getPid() {
|
|
87 |
return pid;
|
|
78 |
public Identifier getIdentifier() {
|
|
79 |
return identifier;
|
|
88 | 80 |
} |
89 | 81 |
|
90 | 82 |
/** |
91 | 83 |
* Set the identifier of the data object involving the event. |
92 | 84 |
* @param pid |
93 | 85 |
*/ |
94 |
public void setPid(Identifier pid) {
|
|
95 |
this.pid = pid;
|
|
86 |
public void setIdentifier(Identifier pid) {
|
|
87 |
this.identifier = pid;
|
|
96 | 88 |
} |
97 | 89 |
|
98 | 90 |
/** |
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/event/HazelcastIndexEventLog.java | ||
---|---|---|
1 |
/** |
|
2 |
* '$RCSfile$' |
|
3 |
* Purpose: A class which creates a instance of an IndexEventLog. |
|
4 |
* Copyright: 2013 Regents of the University of California and the |
|
5 |
* National Center for Ecological Analysis and Synthesis |
|
6 |
* Authors: Leinfelder |
|
7 |
* |
|
8 |
* This program is free software; you can redistribute it and/or modify |
|
9 |
* it under the terms of the GNU General Public License as published by |
|
10 |
* the Free Software Foundation; either version 2 of the License, or |
|
11 |
* (at your option) any later version. |
|
12 |
* |
|
13 |
* This program is distributed in the hope that it will be useful, |
|
14 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
15 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
16 |
* GNU General Public License for more details. |
|
17 |
* |
|
18 |
* You should have received a copy of the GNU General Public License |
|
19 |
* along with this program; if not, write to the Free Software |
|
20 |
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
|
21 |
*/ |
|
22 |
package edu.ucsb.nceas.metacat.index.event; |
|
23 |
|
|
24 |
import java.util.ArrayList; |
|
25 |
import java.util.Date; |
|
26 |
import java.util.List; |
|
27 |
|
|
28 |
import org.dataone.service.types.v1.Event; |
|
29 |
import org.dataone.service.types.v1.Identifier; |
|
30 |
|
|
31 |
import edu.ucsb.nceas.metacat.index.DistributedMapsFactory; |
|
32 |
|
|
33 |
/** |
|
34 |
* @author leinfelder |
|
35 |
* |
|
36 |
*/ |
|
37 |
public class HazelcastIndexEventLog implements IndexEventLog { |
|
38 |
|
|
39 |
/* (non-Javadoc) |
|
40 |
* @see edu.ucsb.nceas.metacat.index.event.IndexEventLog#write(edu.ucsb.nceas.metacat.index.event.IndexEvent) |
|
41 |
*/ |
|
42 |
@Override |
|
43 |
public void write(IndexEvent event) throws IndexEventLogException { |
|
44 |
// write to the map |
|
45 |
try { |
|
46 |
DistributedMapsFactory.getIndexEventMap().put(event.getIdentifier(), event); |
|
47 |
} catch (Exception e) { |
|
48 |
throw new IndexEventLogException("Could not write to event map", e); |
|
49 |
} |
|
50 |
} |
|
51 |
|
|
52 |
/* (non-Javadoc) |
|
53 |
* @see edu.ucsb.nceas.metacat.index.event.IndexEventLog#remove(java.lang.String) |
|
54 |
*/ |
|
55 |
@Override |
|
56 |
public void remove(Identifier identifier) throws IndexEventLogException { |
|
57 |
// remove from the map |
|
58 |
try { |
|
59 |
DistributedMapsFactory.getIndexEventMap().remove(identifier); |
|
60 |
} catch (Exception e) { |
|
61 |
throw new IndexEventLogException("Could not remove from event map", e); |
|
62 |
} |
|
63 |
|
|
64 |
} |
|
65 |
|
|
66 |
/* (non-Javadoc) |
|
67 |
* @see edu.ucsb.nceas.metacat.index.event.IndexEventLog#getEvents(int, org.dataone.service.types.v1.Identifier, java.util.Date, java.util.Date) |
|
68 |
*/ |
|
69 |
@Override |
|
70 |
public List<IndexEvent> getEvents(Event action, Identifier pid, Date start, Date end) throws IndexEventLogException { |
|
71 |
try { |
|
72 |
// TODO: query the map using the parameters |
|
73 |
return new ArrayList<IndexEvent>(DistributedMapsFactory.getIndexEventMap().values()); |
|
74 |
} catch (Exception e) { |
|
75 |
throw new IndexEventLogException("Could not remove from event map", e); |
|
76 |
} |
|
77 |
} |
|
78 |
|
|
79 |
/* (non-Javadoc) |
|
80 |
* @see edu.ucsb.nceas.metacat.index.event.IndexEventLog#getLastProcessDate() |
|
81 |
*/ |
|
82 |
@Override |
|
83 |
public Date getLastProcessDate() throws IndexEventLogException { |
|
84 |
// TODO Auto-generated method stub |
|
85 |
return null; |
|
86 |
} |
|
87 |
|
|
88 |
/* (non-Javadoc) |
|
89 |
* @see edu.ucsb.nceas.metacat.index.event.IndexEventLog#setLastProcessDate(java.util.Date) |
|
90 |
*/ |
|
91 |
@Override |
|
92 |
public void setLastProcessDate(Date date) throws IndexEventLogException { |
|
93 |
// TODO Auto-generated method stub |
|
94 |
|
|
95 |
} |
|
96 |
|
|
97 |
} |
|
0 | 98 |
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/event/IndexEventFileLog.java | ||
---|---|---|
33 | 33 |
|
34 | 34 |
import org.apache.commons.io.FileUtils; |
35 | 35 |
import org.dataone.configuration.Settings; |
36 |
import org.dataone.service.types.v1.Event; |
|
36 | 37 |
import org.dataone.service.types.v1.Identifier; |
37 | 38 |
|
38 | 39 |
|
... | ... | |
83 | 84 |
Vector<String> lines = new Vector<String>(); |
84 | 85 |
StringBuffer lineBuffer = new StringBuffer(); |
85 | 86 |
lineBuffer.append("\""+index+"\""+FIELDSEPERATOR); |
86 |
int type = event.getType();
|
|
87 |
lineBuffer.append("\""+type+"\""+FIELDSEPERATOR);
|
|
87 |
Event type = event.getAction();
|
|
88 |
lineBuffer.append("\"" + type.xmlValue() + "\"" + FIELDSEPERATOR);
|
|
88 | 89 |
Date date = event.getDate(); |
89 | 90 |
if(date != null) { |
90 | 91 |
DateFormat formate = new SimpleDateFormat(); |
91 | 92 |
lineBuffer.append("\""+formate.format(date)+"\""+FIELDSEPERATOR); |
92 | 93 |
} |
93 |
Identifier id = event.getPid();
|
|
94 |
Identifier id = event.getIdentifier();
|
|
94 | 95 |
if(id != null) { |
95 | 96 |
lineBuffer.append("\""+id.getValue()+"\""+FIELDSEPERATOR); |
96 | 97 |
} |
... | ... | |
120 | 121 |
|
121 | 122 |
/** |
122 | 123 |
* Gets the list of IndexEvent matching the specified set of filters. The filter parameters can be null |
123 |
* @param type the type of the event
|
|
124 |
* @param action the action of the event
|
|
124 | 125 |
* @param pid the identifier of the data object in the event |
125 | 126 |
* @param start the start time of the time range for query |
126 | 127 |
* @param end the end time of the time range for query |
127 | 128 |
* @return |
128 | 129 |
* @throws IndexEventLogException |
129 | 130 |
*/ |
130 |
public List<IndexEvent> getEvents(int type, Identifier pid, Date start, Date end) throws IndexEventLogException {
|
|
131 |
public List<IndexEvent> getEvents(Event action, Identifier pid, Date start, Date end) throws IndexEventLogException {
|
|
131 | 132 |
List<IndexEvent> list = null; |
132 | 133 |
return list; |
133 | 134 |
} |
134 | 135 |
|
135 |
|
|
136 |
/** |
|
137 |
* Get the list of identifiers which were failed to build the solr index since the |
|
138 |
* previous timed indexing (including the ones in the timed indexing). |
|
139 |
* @return the list of failure identifiers. The null will be returned if no failure. |
|
140 |
*/ |
|
141 |
public List<String> getFailedPids() throws IndexEventLogException { |
|
142 |
return null; |
|
143 |
} |
|
144 | 136 |
|
145 |
|
|
146 | 137 |
/** |
147 | 138 |
* Get the latest SystemMetadata modification Date of the objects that were built |
148 | 139 |
* the solr index during the previous timed indexing process. |
... | ... | |
182 | 173 |
} |
183 | 174 |
|
184 | 175 |
@Override |
185 |
public void remove(String identifier) throws IndexEventLogException {
|
|
176 |
public void remove(Identifier identifier) throws IndexEventLogException {
|
|
186 | 177 |
// TODO Auto-generated method stub |
187 | 178 |
|
188 | 179 |
} |
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/event/IndexEventLog.java | ||
---|---|---|
24 | 24 |
import java.util.Date; |
25 | 25 |
import java.util.List; |
26 | 26 |
|
27 |
import org.dataone.service.types.v1.Event; |
|
27 | 28 |
import org.dataone.service.types.v1.Identifier; |
28 | 29 |
|
29 | 30 |
/** |
... | ... | |
46 | 47 |
* @param identifier |
47 | 48 |
* @throws IndexEventLogException |
48 | 49 |
*/ |
49 |
public void remove(String identifier) throws IndexEventLogException;
|
|
50 |
public void remove(Identifier identifier) throws IndexEventLogException;
|
|
50 | 51 |
|
51 |
|
|
52 | 52 |
/** |
53 | 53 |
* Gets the list of IndexEvent matching the specified set of filters. The filter parameters can be null |
54 | 54 |
* @param type the type of the event |
... | ... | |
58 | 58 |
* @return |
59 | 59 |
* @throws IndexEventLogException |
60 | 60 |
*/ |
61 |
public List<IndexEvent> getEvents(int type, Identifier pid, Date start, Date end) throws IndexEventLogException;
|
|
61 |
public List<IndexEvent> getEvents(Event action, Identifier pid, Date start, Date end) throws IndexEventLogException;
|
|
62 | 62 |
|
63 | 63 |
/** |
64 |
* Get the list of identifiers which were failed to build the solr index since the |
|
65 |
* previous timed indexing (including the ones in the timed indexing). |
|
66 |
* @return the list of failure identifiers. The null will be returned if no failure. |
|
67 |
*/ |
|
68 |
public List<String> getFailedPids() throws IndexEventLogException; |
|
69 |
|
|
70 |
|
|
71 |
/** |
|
72 | 64 |
* Get the latest SystemMetadata modification Date of the objects that were built |
73 | 65 |
* the solr index during the previous timed indexing process. |
74 | 66 |
* @return the date. The null will be returned if there is no such date. |
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java | ||
---|---|---|
58 | 58 |
import org.apache.solr.common.SolrDocument; |
59 | 59 |
import org.apache.solr.common.SolrDocumentList; |
60 | 60 |
import org.apache.solr.common.SolrInputDocument; |
61 |
import org.apache.solr.common.util.NamedList; |
|
62 |
|
|
63 | 61 |
import org.dataone.cn.indexer.XMLNamespaceConfig; |
64 | 62 |
import org.dataone.cn.indexer.parser.IDocumentSubprocessor; |
65 | 63 |
import org.dataone.cn.indexer.parser.SolrField; |
66 | 64 |
import org.dataone.cn.indexer.resourcemap.ResourceEntry; |
67 | 65 |
import org.dataone.cn.indexer.resourcemap.ResourceMap; |
68 | 66 |
import org.dataone.cn.indexer.solrhttp.SolrDoc; |
69 |
import org.dataone.cn.indexer.solrhttp.SolrElementAdd; |
|
70 | 67 |
import org.dataone.cn.indexer.solrhttp.SolrElementField; |
71 | 68 |
import org.dataone.service.exceptions.NotFound; |
72 | 69 |
import org.dataone.service.exceptions.NotImplemented; |
73 | 70 |
import org.dataone.service.exceptions.ServiceFailure; |
74 | 71 |
import org.dataone.service.exceptions.UnsupportedType; |
72 |
import org.dataone.service.types.v1.Event; |
|
75 | 73 |
import org.dataone.service.types.v1.Identifier; |
76 | 74 |
import org.dataone.service.types.v1.SystemMetadata; |
77 | 75 |
import org.dataone.service.util.TypeMarshaller; |
78 | 76 |
import org.jibx.runtime.JiBXException; |
79 | 77 |
import org.w3c.dom.Document; |
80 |
import org.w3c.dom.NameList; |
|
81 | 78 |
import org.xml.sax.SAXException; |
82 | 79 |
|
83 | 80 |
import edu.ucsb.nceas.metacat.index.event.EventlogFactory; |
... | ... | |
397 | 394 |
event.setDate(Calendar.getInstance().getTime()); |
398 | 395 |
Identifier pid = new Identifier(); |
399 | 396 |
pid.setValue(doc.getIdentifier()); |
400 |
event.setPid(pid);
|
|
397 |
event.setIdentifier(pid);
|
|
401 | 398 |
try { |
402 | 399 |
UpdateResponse response = solrServer.add(solrDoc); |
403 | 400 |
solrServer.commit(); |
... | ... | |
409 | 406 |
log.error("SolrIndex.insertToIndex - IndexEventLog can't log the index inserting event :"+e.getMessage()); |
410 | 407 |
}*/ |
411 | 408 |
} catch (SolrServerException e) { |
412 |
event.setType(IndexEvent.FAILUREINSERT);
|
|
409 |
event.setAction(Event.CREATE);
|
|
413 | 410 |
event.setDescription("Failed to insert the solr index for the id "+pid.getValue()+" since "+e.getMessage()); |
414 | 411 |
try { |
415 | 412 |
EventlogFactory.createIndexEventLog().write(event); |
... | ... | |
418 | 415 |
} |
419 | 416 |
throw e; |
420 | 417 |
} catch (IOException e) { |
421 |
event.setType(IndexEvent.FAILUREINSERT);
|
|
418 |
event.setAction(Event.CREATE);
|
|
422 | 419 |
event.setDescription("Failed to insert the solr index for the id "+pid.getValue()+" since "+e.getMessage()); |
423 | 420 |
try { |
424 | 421 |
EventlogFactory.createIndexEventLog().write(event); |
... | ... | |
628 | 625 |
event.setDate(Calendar.getInstance().getTime()); |
629 | 626 |
Identifier identifier = new Identifier(); |
630 | 627 |
identifier.setValue(pid); |
631 |
event.setPid(identifier);
|
|
628 |
event.setIdentifier(identifier);
|
|
632 | 629 |
try { |
633 | 630 |
solrServer.deleteById(pid); |
634 | 631 |
solrServer.commit(); |
... | ... | |
640 | 637 |
log.error("SolrIndex.removeFromIndex - IndexEventLog can't log the index deleting event :"+e.getMessage()); |
641 | 638 |
}*/ |
642 | 639 |
} catch (SolrServerException e) { |
643 |
event.setType(IndexEvent.FAILUREDELETE);
|
|
640 |
event.setAction(Event.DELETE);
|
|
644 | 641 |
event.setDescription("Failurely remove the solr index for the id "+identifier.getValue()+" since "+e.getMessage()); |
645 | 642 |
try { |
646 | 643 |
EventlogFactory.createIndexEventLog().write(event); |
... | ... | |
650 | 647 |
throw e; |
651 | 648 |
|
652 | 649 |
} catch (IOException e) { |
653 |
event.setType(IndexEvent.FAILUREDELETE);
|
|
650 |
event.setAction(Event.DELETE);
|
|
654 | 651 |
event.setDescription("Failurely remove the solr index for the id "+identifier.getValue()+" since "+e.getMessage()); |
655 | 652 |
try { |
656 | 653 |
EventlogFactory.createIndexEventLog().write(event); |
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/DistributedMapsFactory.java | ||
---|---|---|
44 | 44 |
import com.hazelcast.core.IMap; |
45 | 45 |
import com.hazelcast.core.ISet; |
46 | 46 |
|
47 |
import edu.ucsb.nceas.metacat.index.event.IndexEvent; |
|
47 | 48 |
|
49 |
|
|
48 | 50 |
/** |
49 | 51 |
* A factory to get distributed maps from the haszel cast client. |
50 | 52 |
* @author tao |
... | ... | |
69 | 71 |
/* The Hazelcast distributed identifiers set */ |
70 | 72 |
private static ISet<Identifier> identifiersSet = null; |
71 | 73 |
|
72 |
/* |
|
74 |
// for sending index events to metacat |
|
75 |
private static String hzIndexEventMap = null; |
|
76 |
private static IMap<Identifier, IndexEvent> indexEventMap = null; |
|
77 |
|
|
78 |
/** |
|
73 | 79 |
* Start the hazel cast client |
74 | 80 |
*/ |
75 | 81 |
private static void startHazelCastClient() throws FileNotFoundException, ServiceFailure{ |
... | ... | |
92 | 98 |
// the index queue name to listen to |
93 | 99 |
hzIndexQueue = Settings.getConfiguration().getString("index.hazelcast.indexqueue"); |
94 | 100 |
|
101 |
// the index event map name to send events to |
|
102 |
hzIndexEventMap = Settings.getConfiguration().getString("index.hazelcast.indexeventmap"); |
|
103 |
|
|
95 | 104 |
// get config values |
96 | 105 |
hzSystemMetadata = Settings.getConfiguration().getString( |
97 | 106 |
"dataone.hazelcast.storageCluster.systemMetadataMap"); |
... | ... | |
251 | 260 |
return indexQueue; |
252 | 261 |
} |
253 | 262 |
|
263 |
/** |
|
264 |
* Get the index event map |
|
265 |
* @return the index event map for writing/reading events |
|
266 |
* @throws FileNotFoundException |
|
267 |
* @throws ServiceFailure |
|
268 |
*/ |
|
269 |
public static IMap<Identifier, IndexEvent> getIndexEventMap() throws FileNotFoundException, ServiceFailure { |
|
270 |
if(hzClient == null) { |
|
271 |
startHazelCastClient(); |
|
272 |
} |
|
273 |
indexEventMap = hzClient.getMap(hzIndexEventMap); |
|
274 |
return indexEventMap; |
|
275 |
} |
|
276 |
|
|
254 | 277 |
} |
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/IndexGenerator.java | ||
---|---|---|
30 | 30 |
import java.io.FileNotFoundException; |
31 | 31 |
import java.io.InputStream; |
32 | 32 |
import java.util.ArrayList; |
33 |
import java.util.Calendar; |
|
34 | 33 |
import java.util.Date; |
35 | 34 |
import java.util.List; |
36 | 35 |
import java.util.TimerTask; |
37 |
import java.util.Vector; |
|
38 | 36 |
|
39 | 37 |
import org.apache.commons.logging.Log; |
40 | 38 |
import org.apache.commons.logging.LogFactory; |
41 | 39 |
import org.apache.solr.client.solrj.SolrServerException; |
42 |
import org.dataone.client.MNode; |
|
43 | 40 |
import org.dataone.configuration.Settings; |
44 | 41 |
import org.dataone.service.exceptions.InvalidRequest; |
45 | 42 |
import org.dataone.service.exceptions.InvalidToken; |
... | ... | |
48 | 45 |
import org.dataone.service.exceptions.ServiceFailure; |
49 | 46 |
import org.dataone.service.types.v1.Identifier; |
50 | 47 |
import org.dataone.service.types.v1.ObjectFormatIdentifier; |
51 |
import org.dataone.service.types.v1.ObjectInfo; |
|
52 |
import org.dataone.service.types.v1.ObjectList; |
|
53 | 48 |
import org.dataone.service.types.v1.SystemMetadata; |
54 | 49 |
|
55 | 50 |
import com.hazelcast.core.IMap; |
... | ... | |
223 | 218 |
} |
224 | 219 |
|
225 | 220 |
//add the failedPids |
226 |
List<String> failedPids = EventlogFactory.createIndexEventLog().getFailedPids();
|
|
221 |
List<IndexEvent> failedEvents = EventlogFactory.createIndexEventLog().getEvents(null, null, null, null);
|
|
227 | 222 |
List<String> failedOtherIds = new ArrayList<String>(); |
228 | 223 |
List<String> failedResourceMapIds = new ArrayList<String>(); |
229 |
if(failedPids != null) { |
|
230 |
for(String id : failedPids) { |
|
224 |
if(failedEvents != null) { |
|
225 |
for(IndexEvent event : failedEvents) { |
|
226 |
String id = event.getIdentifier().getValue(); |
|
231 | 227 |
SystemMetadata sysmeta = getSystemMetadata(id); |
232 | 228 |
if(sysmeta != null && !sysmeta.getArchived()) { |
233 | 229 |
ObjectFormatIdentifier formatId =sysmeta.getFormatId(); |
lib/metacat.properties | ||
---|---|---|
596 | 596 |
index.regenerate.interval=7200000 |
597 | 597 |
index.eventlog.classname=edu.ucsb.nceas.metacat.index.event.IndexEventFileLog |
598 | 598 |
index.hazelcast.indexqueue=hzIndexQueue |
599 |
index.hazelcast.indexeventmap=hzIndexEventMap |
|
599 | 600 |
|
600 | 601 |
############# SOLR Search Section ########################################### |
601 | 602 |
#Embedded (default): |
Also available in: Unified diff
refactor IndexEventLog a bit to simplify type/action information. prep for serializing IndexEvent objects to Metacat. https://projects.ecoinformatics.org/ecoinfo/issues/5944