Revision 7793
Added by Jing Tao over 11 years ago
| metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/IndexGenerator.java | ||
|---|---|---|
| 52 | 52 |
import org.dataone.service.types.v1.SystemMetadata; |
| 53 | 53 |
|
| 54 | 54 |
import com.hazelcast.core.IMap; |
| 55 |
import com.hazelcast.core.ISet; |
|
| 55 | 56 |
|
| 56 | 57 |
|
| 57 | 58 |
/** |
| ... | ... | |
| 83 | 84 |
private IMap<Identifier, SystemMetadata> systemMetadataMap; |
| 84 | 85 |
private IMap<Identifier, String> objectPathMap; |
| 85 | 86 |
private Log log = LogFactory.getLog(IndexGenerator.class); |
| 86 |
private MNode mNode = null; |
|
| 87 |
//private MNode mNode = null;
|
|
| 87 | 88 |
private static List<String> resourceMapNamespaces = null; |
| 88 | 89 |
|
| 89 | 90 |
/** |
| ... | ... | |
| 95 | 96 |
this.solrIndex = solrIndex; |
| 96 | 97 |
resourceMapNamespaces = Settings.getConfiguration().getList(RESOURCEMAPPROPERYNAME); |
| 97 | 98 |
//this.systemMetadataListener = systemMetadataListener; |
| 98 |
this.mNode = new MNode(buildMNBaseURL()); |
|
| 99 |
//this.mNode = new MNode(buildMNBaseURL());
|
|
| 99 | 100 |
try {
|
| 100 | 101 |
waitingTime = Settings.getConfiguration().getInt(WAITIMEPOPERTYNAME); |
| 101 | 102 |
maxAttempts = Settings.getConfiguration().getInt(MAXATTEMPTSPROPERTYNAME); |
| ... | ... | |
| 174 | 175 |
List<String> solrIds = null; |
| 175 | 176 |
initSystemMetadataMap(); |
| 176 | 177 |
initObjectPathMap(); |
| 177 |
List[] allMetacatIds = getMetadataIds(since, until);
|
|
| 178 |
List[] allMetacatIds = getMetacatIds(since, until);
|
|
| 178 | 179 |
List<String> otherMetacatIds = allMetacatIds[FIRST]; |
| 179 | 180 |
List<String> resourceMapIds = allMetacatIds[SECOND]; |
| 180 | 181 |
log.info("the metacat ids (exception resource map -----------------------------"+otherMetacatIds);
|
| ... | ... | |
| 259 | 260 |
* The first element of the list is the ids except the resource map. The second elements of the list is the ids of the resource map. |
| 260 | 261 |
* The reason to split them is when we index the resource map, we need the index of the document in the resource map ready. |
| 261 | 262 |
*/ |
| 262 |
private List[] getMetadataIds(Date since, Date until) throws InvalidRequest,
|
|
| 263 |
InvalidToken, NotAuthorized, NotImplemented, ServiceFailure {
|
|
| 263 |
private List[] getMetacatIds(Date since, Date until) throws InvalidRequest,
|
|
| 264 |
InvalidToken, NotAuthorized, NotImplemented, ServiceFailure, FileNotFoundException {
|
|
| 264 | 265 |
|
| 265 | 266 |
List<String> resourceMapIds = new ArrayList(); |
| 266 | 267 |
List<String> otherIds = new ArrayList(); |
| 267 | 268 |
List[] ids = new List[2]; |
| 268 | 269 |
ids[FIRST]= otherIds; |
| 269 | 270 |
ids[SECOND] = resourceMapIds; |
| 270 |
ObjectList objects = null; |
|
| 271 |
int times = 0; |
|
| 272 |
while (true) {
|
|
| 273 |
try {
|
|
| 274 |
mNode.ping(); |
|
| 275 |
break; |
|
| 276 |
} catch (Exception e) {
|
|
| 277 |
if(times <= maxAttempts) {
|
|
| 278 |
log.warn("IndexGenerator.getMetadataIds - the mnode "+ mNode.getNodeBaseServiceUrl()+
|
|
| 279 |
" is not ready :" +e.getMessage()+"\nWe will try to access it "+waitingTime/1000+" seconds later "); |
|
| 280 |
try {
|
|
| 281 |
Thread.sleep(waitingTime); |
|
| 282 |
} catch (Exception ee) {
|
|
| 283 |
log.warn("IndexGenerator.getMetadataIds - the thread can't sleep for "+ waitingTime/1000+" seconds to wait the MNode");
|
|
| 271 |
ISet<Identifier> metacatIds = DistributedMapsFactory.getIdentifiersSet(); |
|
| 272 |
if(metacatIds != null) {
|
|
| 273 |
for(Identifier identifier : metacatIds) {
|
|
| 274 |
if(identifier != null && identifier.getValue() != null && !identifier.getValue().equals("")) {
|
|
| 275 |
SystemMetadata sysmeta = getSystemMetadata(identifier.getValue()); |
|
| 276 |
if(sysmeta != null && !sysmeta.getArchived()) {
|
|
| 277 |
ObjectFormatIdentifier formatId =sysmeta.getFormatId(); |
|
| 278 |
//System.out.println("the object format id is "+formatId.getValue());
|
|
| 279 |
//System.out.println("the ============ resourcMapNamespaces"+resourceMapNamespaces);
|
|
| 280 |
boolean correctTimeRange = false; |
|
| 281 |
Date sysDate = sysmeta.getDateSysMetadataModified(); |
|
| 282 |
if(since == null && until == null) {
|
|
| 283 |
correctTimeRange = true; |
|
| 284 |
} else if (since != null && until == null) {
|
|
| 285 |
if(sysDate.getTime() >= since.getTime()) {
|
|
| 286 |
correctTimeRange = true; |
|
| 287 |
} |
|
| 288 |
} else if (since == null && until != null) {
|
|
| 289 |
if(sysDate.getTime() <= until.getTime()) {
|
|
| 290 |
correctTimeRange = true; |
|
| 291 |
} |
|
| 292 |
} else if (since != null && until != null) {
|
|
| 293 |
if(sysDate.getTime() >= since.getTime() && sysDate.getTime() <= until.getTime()) {
|
|
| 294 |
correctTimeRange = true; |
|
| 295 |
} |
|
| 296 |
} |
|
| 297 |
if(correctTimeRange && formatId != null && formatId.getValue() != null && resourceMapNamespaces != null && isResourceMap(formatId)) {
|
|
| 298 |
resourceMapIds.add(identifier.getValue()); |
|
| 299 |
} else {
|
|
| 300 |
otherIds.add(identifier.getValue()); |
|
| 301 |
} |
|
| 284 | 302 |
} |
| 285 |
|
|
| 286 |
} else {
|
|
| 287 |
throw new ServiceFailure("0000", "IndexGenerator.getMetadataIds - the mnode "+ mNode.getNodeBaseServiceUrl()+
|
|
| 288 |
" is not ready even though Metacat-index wailted for "+maxAttempts*waitingTime/1000+" seconds. We can't get the objects list from it and the building index can't happen this time"); |
|
| 289 | 303 |
} |
| 290 |
|
|
| 291 | 304 |
} |
| 292 |
times++; |
|
| 293 | 305 |
} |
| 294 |
if(since == null && until == null) {
|
|
| 295 |
objects = mNode.listObjects(); |
|
| 296 |
} else {
|
|
| 297 |
objects = mNode.listObjects(since, until, null, true, 0, Integer.MAX_VALUE); |
|
| 298 |
} |
|
| 299 |
if(objects != null) {
|
|
| 306 |
/*if(objects != null) {
|
|
| 300 | 307 |
List<ObjectInfo> objectInfoList = objects.getObjectInfoList(); |
| 301 | 308 |
if(objectInfoList != null) {
|
| 302 | 309 |
for(ObjectInfo info : objectInfoList) {
|
| ... | ... | |
| 319 | 326 |
} |
| 320 | 327 |
} |
| 321 | 328 |
} |
| 322 |
} |
|
| 329 |
}*/
|
|
| 323 | 330 |
return ids; |
| 324 | 331 |
} |
| 325 | 332 |
|
| ... | ... | |
| 339 | 346 |
return isResourceMap; |
| 340 | 347 |
} |
| 341 | 348 |
|
| 342 |
/* |
|
| 343 |
* Build up the mn base url |
|
| 344 |
*/ |
|
| 345 |
private String buildMNBaseURL() {
|
|
| 346 |
String httpPort = Settings.getConfiguration().getString("server.httpPort");
|
|
| 347 |
String serverURL = "http://"; |
|
| 348 |
if(httpPort.equals("443") || httpPort.equals("8443"))
|
|
| 349 |
{
|
|
| 350 |
serverURL = "https://"; |
|
| 351 |
} |
|
| 352 |
serverURL = serverURL+Settings.getConfiguration().getString("server.name");
|
|
| 353 |
if (!httpPort.equals("80")) {
|
|
| 354 |
serverURL += ":" + httpPort; |
|
| 355 |
} |
|
| 356 |
serverURL = serverURL +"/"+ Settings.getConfiguration().getString("application.context")+MNAPPENDIX;
|
|
| 357 |
log.info("IndexGenerator.buildMNBaseURL - the base url of MNode is "+serverURL);
|
|
| 358 |
return serverURL; |
|
| 359 |
} |
|
| 349 |
|
|
| 360 | 350 |
|
| 361 |
|
|
| 362 | 351 |
/* |
| 363 | 352 |
* Generate index for the id. |
| 364 | 353 |
*/ |
Also available in: Unified diff
Use the identifier set to get the list of ids in the member node.