Project

General

Profile

« Previous | Next » 

Revision 8293

Added by Jing Tao over 10 years ago

combine the index code for failed ids and other ids.

View differences:

IndexGenerator.java
90 90 
    //private SystemMetadataEventListener systemMetadataListener = null;
91 91 
    private IMap<Identifier, SystemMetadata> systemMetadataMap;
92 92 
    private IMap<Identifier, String> objectPathMap;
93 
    private ISet<SystemMetadata> indexQueue;
93 94 
    private Log log = LogFactory.getLog(IndexGenerator.class);
94 95 
    //private MNode mNode = null;
95 96 
    private static List<String> resourceMapNamespaces = null;
......
208 209 
        List<String> solrIds = null;
209 210 
        initSystemMetadataMap();
210 211 
        initObjectPathMap();
212 
        initIndexQueue();
211 213 
        List[] metacatIds = getMetacatIds(since, until);
212 214 
        List<String> otherMetacatIds = metacatIds[FIRST];
213 215 
        List<String> resourceMapIds =  metacatIds[SECOND];
......
269 271
270 272 
        //add the failedPids
271 273 
        List<IndexEvent> failedEvents = EventlogFactory.createIndexEventLog().getEvents(null, null, null, null);
272 
        List<IndexEvent> failedOtherIds = new ArrayList<IndexEvent>();
273 
        List<IndexEvent> failedResourceMapIds = new ArrayList<IndexEvent>();
274 
        List<String> failedOtherIds = new ArrayList<String>();
275 
        List<String> failedResourceMapIds = new ArrayList<String>();
274 276 
        if(failedEvents != null) {
275 277 
            for(IndexEvent event : failedEvents) {
276 278 
            	String id = event.getIdentifier().getValue();
......
278 280 
                if(sysmeta != null) {
279 281 
                    ObjectFormatIdentifier formatId =sysmeta.getFormatId();
280 282 
                    if(formatId != null && formatId.getValue() != null && resourceMapNamespaces != null && isResourceMap(formatId)) {
281 
                        failedResourceMapIds.add(event);
283 
                        failedResourceMapIds.add(id);
282 284 
                    } else {
283 
                        failedOtherIds.add(event);
285 
                        failedOtherIds.add(id);
284 286 
                    }
285 287 
                }
286 288 
            }
287 289 
        }
288 
        indexFailedIds(failedOtherIds);
289 
        indexFailedIds(failedResourceMapIds);
290 
        //indexFailedIds(failedOtherIds);
291 
        //indexFailedIds(failedResourceMapIds);
290 292
293 
        index(failedOtherIds);
294 
        index(failedResourceMapIds);
295
291 296 
        /*if(!failedOtherIds.isEmpty()) {
292 297 
            failedOtherIds.addAll(otherMetacatIds);
293 298 
        } else {
......
299 304 
        } else {
300 305 
            failedResourceMapIds = resourceMapIds;
301 306 
        }*/
302
307 
        //log.info("the ids in index_event for reindex ( except the resourcemap)=====================================\n "+failedOtherIds);
308 
        //log.info("the resourcemap ids in index_event for reindex =====================================\n "+failedResourceMapIds);
303 309 
        log.info("the metacat ids (except the resource map ids)-----------------------------"+otherMetacatIds);
304 310 
        //logFile(otherMetacatIds, "ids-for-timed-indexing-log");
305 311 
        //log.info("the deleted metacat ids (except the resource map ids)-----------------------------"+otherDeletedMetacatIds);
......
352 358 
        if(metacatIds != null) {
353 359 
            for(String metacatId : metacatIds) {
354 360 
                if(metacatId != null) {
355 
                        try {
356 
                            generateIndex(metacatId);
357 
                        } catch (Exception e) {
358 
                            IndexEvent event = new IndexEvent();
359 
                            Identifier pid = new Identifier();
360 
                            pid.setValue(metacatId);
361 
                            event.setIdentifier(pid);
362 
                            event.setDate(Calendar.getInstance().getTime());
363 
                            event.setAction(Event.CREATE);
364 
                            String error = "IndexGenerator.index - Metacat Index couldn't generate the index for the id - "+metacatId+" because "+e.getMessage();
365 
                            event.setDescription(error);
366 
                            try {
367 
                                EventlogFactory.createIndexEventLog().write(event);
368 
                            } catch (Exception ee) {
369 
                                log.error("SolrIndex.insertToIndex - IndexEventLog can't log the index inserting event :"+ee.getMessage());
370 
                            }
371 
                            log.error(error);
372 
                        }
373
374
361 
                     generateIndex(metacatId);
375 362 
                }
376 363 
            }
377 364 
        }
......
380 367 
    /*
381 368 
     * Index those ids which failed in the process (We got them from the EventLog)
382 369 
     */
383 
    private void indexFailedIds(List<IndexEvent> events) {
370 
    /*private void indexFailedIds(List<IndexEvent> events) {
384 371 
        if(events != null) {
385 372 
            for(IndexEvent event : events) {
386 373 
                if(event != null) {
......
396 383 
                                } catch (Exception e) {
397 384 
                                    log.error("IndexGenerator.indexFailedIds - Metacat Index couldn't generate the index for the id - "+id+" because "+e.getMessage());
398 385 
                                }
399 
                            /*} else if (action != null && action.equals(Event.DELETE)) {
400 
                                try {
401 
                                    removeIndex(id);
402 
                                    EventlogFactory.createIndexEventLog().remove(identifier);
403 
                                } catch (Exception e) {
404 
                                    log.error("IndexGenerator.indexFailedIds - Metacat Index couldn't remove the index for the id - "+id+" because "+e.getMessage());
405 
                                }
406 
                            }*/
386
407 387 
                        }
408 388 
                    }
409 389 
                }
410 390 
            }
411 391 
        }
412 
    }
392 
    }*/
413 393
414 394 
    public void run() {
415 395 
        /*IndexEvent event = new IndexEvent();
......
493 473 
        }
494 474 
    }
495 475
496 
    /*
497 
     * Get the indexed ids list from the solr server.
498 
     * An empty list will be returned if there is no ids.
499 
     */
500 
    private List<String> getSolrDocIds() throws SolrServerException {
501 
        List<String> ids = solrIndex.getSolrIds();
502 
        return ids;
503 
    }
476
504 477
505 478 
    /*
506 479 
     * Get an array of the list of ids of the metacat which has the systemmetadata modification in the range.
......
646 619 
    /*
647 620 
     * Generate index for the id.
648 621 
     */
649 
    private void generateIndex(String id) throws Exception {
622 
    private void generateIndex(String id)  {
650 623 
        if(id != null)  {
651 624 
                SystemMetadata sysmeta = getSystemMetadata(id);
652 625 
                //only update none-archived id.
653 626 
                //if(sysmeta != null && !sysmeta.getArchived() && sysmeta.getObsoletedBy() == null) {
654 
                 if(sysmeta != null) {
627 
                try {
628 
                    if(sysmeta != null) {
655 629 
                        InputStream data = getDataObject(id);
656 630 
                        /*Identifier obsolete = sysmeta.getObsoletes();
657 631 
                        List<String> obsoleteChain = null;
......
659 633 
                            obsoleteChain = getObsoletes(id);
660 634 
                        }*/
661 635 
                        solrIndex.update(id, sysmeta, data);
662 
                } else {
663 
                    throw new Exception("IndexGenerator.generate - there is no found SystemMetadata associated with the id "+id);
636 
                        try {
637 
                            Identifier identifier = new Identifier();
638 
                            identifier.setValue(id);
639 
                            EventlogFactory.createIndexEventLog().remove(identifier);
640 
                        } catch (Exception ee) {
641 
                            log.error("IndexGenerator.index - can't remove the id "+id +" from the index_event table since - "+ee.getMessage());
642 
                        }
643 
                    } else {
644 
                        throw new Exception("IndexGenerator.generate - there is no found SystemMetadata associated with the id "+id);
645 
                    }
646 
                } catch (Exception e) {
647 
                    IndexEvent event = new IndexEvent();
648 
                    Identifier pid = new Identifier();
649 
                    pid.setValue(id);
650 
                    event.setIdentifier(pid);
651 
                    event.setDate(Calendar.getInstance().getTime());
652 
                    event.setAction(Event.CREATE);
653 
                    String error = "IndexGenerator.index - Metacat Index couldn't generate the index for the id - "+id+" because "+e.getMessage();
654 
                    event.setDescription(error);
655 
                    try {
656 
                        EventlogFactory.createIndexEventLog().write(event);
657 
                    } catch (Exception ee) {
658 
                        log.error("SolrIndex.insertToIndex - IndexEventLog can't log the index inserting event :"+ee.getMessage());
659 
                    }
660 
                    log.error(error);
664 661 
                }
662
665 663
666 664 
        }
667 665 
    }
......
722 720 
            objectPathMap = DistributedMapsFactory.getObjectPathMap();
723 721 
        }
724 722 
    }
723
724
725
726 
    /*
727 
     * Initialize the index queue
728 
     */
729 
    private void initIndexQueue() throws FileNotFoundException, ServiceFailure {
730 
        if(indexQueue == null) {
731 
            indexQueue = DistributedMapsFactory.getIndexQueue();
732 
        }
733 
    }
725 734 
    /**
726 735 
     * Get an InputStream as the data object for the specific pid.
727 736 
     * @param pid

Also available in: Unified diff