Project

General

Profile

« Previous | Next » 

Revision 8291

Added by Jing Tao about 11 years ago

The IndexGenerator will index the obsoleted data objects as well.

View differences:

metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/ApplicationController.java
211 211 
            //indexThread.start();
212 212 
            Timer indexTimer = new Timer();
213 213 
            //indexTimer.scheduleAtFixedRate(generator, Calendar.getInstance().getTime(), period);
214 
            indexTimer.schedule(generator, Calendar.getInstance().getTime(), period);
214 
            indexTimer.schedule(generator, 60000, period);
215 215 
        }
216 216
217 217 
    }
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SystemMetadataEventListener.java
205 205 
            event.setAction(Event.CREATE);
206 206 
	    }
207 207 
	    else if(systemMetadata.getArchived() || systemMetadata.getObsoletedBy() != null) {
208 
            action = Event.DELETE.xmlValue();
209 
            event.setAction(Event.DELETE);
208 
            action = Event.UPDATE.xmlValue();
209 
            event.setAction(Event.UPDATE);
210 210 
        } else {
211 211 
            action = Event.CREATE.xmlValue();
212 212 
            event.setAction(Event.CREATE);
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/IndexGenerator.java
220 220 
        List[] metacatIds = getMetacatIds(since, until);
221 221 
        List<String> otherMetacatIds = metacatIds[FIRST];
222 222 
        List<String> resourceMapIds =  metacatIds[SECOND];
223 
        List<String> otherDeletedMetacatIds = metacatIds[THIRD];
224 
        List<String> resourceMapDeletedIds = metacatIds[FOURTH];
223 
        //List<String> otherDeletedMetacatIds = metacatIds[THIRD];
224 
        //List<String> resourceMapDeletedIds = metacatIds[FOURTH];
225 225
226 226 
        //figure out the procesedDate by comparing the last element of otherMetacatIds and resourceMapIds.
227 227 
        List<Long> maxCollection = new ArrayList<Long>();
......
234 234 
            maxCollection.add(new Long(latestOtherId.getTime()));
235 235 
        }
236 236
237 
        Date latestDeletedOtherIds = null;
237 
        /*Date latestDeletedOtherIds = null;
238 238 
        if (otherDeletedMetacatIds != null && !otherDeletedMetacatIds.isEmpty()) {
239 239 
            int size = otherDeletedMetacatIds.size();
240 240 
            String id = otherDeletedMetacatIds.get(size-1);
241 241 
            SystemMetadata sysmeta = getSystemMetadata(id);
242 242 
            latestDeletedOtherIds = sysmeta.getDateSysMetadataModified();
243 243 
            maxCollection.add(new Long(latestDeletedOtherIds.getTime()));
244 
        }
244 
        }*/
245 245
246 246 
        Date latestResourceId = null;
247 247 
        if (resourceMapIds != null && !resourceMapIds.isEmpty()) {
......
252 252 
            maxCollection.add(new Long(latestResourceId.getTime()));
253 253 
        }
254 254
255 
        Date latestDeletedResourceId = null;
255 
        /*Date latestDeletedResourceId = null;
256 256 
        if(resourceMapDeletedIds != null && !resourceMapDeletedIds.isEmpty()) {
257 257 
            int size = resourceMapDeletedIds.size();
258 258 
            String id = resourceMapDeletedIds.get(size-1);
259 259 
            SystemMetadata sysmeta = getSystemMetadata(id);
260 260 
            latestDeletedResourceId = sysmeta.getDateSysMetadataModified();
261 261 
            maxCollection.add(new Long(latestDeletedResourceId.getTime()));
262 
        }
262 
        }*/
263 263
264 264 
        if(!maxCollection.isEmpty()) {
265 265 
            Long max = Collections.max(maxCollection);
......
311 311
312 312 
        log.info("the metacat ids (except the resource map ids)-----------------------------"+otherMetacatIds);
313 313 
        //logFile(otherMetacatIds, "ids-for-timed-indexing-log");
314 
        log.info("the deleted metacat ids (except the resource map ids)-----------------------------"+otherDeletedMetacatIds);
314 
        //log.info("the deleted metacat ids (except the resource map ids)-----------------------------"+otherDeletedMetacatIds);
315 315 
        log.info("the metacat resroucemap ids -----------------------------"+resourceMapIds);
316 316 
        //logFile(resourceMapIds, "ids-for-timed-indexing-log");
317 
        log.info("the deleted metacat resroucemap ids -----------------------------"+resourceMapDeletedIds);
317 
        //log.info("the deleted metacat resroucemap ids -----------------------------"+resourceMapDeletedIds);
318 318 
        index(otherMetacatIds);
319 
        removeIndex(otherDeletedMetacatIds);
319 
        //removeIndex(otherDeletedMetacatIds);
320 320 
        index(resourceMapIds);
321 
        removeIndex(resourceMapDeletedIds);
321 
        //removeIndex(resourceMapDeletedIds);
322 322
323 323 
        //record the timed index.
324 324 
        if(processedDate != null) {
......
398 398 
                        String id = identifier.getValue();
399 399 
                        if(id != null) {
400 400 
                            Event action = event.getAction();
401 
                            if (action != null && action.equals(Event.CREATE)) {
401 
                            //if (action != null && action.equals(Event.CREATE)) {
402 402 
                                try {
403 403 
                                    generateIndex(id);
404 404 
                                    EventlogFactory.createIndexEventLog().remove(identifier);
405 405 
                                } catch (Exception e) {
406 406 
                                    log.error("IndexGenerator.indexFailedIds - Metacat Index couldn't generate the index for the id - "+id+" because "+e.getMessage());
407 407 
                                }
408 
                            } else if (action != null && action.equals(Event.DELETE)) {
408 
                            /*} else if (action != null && action.equals(Event.DELETE)) {
409 409 
                                try {
410 410 
                                    removeIndex(id);
411 411 
                                    EventlogFactory.createIndexEventLog().remove(identifier);
412 412 
                                } catch (Exception e) {
413 413 
                                    log.error("IndexGenerator.indexFailedIds - Metacat Index couldn't remove the index for the id - "+id+" because "+e.getMessage());
414 414 
                                }
415 
                            }
415 
                            }*/
416 416 
                        }
417 417 
                    }
418 418 
                }
......
523 523 
                        InvalidToken, NotAuthorized, NotImplemented, ServiceFailure, FileNotFoundException {
524 524 
        String fileName = "ids-from-hazelcast";
525 525 
        List<String> resourceMapIds = new ArrayList();
526 
        List<String> resourceMapDeletedIds = new ArrayList();
526 
        //List<String> resourceMapDeletedIds = new ArrayList();
527 527 
        List<String> otherIds = new ArrayList();
528 
        List<String> otherDeletedIds = new ArrayList();
529 
        List[] ids = new List[4];
528 
        //List<String> otherDeletedIds = new ArrayList();
529 
        List[] ids = new List[2];
530 530 
        ids[FIRST]= otherIds;
531 531 
        ids[SECOND] = resourceMapIds;
532 
        ids[THIRD]  = otherDeletedIds;
533 
        ids[FOURTH] = resourceMapDeletedIds;
532 
        //ids[THIRD]  = otherDeletedIds;
533 
        //ids[FOURTH] = resourceMapDeletedIds;
534 534 
        ISet<Identifier> metacatIds = DistributedMapsFactory.getIdentifiersSet();
535 535 
        Date otherPreviousDate = null;
536 536 
        Date otherDeletedPreviousDate = null;
......
566 566 
                        }
567 567 
                        if(correctTimeRange && formatId != null && formatId.getValue() != null && resourceMapNamespaces != null && isResourceMap(formatId)) {
568 568 
                            //for the resource map
569 
                            if(sysmeta.getArchived() || sysmeta.getObsoletedBy() != null) {
569 
                            /*if(sysmeta.getArchived() || sysmeta.getObsoletedBy() != null) {
570 570 
                                //archived ids
571 571 
                                if(!resourceMapDeletedIds.isEmpty()) {
572 572 
                                    if(sysDate.getTime() > resourceMapDeletedPreviousDate.getTime()) {
......
580 580 
                                    resourceMapDeletedIds.add(identifier.getValue());
581 581 
                                    resourceMapDeletedPreviousDate = sysDate;//init resourcemapPreviousDate
582 582 
                                }
583 
                            } else {
584 
                                // current ids
583 
                            } else {*/
584 
                                // for all ids
585 585 
                                if(!resourceMapIds.isEmpty()) {
586 586 
                                    if(sysDate.getTime() > resourceMapPreviousDate.getTime()) {
587 587 
                                        resourceMapIds.add(identifier.getValue());//append to the end of the list if current is later than the previous one
......
594 594 
                                    resourceMapIds.add(identifier.getValue());
595 595 
                                    resourceMapPreviousDate = sysDate;//init resourcemapPreviousDate
596 596 
                                }
597 
                            }
597 
                            //}
598 598 
                        } else if (correctTimeRange) {
599 
                            if(sysmeta.getArchived() || sysmeta.getObsoletedBy() != null) {
599 
                            /*if(sysmeta.getArchived() || sysmeta.getObsoletedBy() != null) {
600 600 
                                //for the archived ids
601 601 
                                if(!otherDeletedIds.isEmpty()) {
602 602 
                                    if(sysDate.getTime() > otherDeletedPreviousDate.getTime()) {
......
610 610 
                                    otherDeletedIds.add(identifier.getValue());
611 611 
                                    otherDeletedPreviousDate = sysDate;//init otherDeletedPreviousDate
612 612 
                                }
613 
                            } else {
614 
                                //for the current ids
613 
                            } else {*/
614 
                                //for all ids
615 615 
                                if(!otherIds.isEmpty()) {
616 616 
                                    if(sysDate.getTime() > otherPreviousDate.getTime()) {
617 617 
                                        otherIds.add(identifier.getValue());
......
624 624 
                                    otherIds.add(identifier.getValue());
625 625 
                                    otherPreviousDate = sysDate;//init otherPreviousDate
626 626 
                                }
627 
                            }
627 
                            //}
628 628 
                        }
629 629
630 630 
                    }
......
659 659 
        if(id != null)  {
660 660 
                SystemMetadata sysmeta = getSystemMetadata(id);
661 661 
                //only update none-archived id.
662 
                if(sysmeta != null && !sysmeta.getArchived() && sysmeta.getObsoletedBy() == null) {
662 
                //if(sysmeta != null && !sysmeta.getArchived() && sysmeta.getObsoletedBy() == null) {
663 
                 if(sysmeta != null) {
663 664 
                        InputStream data = getDataObject(id);
664 665 
                        /*Identifier obsolete = sysmeta.getObsoletes();
665 666 
                        List<String> obsoleteChain = null;
......
677 678 
    /*
678 679 
     * Remove the solr index for the list of ids
679 680 
     */
680 
    private void removeIndex(List<String> ids) {
681 
    /*private void removeIndex(List<String> ids) {
681 682 
        if(ids!= null) {
682 683 
            for(String id :ids) {
683 684 
                try {
......
701 702
702 703 
            }
703 704 
        }
704 
    }
705 
    }*/
705 706
706 707 
    /*
707 708 
     * Remove the index for the id
708 709 
     */
709 
    private void removeIndex(String id) throws ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, IOException, SolrServerException, SAXException, ParserConfigurationException, OREParserException  {
710 
    /*private void removeIndex(String id) throws ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, IOException, SolrServerException, SAXException, ParserConfigurationException, OREParserException  {
710 711 
        if(id != null) {
711 712 
            //solrIndex.remove(id);
712 713 
        }
713 
    }
714 
    }*/
714 715
715 716 
    /*
716 717 
     * Initialize the system metadata map

Also available in: Unified diff