Project

General

Profile

« Previous | Next » 

Revision 7786

Added by Jing Tao almost 11 years ago

Add code to handle delete data package information when delete a pid in the solr index.

View differences:

metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/resourcemap/ResourceMapSubprocessor.java
162 162
    /*
163 163
     * Get the SolrDoc for the specified id 
164 164
     */
165
    private static SolrDoc getSolrDoc(String id) throws SolrServerException, IOException, ParserConfigurationException, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType {
165
    public static SolrDoc getSolrDoc(String id) throws SolrServerException, IOException, ParserConfigurationException, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType {
166 166
        SolrDoc solrDoc = null;
167 167
        if(solrServer != null) {
168 168
           String query = QUERY+"\""+id+"\"";
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java
27 27
package edu.ucsb.nceas.metacat.index;
28 28

  
29 29
import java.io.ByteArrayInputStream;
30
import java.io.FileNotFoundException;
30 31
import java.io.IOException;
31 32
import java.io.InputStream;
32 33
import java.util.ArrayList;
......
45 46

  
46 47
import org.apache.commons.codec.EncoderException;
47 48
import org.apache.commons.io.output.ByteArrayOutputStream;
49
import org.apache.commons.lang.StringUtils;
48 50
import org.apache.commons.logging.Log;
49 51
import org.apache.commons.logging.LogFactory;
50 52
import org.apache.solr.client.solrj.SolrQuery;
......
56 58
import org.apache.solr.common.SolrDocumentList;
57 59
import org.apache.solr.common.SolrInputDocument;
58 60
import org.apache.solr.common.util.NamedList;
61

  
59 62
import org.dataone.cn.indexer.XMLNamespaceConfig;
60 63
import org.dataone.cn.indexer.parser.IDocumentSubprocessor;
61 64
import org.dataone.cn.indexer.parser.SolrField;
65
import org.dataone.cn.indexer.resourcemap.ResourceEntry;
66
import org.dataone.cn.indexer.resourcemap.ResourceMap;
62 67
import org.dataone.cn.indexer.solrhttp.SolrDoc;
68
import org.dataone.cn.indexer.solrhttp.SolrElementAdd;
63 69
import org.dataone.cn.indexer.solrhttp.SolrElementField;
64 70
import org.dataone.service.exceptions.NotFound;
65 71
import org.dataone.service.exceptions.NotImplemented;
72
import org.dataone.service.exceptions.ServiceFailure;
66 73
import org.dataone.service.exceptions.UnsupportedType;
67 74
import org.dataone.service.types.v1.Identifier;
68 75
import org.dataone.service.types.v1.SystemMetadata;
......
353 360
        if(docs != null) {
354 361
            Set<String> ids = docs.keySet();
355 362
            for(String id : ids) {
356
                SolrInputDocument solrDoc = new SolrInputDocument();
357 363
                if(id != null) {
358 364
                    SolrDoc doc = docs.get(id);
359
                    if(doc != null) {
360
                        List<SolrElementField> list = doc.getFieldList();
361
                        if(list != null) {
362
                            //solrDoc.addField(METACATPIDFIELD, pid);
363
                            Iterator<SolrElementField> iterator = list.iterator();
364
                            while (iterator.hasNext()) {
365
                                SolrElementField field = iterator.next();
366
                                if(field != null) {
367
                                    String value = field.getValue();
368
                                    String name = field.getName();
369
                                    //System.out.println("add name/value pair - "+name+"/"+value);
370
                                    solrDoc.addField(name, value);
371
                                }
372
                            }
373
                        }
365
                    insertToIndex(doc);
366
                }
367
                
368
            }
369
        }
370
    }
371
    
372
    /*
373
     * Insert a SolrDoc to the solr server.
374
     */
375
    private synchronized void insertToIndex(SolrDoc doc) throws SolrServerException, IOException {
376
        if(doc != null ) {
377
            SolrInputDocument solrDoc = new SolrInputDocument();
378
            List<SolrElementField> list = doc.getFieldList();
379
            if(list != null) {
380
                //solrDoc.addField(METACATPIDFIELD, pid);
381
                Iterator<SolrElementField> iterator = list.iterator();
382
                while (iterator.hasNext()) {
383
                    SolrElementField field = iterator.next();
384
                    if(field != null) {
385
                        String value = field.getValue();
386
                        String name = field.getName();
387
                        //System.out.println("add name/value pair - "+name+"/"+value);
388
                        solrDoc.addField(name, value);
374 389
                    }
375 390
                }
376
                if(!solrDoc.isEmpty()) {
377
                    UpdateResponse response = solrServer.add(solrDoc);
378
                    solrServer.commit();
379
                    //System.out.println("=================the response is:\n"+response.toString());
380
                }
381 391
            }
392
            if(!solrDoc.isEmpty()) {
393
                UpdateResponse response = solrServer.add(solrDoc);
394
                solrServer.commit();
395
                //System.out.println("=================the response is:\n"+response.toString());
396
            }
382 397
        }
383 398
    }
384 399
    
......
400 415
     * @throws UnsupportedType 
401 416
     * @throws NotFound 
402 417
     * @throws NotImplemented 
418
     * @throws ServiceFailure 
403 419
     */
404 420
    public void update(String pid, List<String> obsoleteIds, SystemMetadata systemMetadata, InputStream data) 
405 421
                    throws IOException, SAXException, ParserConfigurationException,
406
                    XPathExpressionException, SolrServerException, JiBXException, EncoderException, NotImplemented, NotFound, UnsupportedType {
422
                    XPathExpressionException, SolrServerException, JiBXException, EncoderException, NotImplemented, NotFound, UnsupportedType, ServiceFailure {
407 423
        checkParams(pid, systemMetadata, data);
408 424
        boolean isArchive = systemMetadata.getArchived();
409 425
        if(isArchive) {
......
426 442
    }
427 443
    
428 444
    
429
    private void removeObsoletesChain(String obsoleteId, List<String> obsoleteIdChain) throws SolrServerException, IOException {
445
    private void removeObsoletesChain(String obsoleteId, List<String> obsoleteIdChain) throws SolrServerException, IOException, ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SAXException, ParserConfigurationException {
430 446
        if(obsoleteId != null && !obsoleteId.trim().equals("")) {
431 447
            if(obsoleteIdChain == null || obsoleteIdChain.isEmpty()) {
432 448
                throw new SolrServerException("SolrIndex.removeObsoletesChain - The obsoletes chain can't be null or empty since the system metadata already has the obsoletes element."); 
......
445 461
     * @param pidList
446 462
     * @throws IOException
447 463
     * @throws SolrServerException
464
     * @throws ParserConfigurationException 
465
     * @throws SAXException 
466
     * @throws UnsupportedType 
467
     * @throws NotFound 
468
     * @throws NotImplemented 
469
     * @throws XPathExpressionException 
470
     * @throws ServiceFailure 
448 471
     */
449
    private void remove(List<String> pidList) throws IOException, SolrServerException {
472
    private void remove(List<String> pidList) throws IOException, SolrServerException, ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SAXException, ParserConfigurationException {
450 473
        if(pidList != null) {
451 474
            for(String id : pidList) {
452 475
                remove(id);
......
459 482
     * @param pid  the pid which the indexes are associated with
460 483
     * @throws IOException
461 484
     * @throws SolrServerException
485
     * @throws ParserConfigurationException 
486
     * @throws SAXException 
487
     * @throws UnsupportedType 
488
     * @throws NotFound 
489
     * @throws NotImplemented 
490
     * @throws XPathExpressionException 
491
     * @throws ServiceFailure 
462 492
     */
463
    public void remove(String pid) throws IOException, SolrServerException {
493
    public void remove(String pid) throws IOException, SolrServerException, ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SAXException, ParserConfigurationException {
494
        if (isDataPackage(pid)) {
495
            removeDataPackage(pid);
496
        } else if (isPartOfDataPackage(pid)) {
497
            removeFromDataPackage(pid);
498
        } else {
499
            removeFromIndex(pid);
500
        }
501
    }
502
    
503
   
504
   
505

  
506
    /*
507
     * Remove a resource map pid
508
     */
509
    private void removeDataPackage(String pid) throws ServiceFailure, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException  {
510
        Document resourceMapDoc = generateXmlDocument(DistributedMapsFactory.getDataObject(pid));
511
        ResourceMap resourceMap = new ResourceMap(resourceMapDoc);
512
        List<String> documentIds = resourceMap.getAllDocumentIDs();
513
        List<SolrDoc> indexDocuments =ResourceMapSubprocessor.getSolrDocs(documentIds);
514
        removeFromIndex(pid);
515
        //List<SolrDoc> docsToUpdate = new ArrayList<SolrDoc>();
516
        // for each document in data package:
517
        for (SolrDoc indexDoc : indexDocuments) {
518

  
519
            if (indexDoc.getIdentifier().equals(pid)) {
520
                continue; // skipping the resource map, no need update
521
                          // it.
522
                          // will
523
                          // be removed.
524
            }
525

  
526
            // Remove resourceMap reference
527
            indexDoc.removeFieldsWithValue(SolrElementField.FIELD_RESOURCEMAP,
528
                    resourceMap.getIdentifier());
529

  
530
            // // Remove documents/documentedby values for this resource
531
            // map
532
            for (ResourceEntry entry : resourceMap.getMappedReferences()) {
533
                if (indexDoc.getIdentifier().equals(entry.getIdentifier())) {
534
                    for (String documentedBy : entry.getDocumentedBy()) {
535
                        // Using removeOneFieldWithValue in-case same
536
                        // documents
537
                        // are in more than one data package. just
538
                        // remove
539
                        // one
540
                        // instance of data package info.
541
                        indexDoc.removeOneFieldWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY,
542
                                documentedBy);
543
                    }
544
                    for (String documents : entry.getDocuments()) {
545
                        indexDoc.removeOneFieldWithValue(SolrElementField.FIELD_DOCUMENTS,
546
                                documents);
547
                    }
548
                    break;
549
                }
550
            }
551
            removeFromIndex(indexDoc.getIdentifier());
552
            insertToIndex(indexDoc);
553
            //docsToUpdate.add(indexDoc);
554
        }
555
        //SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate);
556
        //httpService.sendUpdate(solrIndexUri, addCommand);
557
    }
558

  
559
    private void removeFromDataPackage(String pid) throws XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException, SAXException  {
560
        SolrDoc indexedDoc = ResourceMapSubprocessor.getSolrDoc(pid);
561
        removeFromIndex(pid);
562
        List<SolrDoc> docsToUpdate = new ArrayList<SolrDoc>();
563

  
564
        List<String> documents = indexedDoc.getAllFieldValues(SolrElementField.FIELD_DOCUMENTS);
565
        for (String documentsValue : documents) {
566
            SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentsValue);
567
            solrDoc.removeFieldsWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY, pid);
568
            removeFromIndex(documentsValue);
569
            insertToIndex(solrDoc);
570
        }
571

  
572
        List<String> documentedBy = indexedDoc
573
                .getAllFieldValues(SolrElementField.FIELD_ISDOCUMENTEDBY);
574
        for (String documentedByValue : documentedBy) {
575
            SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentedByValue);
576
            solrDoc.removeFieldsWithValue(SolrElementField.FIELD_DOCUMENTS, documentedByValue);
577
            //docsToUpdate.add(solrDoc);
578
            removeFromIndex(documentedByValue);
579
            insertToIndex(solrDoc);
580
        }
581

  
582
        //SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate);
583
        //httpService.sendUpdate(solrIndexUri, addCommand);
584
    }
585

  
586
    /*
587
     * Remove a pid from the solr index
588
     */
589
    private void removeFromIndex(String pid) throws SolrServerException, IOException {
464 590
        if(pid != null && !pid.trim().equals("")) {
465 591
            solrServer.deleteById(pid);
466 592
            solrServer.commit();
467 593
        }
468
       
469
       
470 594
    }
471 595

  
596
    /*
597
     * Is the pid a resource map
598
     */
599
    private boolean isDataPackage(String pid) throws FileNotFoundException, ServiceFailure {
600
        boolean isDataPackage = false;
601
        SystemMetadata sysmeta = DistributedMapsFactory.getSystemMetadata(pid);
602
        if(sysmeta != null) {
603
            isDataPackage = IndexGenerator.isResourceMap(sysmeta.getFormatId());
604
        }
605
        return isDataPackage;
606
    }
607

  
608
    private boolean isPartOfDataPackage(String pid) throws XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException, SAXException {
609
        SolrDoc dataPackageIndexDoc = ResourceMapSubprocessor.getSolrDoc(pid);
610
        if (dataPackageIndexDoc != null) {
611
            String resourceMapId = dataPackageIndexDoc
612
                    .getFirstFieldValue(SolrElementField.FIELD_RESOURCEMAP);
613
            return StringUtils.isNotEmpty(resourceMapId);
614
        } else {
615
            return false;
616
        }
617
    }
618

  
472 619
    /**
473 620
     * Get the solrServer
474 621
     * @return
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/IndexGenerator.java
83 83
    private IMap<Identifier, String> objectPathMap;
84 84
    private Log log = LogFactory.getLog(IndexGenerator.class);
85 85
    private MNode mNode = null;
86
    private List<String> resourceMapNamespaces = null;
86
    private static List<String> resourceMapNamespaces = null;
87 87
    
88 88
    /**
89 89
     * Constructor
......
325 325
    /*
326 326
     * If the specified ObjectFormatIdentifier is a resrouce map namespace.
327 327
     */
328
    private boolean isResourceMap(ObjectFormatIdentifier formatId) {
328
    public static boolean isResourceMap(ObjectFormatIdentifier formatId) {
329 329
        boolean isResourceMap = false;
330 330
        if(formatId != null && resourceMapNamespaces != null) {
331 331
            for(String namespace : resourceMapNamespaces) {

Also available in: Unified diff