/ - Diff - Metacat - Ecoinformatics Redmine

« Previous | Next »

Revision 7786

Added by Jing Tao over 11 years ago

Add code to handle delete data package information when delete a pid in the solr index.

         /*
          * Get the SolrDoc for the specified id
          */
         private static SolrDoc getSolrDoc(String id) throws SolrServerException, IOException, ParserConfigurationException, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType {
         public static SolrDoc getSolrDoc(String id) throws SolrServerException, IOException, ParserConfigurationException, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType {
             SolrDoc solrDoc = null;
             if(solrServer != null) {
                String query = QUERY+"\""+id+"\"";

     package edu.ucsb.nceas.metacat.index;
     import java.io.ByteArrayInputStream;
     import java.io.FileNotFoundException;
     import java.io.IOException;
     import java.io.InputStream;
     import java.util.ArrayList;
-...
     import org.apache.commons.codec.EncoderException;
     import org.apache.commons.io.output.ByteArrayOutputStream;
     import org.apache.commons.lang.StringUtils;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import org.apache.solr.client.solrj.SolrQuery;
-...
     import org.apache.solr.common.SolrDocumentList;
     import org.apache.solr.common.SolrInputDocument;
     import org.apache.solr.common.util.NamedList;
     import org.dataone.cn.indexer.XMLNamespaceConfig;
     import org.dataone.cn.indexer.parser.IDocumentSubprocessor;
     import org.dataone.cn.indexer.parser.SolrField;
     import org.dataone.cn.indexer.resourcemap.ResourceEntry;
     import org.dataone.cn.indexer.resourcemap.ResourceMap;
     import org.dataone.cn.indexer.solrhttp.SolrDoc;
     import org.dataone.cn.indexer.solrhttp.SolrElementAdd;
     import org.dataone.cn.indexer.solrhttp.SolrElementField;
     import org.dataone.service.exceptions.NotFound;
     import org.dataone.service.exceptions.NotImplemented;
     import org.dataone.service.exceptions.ServiceFailure;
     import org.dataone.service.exceptions.UnsupportedType;
     import org.dataone.service.types.v1.Identifier;
     import org.dataone.service.types.v1.SystemMetadata;
-...
             if(docs != null) {
                 Set<String> ids = docs.keySet();
                 for(String id : ids) {
                     SolrInputDocument solrDoc = new SolrInputDocument();
                     if(id != null) {
                         SolrDoc doc = docs.get(id);
                         if(doc != null) {
                             List<SolrElementField> list = doc.getFieldList();
                             if(list != null) {
                                 //solrDoc.addField(METACATPIDFIELD, pid);
                                 Iterator<SolrElementField> iterator = list.iterator();
                                 while (iterator.hasNext()) {
                                     SolrElementField field = iterator.next();
                                     if(field != null) {
                                         String value = field.getValue();
                                         String name = field.getName();
                                         //System.out.println("add name/value pair - "+name+"/"+value);
                                         solrDoc.addField(name, value);
+                                    }
+                                }
+                            }
                         insertToIndex(doc);
+                    }
+                }
+            }
+        }
         /*
          * Insert a SolrDoc to the solr server.
          */
         private synchronized void insertToIndex(SolrDoc doc) throws SolrServerException, IOException {
             if(doc != null ) {
                 SolrInputDocument solrDoc = new SolrInputDocument();
                 List<SolrElementField> list = doc.getFieldList();
                 if(list != null) {
                     //solrDoc.addField(METACATPIDFIELD, pid);
                     Iterator<SolrElementField> iterator = list.iterator();
                     while (iterator.hasNext()) {
                         SolrElementField field = iterator.next();
                         if(field != null) {
                             String value = field.getValue();
                             String name = field.getName();
                             //System.out.println("add name/value pair - "+name+"/"+value);
                             solrDoc.addField(name, value);
+                        }
+                    }
                     if(!solrDoc.isEmpty()) {
                         UpdateResponse response = solrServer.add(solrDoc);
                         solrServer.commit();
                         //System.out.println("=================the response is:\n"+response.toString());
+                    }
+                }
                 if(!solrDoc.isEmpty()) {
                     UpdateResponse response = solrServer.add(solrDoc);
                     solrServer.commit();
                     //System.out.println("=================the response is:\n"+response.toString());
+                }
+            }
+        }
-...
          * @throws UnsupportedType
          * @throws NotFound
          * @throws NotImplemented
          * @throws ServiceFailure
          */
         public void update(String pid, List<String> obsoleteIds, SystemMetadata systemMetadata, InputStream data)
                         throws IOException, SAXException, ParserConfigurationException,
                         XPathExpressionException, SolrServerException, JiBXException, EncoderException, NotImplemented, NotFound, UnsupportedType {
                         XPathExpressionException, SolrServerException, JiBXException, EncoderException, NotImplemented, NotFound, UnsupportedType, ServiceFailure {
             checkParams(pid, systemMetadata, data);
             boolean isArchive = systemMetadata.getArchived();
             if(isArchive) {
-...
+        }
         private void removeObsoletesChain(String obsoleteId, List<String> obsoleteIdChain) throws SolrServerException, IOException {
         private void removeObsoletesChain(String obsoleteId, List<String> obsoleteIdChain) throws SolrServerException, IOException, ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SAXException, ParserConfigurationException {
             if(obsoleteId != null && !obsoleteId.trim().equals("")) {
                 if(obsoleteIdChain == null || obsoleteIdChain.isEmpty()) {
                     throw new SolrServerException("SolrIndex.removeObsoletesChain - The obsoletes chain can't be null or empty since the system metadata already has the obsoletes element.");
-...
          * @param pidList
          * @throws IOException
          * @throws SolrServerException
          * @throws ParserConfigurationException
          * @throws SAXException
          * @throws UnsupportedType
          * @throws NotFound
          * @throws NotImplemented
          * @throws XPathExpressionException
          * @throws ServiceFailure
          */
         private void remove(List<String> pidList) throws IOException, SolrServerException {
         private void remove(List<String> pidList) throws IOException, SolrServerException, ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SAXException, ParserConfigurationException {
             if(pidList != null) {
                 for(String id : pidList) {
                     remove(id);
-...
          * @param pid  the pid which the indexes are associated with
          * @throws IOException
          * @throws SolrServerException
          * @throws ParserConfigurationException
          * @throws SAXException
          * @throws UnsupportedType
          * @throws NotFound
          * @throws NotImplemented
          * @throws XPathExpressionException
          * @throws ServiceFailure
          */
         public void remove(String pid) throws IOException, SolrServerException {
         public void remove(String pid) throws IOException, SolrServerException, ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SAXException, ParserConfigurationException {
             if (isDataPackage(pid)) {
                 removeDataPackage(pid);
             } else if (isPartOfDataPackage(pid)) {
                 removeFromDataPackage(pid);
             } else {
                 removeFromIndex(pid);
+            }
+        }
         /*
          * Remove a resource map pid
          */
         private void removeDataPackage(String pid) throws ServiceFailure, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException  {
             Document resourceMapDoc = generateXmlDocument(DistributedMapsFactory.getDataObject(pid));
             ResourceMap resourceMap = new ResourceMap(resourceMapDoc);
             List<String> documentIds = resourceMap.getAllDocumentIDs();
             List<SolrDoc> indexDocuments =ResourceMapSubprocessor.getSolrDocs(documentIds);
             removeFromIndex(pid);
             //List<SolrDoc> docsToUpdate = new ArrayList<SolrDoc>();
             // for each document in data package:
             for (SolrDoc indexDoc : indexDocuments) {
                 if (indexDoc.getIdentifier().equals(pid)) {
                     continue; // skipping the resource map, no need update
                               // it.
                               // will
                               // be removed.
+                }
                 // Remove resourceMap reference
                 indexDoc.removeFieldsWithValue(SolrElementField.FIELD_RESOURCEMAP,
                         resourceMap.getIdentifier());
                 // // Remove documents/documentedby values for this resource
                 // map
                 for (ResourceEntry entry : resourceMap.getMappedReferences()) {
                     if (indexDoc.getIdentifier().equals(entry.getIdentifier())) {
                         for (String documentedBy : entry.getDocumentedBy()) {
                             // Using removeOneFieldWithValue in-case same
                             // documents
                             // are in more than one data package. just
                             // remove
                             // one
                             // instance of data package info.
                             indexDoc.removeOneFieldWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY,
                                     documentedBy);
+                        }
                         for (String documents : entry.getDocuments()) {
                             indexDoc.removeOneFieldWithValue(SolrElementField.FIELD_DOCUMENTS,
                                     documents);
+                        }
                         break;
+                    }
+                }
                 removeFromIndex(indexDoc.getIdentifier());
                 insertToIndex(indexDoc);
                 //docsToUpdate.add(indexDoc);
+            }
             //SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate);
             //httpService.sendUpdate(solrIndexUri, addCommand);
+        }
         private void removeFromDataPackage(String pid) throws XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException, SAXException  {
             SolrDoc indexedDoc = ResourceMapSubprocessor.getSolrDoc(pid);
             removeFromIndex(pid);
             List<SolrDoc> docsToUpdate = new ArrayList<SolrDoc>();
             List<String> documents = indexedDoc.getAllFieldValues(SolrElementField.FIELD_DOCUMENTS);
             for (String documentsValue : documents) {
                 SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentsValue);
                 solrDoc.removeFieldsWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY, pid);
                 removeFromIndex(documentsValue);
                 insertToIndex(solrDoc);
+            }
             List<String> documentedBy = indexedDoc
                     .getAllFieldValues(SolrElementField.FIELD_ISDOCUMENTEDBY);
             for (String documentedByValue : documentedBy) {
                 SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentedByValue);
                 solrDoc.removeFieldsWithValue(SolrElementField.FIELD_DOCUMENTS, documentedByValue);
                 //docsToUpdate.add(solrDoc);
                 removeFromIndex(documentedByValue);
                 insertToIndex(solrDoc);
+            }
             //SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate);
             //httpService.sendUpdate(solrIndexUri, addCommand);
+        }
         /*
          * Remove a pid from the solr index
          */
         private void removeFromIndex(String pid) throws SolrServerException, IOException {
             if(pid != null && !pid.trim().equals("")) {
                 solrServer.deleteById(pid);
                 solrServer.commit();
+            }
+        }
         /*
          * Is the pid a resource map
          */
         private boolean isDataPackage(String pid) throws FileNotFoundException, ServiceFailure {
             boolean isDataPackage = false;
             SystemMetadata sysmeta = DistributedMapsFactory.getSystemMetadata(pid);
             if(sysmeta != null) {
                 isDataPackage = IndexGenerator.isResourceMap(sysmeta.getFormatId());
+            }
             return isDataPackage;
+        }
         private boolean isPartOfDataPackage(String pid) throws XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException, SAXException {
             SolrDoc dataPackageIndexDoc = ResourceMapSubprocessor.getSolrDoc(pid);
             if (dataPackageIndexDoc != null) {
                 String resourceMapId = dataPackageIndexDoc
                         .getFirstFieldValue(SolrElementField.FIELD_RESOURCEMAP);
                 return StringUtils.isNotEmpty(resourceMapId);
             } else {
                 return false;
+            }
+        }
         /**
          * Get the solrServer
          * @return

         private IMap<Identifier, String> objectPathMap;
         private Log log = LogFactory.getLog(IndexGenerator.class);
         private MNode mNode = null;
         private List<String> resourceMapNamespaces = null;
         private static List<String> resourceMapNamespaces = null;
         /**
          * Constructor
-...
         /*
          * If the specified ObjectFormatIdentifier is a resrouce map namespace.
          */
         private boolean isResourceMap(ObjectFormatIdentifier formatId) {
         public static boolean isResourceMap(ObjectFormatIdentifier formatId) {
             boolean isResourceMap = false;
             if(formatId != null && resourceMapNamespaces != null) {
                 for(String namespace : resourceMapNamespaces) {

Also available in: Unified diff

Project

General

Profile

Metacat

Revision 7786

Added by Jing Tao over 11 years ago