Metacat

/**

 *  '$RCSfile$'

 *    Purpose: A class that gets Accession Number, check for uniqueness

 *             and register it into db

 *  Copyright: 2000 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Jivka Bojilova, Matt Jones

 *

 *   '$Author: leinfelder $'

 *     '$Date: 2011-11-02 20:40:12 -0700 (Wed, 02 Nov 2011) $'

 * '$Revision: 6595 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.index;

import java.io.InputStream;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.Set;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import org.apache.commons.io.output.ByteArrayOutputStream;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.common.SolrDocumentList;

import org.dataone.cn.indexer.parser.IDocumentSubprocessor;

import org.dataone.cn.indexer.resourcemap.ResourceMap;

import org.dataone.service.exceptions.NotImplemented;

import org.dataone.service.util.TypeMarshaller;

import org.jibx.runtime.JiBXException;

import edu.ucsb.nceas.metacat.index.event.IndexEvent;

 * A class does insert, update and remove indexes to a SOLR server

 * @author tao

 *

 */

public class SolrIndex {

    private static final String IDQUERY = ID+":*";

    private SolrServer solrServer = null;

    private XMLNamespaceConfig xmlNamespaceConfig = null;

    private static DocumentBuilderFactory documentBuilderFactory = null;

    private static DocumentBuilder builder = null;

    private static XPathFactory xpathFactory = null;

    private static XPath xpath = null;

        documentBuilderFactory = DocumentBuilderFactory.newInstance();

        documentBuilderFactory.setNamespaceAware(true);

        try {

            builder = documentBuilderFactory.newDocumentBuilder();

        } catch (ParserConfigurationException e) {

            e.printStackTrace();

        }

        xpathFactory = XPathFactory.newInstance();

        xpath = xpathFactory.newXPath();

    }

     * Constructor

     * @throws IOException

         this.xmlNamespaceConfig = xmlNamespaceConfig;

         this.sysmetaSolrFields = sysmetaSolrFields;

         init();

        xpath.setNamespaceContext(xmlNamespaceConfig);

        initExpressions();

    }

    private void initExpressions() throws XPathExpressionException {

        for (SolrField field : sysmetaSolrFields) {

            field.initExpression(xpath);

        }

    }

     * Get the list of the Subprocessors in this index.

     * @return the list of the Subprocessors.

     */

    public List<IDocumentSubprocessor> getSubprocessors() {

        return subprocessors;

    }

    /**

     * Set the list of Subprocessors.

     * @param subprocessorList  the list will be set.

     */

    public void setSubprocessors(List<IDocumentSubprocessor> subprocessorList) {

    }

    /**

     * Generate the index for the given information

     * @param id

     * @return

     * @throws IOException

     * @throws SAXException

     * @throws ParserConfigurationException

     * @throws XPathExpressionException

     * @throws NotFound

     * @throws NotImplemented

        // Load the System Metadata document

        TypeMarshaller.marshalTypeToOutputStream(systemMetadata, systemMetadataOutputStream);

        ByteArrayInputStream systemMetadataStream = new ByteArrayInputStream(systemMetadataOutputStream.toByteArray());

        Document sysMetaDoc = generateXmlDocument(systemMetadataStream);

            log.error("Could not load System metadata for ID: " + id);

            return null;

        }

        // Extract the field values from the System Metadata

        List<SolrElementField> sysSolrFields = processSysmetaFields(sysMetaDoc, id);

        SolrDoc indexDocument = new SolrDoc(sysSolrFields);

        Map<String, SolrDoc> docs = new HashMap<String, SolrDoc>();

        docs.put(id, indexDocument);

        // Determine if subprocessors are available for this ID

        if (subprocessors != null) {

                    // for each subprocessor loaded from the spring config

                    for (IDocumentSubprocessor subprocessor : subprocessors) {

                        // Does this subprocessor apply?

                        if (subprocessor.canProcess(sysMetaDoc)) {

                            // if so, then extract the additional information from the

                            // document.

                            try {

                                // docObject = the resource map document or science

                                // metadata document.

                                // note that resource map processing touches all objects

                                // referenced by the resource map.

                                Document docObject = generateXmlDocument(dataStream);

                                if (docObject == null) {

                                    log.error("Could not load OBJECT for ID " + id );

                                } else {

                                    docs = subprocessor.processDocument(id, docs, docObject);

                                }

                            } catch (Exception e) {

                                log.error(e.getStackTrace().toString());

                            }

                        }

                    }

       }

       // TODO: in the XPathDocumentParser class in d1_cn_index_process module,

       // merge is only for resource map. We need more work here.

       for (SolrDoc mergeDoc : docs.values()) {

           if (!mergeDoc.isMerged()) {

       }

       //SolrElementAdd addCommand = getAddCommand(new ArrayList<SolrDoc>(docs.values()));

       return docs;

    }

     * Merge updates with existing solr documents

     *

     * This method appears to re-set the data package field data into the

     * document about to be updated in the solr index. Since packaging

     * information is derived from the package document (resource map), this

     * information is not present when processing a document contained in a data

     * package. This method replaces those values from the existing solr index

     * record for the document being processed. -- sroseboo, 1-18-12

     *

     * @param indexDocument

     * @return

     * @throws IOException

     * @throws EncoderException

     * @throws XPathExpressionException

     * @throws SAXException

     * @throws ParserConfigurationException

     * @throws SolrServerException

     * @throws NotFound

     * @throws NotImplemented

    // TODO:combine merge function with resourcemap merge function

    private SolrDoc mergeWithIndexedDocument(SolrDoc indexDocument) throws IOException,

        ids.add(indexDocument.getIdentifier());

        List<SolrDoc> indexedDocuments = ResourceMapSubprocessor.getSolrDocs(ids);

        SolrDoc indexedDocument = indexedDocuments == null || indexedDocuments.size() <= 0 ? null

                : indexedDocuments.get(0);

        if (indexedDocument == null || indexedDocument.getFieldList().size() <= 0) {

            return indexDocument;

        } else {

            for (SolrElementField field : indexedDocument.getFieldList()) {

                if ((field.getName().equals(SolrElementField.FIELD_ISDOCUMENTEDBY)

                        || field.getName().equals(SolrElementField.FIELD_DOCUMENTS) || field

                        .getName().equals(SolrElementField.FIELD_RESOURCEMAP))

                        && !indexDocument.hasFieldWithValue(field.getName(), field.getValue())) {

                    indexDocument.addField(field);

                }

            }

            indexDocument.setMerged(true);

            return indexDocument;

        }

    }

     * Generate a Document from the InputStream

     */

    private Document generateXmlDocument(InputStream smdStream) throws SAXException {

        Document doc = null;

        try {

            doc = builder.parse(smdStream);

        } catch (IOException e) {

            log.error(e.getMessage(), e);

        }

        return doc;

    }

    /*

     * Index the fields of the system metadata

     */

    private List<SolrElementField> processSysmetaFields(Document doc, String identifier) {

        List<SolrElementField> fieldList = new ArrayList<SolrElementField>();

        // solrFields is the list of fields defined in the application context

        for (SolrField field : sysmetaSolrFields) {

            try {

                // the field.getFields method can return a single value or

                // multiple values for multi-valued fields

                // or can return multiple SOLR document fields.

                fieldList.addAll(field.getFields(doc, identifier));

            } catch (Exception e) {

                e.printStackTrace();

            }

        }

        return fieldList;

    }

    /**

     * @param pid

     * @param systemMetadata

     * @param data

     * @throws SolrServerException

     */

    private void checkParams(String pid, SystemMetadata systemMetadata, InputStream data) throws SolrServerException {

        if(pid == null || pid.trim().equals("")) {

            throw new SolrServerException("The identifier of the indexed document should not be null or blank.");

        }

        if(systemMetadata == null) {

            throw new SolrServerException("The system metadata of the indexed document should not be null.");

        }

        if(data == null) {

            throw new SolrServerException("The indexed document itself should not be null.");

        }

    }

    /**

     * @param systemMetadata  the system metadata associated with the data object

     * @param data  the data object itself

     * @throws SolrServerException

     * @throws NotFound

     * @throws NotImplemented

        //transform the Map to the SolrInputDocument which can be used by the solr server

        if(docs != null) {

            Set<String> ids = docs.keySet();

            for(String id : ids) {

                if(id != null) {

                    SolrDoc doc = docs.get(id);

                }

            }

        }

    }

    /*

     * Insert a SolrDoc to the solr server.

     */

    private synchronized void insertToIndex(SolrDoc doc) throws SolrServerException, IOException {

        if(doc != null ) {

            SolrInputDocument solrDoc = new SolrInputDocument();

            List<SolrElementField> list = doc.getFieldList();

            if(list != null) {

                //solrDoc.addField(METACATPIDFIELD, pid);

                Iterator<SolrElementField> iterator = list.iterator();

                while (iterator.hasNext()) {

                    SolrElementField field = iterator.next();

                    if(field != null) {

                        String value = field.getValue();

                        String name = field.getName();

                        //System.out.println("add name/value pair - "+name+"/"+value);

                        solrDoc.addField(name, value);

                }

            }

                event.setDate(Calendar.getInstance().getTime());

                Identifier pid = new Identifier();

                pid.setValue(doc.getIdentifier());

                event.setPid(pid);

                try {

                    solrServer.commit();

                    event.setType(IndexEvent.SUCCESSINSERT);

                    event.setDescription("Successfully insert the solr index for the id "+pid.getValue());

                    try {

                        EventlogFactory.createIndexEventLog().write(event);

                    } catch (Exception e) {

                        log.error("SolrIndex.insertToIndex - IndexEventLog can't insert the solr doc to the solr server :"+e.getMessage());

                    }

                } catch (SolrServerException e) {

                    event.setType(IndexEvent.FAILUREINSERT);

                    event.setDescription("Failed to insert the solr index for the id "+pid.getValue()+" since "+e.getMessage());

                    try {

                        EventlogFactory.createIndexEventLog().write(event);

                    } catch (Exception ee) {

                        log.error("SolrIndex.insertToIndex - IndexEventLog can't insert the solr doc to the solr server :"+ee.getMessage());

                    }

                    throw e;

                } catch (IOException e) {

                    event.setType(IndexEvent.FAILUREINSERT);

                    event.setDescription("Failed to insert the solr index for the id "+pid.getValue()+" since "+e.getMessage());

                    try {

                        EventlogFactory.createIndexEventLog().write(event);

                    } catch (Exception ee) {

                        log.error("SolrIndex.insertToIndex - IndexEventLog can't insert the solr doc to the solr server :"+ee.getMessage());

                    }

                    throw e;

            }

    }

    /**

     * 1. Archive (or delete) - if the the system metadata shows the value of the archive is true,

     *    remove the index for the document and its previous versions if it has.

     * 2. Update an existing doc - if the the system metadata shows the value of the archive is false and it has an obsoletes,

     *    remove the index for the previous version(s) and generate new index for the doc.

     * 3. Add a new doc - if the system metadata shows the value of the archive is false and it hasn't an obsoletes, generate the

     *    index for the doc.

     * @param pid  the id of the document

     * @param data  the data object itself

     * @throws SolrServerException

     * @throws JiBXException

     * @throws NotFound

     * @throws NotImplemented

        boolean isArchive = systemMetadata.getArchived();

        if(isArchive) {

            //archive(delete)

            Identifier obsolete = systemMetadata.getObsoletes();

            if(obsolete != null) {

                removeObsoletesChain(obsolete.getValue(), obsoleteIds);

            }

            remove(pid);

            Identifier obsolete = systemMetadata.getObsoletes();

            if(obsolete != null) {

                removeObsoletesChain(obsolete.getValue(), obsoleteIds);

            }

            //generate index for either add or update.

            insert(pid, systemMetadata, data);

    }

            if(obsoleteIdChain == null || obsoleteIdChain.isEmpty()) {

                throw new SolrServerException("SolrIndex.removeObsoletesChain - The obsoletes chain can't be null or empty since the system metadata already has the obsoletes element.");

            }

            if(!obsoleteIdChain.contains(obsoleteId)) {

                throw new SolrServerException("SolrIndex.removeObsoletesChain - The obsoletes elment in the system metadata is not in the obsoleteId chain");

            }

            remove(obsoleteIdChain);

        } else {

            throw new SolrServerException("SolrIndex.removeObsoletesChain - The obsolete id should be null.");

        }

    }

     * Remove all the indexes associated with the pids in the list.

     * @param pidList

     * @throws IOException

     * @throws SolrServerException

     * @throws SAXException

     * @throws UnsupportedType

     * @throws NotFound

     * @throws NotImplemented

     * @throws XPathExpressionException

     * @throws ServiceFailure

            for(String id : pidList) {

                remove(id);

            }

        }

    }

    /**

     * Remove the indexed associated with specified pid.

     * @param pid  the pid which the indexes are associated with

     * @throws IOException

     * @throws SolrServerException

     * @throws SAXException

     * @throws UnsupportedType

     * @throws NotFound

     * @throws NotImplemented

     * @throws XPathExpressionException

     * @throws ServiceFailure

        if (isDataPackage(pid)) {

            removeDataPackage(pid);

        } else if (isPartOfDataPackage(pid)) {

            removeFromDataPackage(pid);

        } else {

            removeFromIndex(pid);

        }

    }

    /*

     * Remove a resource map pid

     */

    private void removeDataPackage(String pid) throws ServiceFailure, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException  {

        Document resourceMapDoc = generateXmlDocument(DistributedMapsFactory.getDataObject(pid));

        ResourceMap resourceMap = new ResourceMap(resourceMapDoc);

        List<String> documentIds = resourceMap.getAllDocumentIDs();

        List<SolrDoc> indexDocuments =ResourceMapSubprocessor.getSolrDocs(documentIds);

        removeFromIndex(pid);

        //List<SolrDoc> docsToUpdate = new ArrayList<SolrDoc>();

        // for each document in data package:

        for (SolrDoc indexDoc : indexDocuments) {

            if (indexDoc.getIdentifier().equals(pid)) {

                continue; // skipping the resource map, no need update

                          // it.

                          // will

                          // be removed.

            }

            // Remove resourceMap reference

            indexDoc.removeFieldsWithValue(SolrElementField.FIELD_RESOURCEMAP,

                    resourceMap.getIdentifier());

            // // Remove documents/documentedby values for this resource

            // map

            for (ResourceEntry entry : resourceMap.getMappedReferences()) {

                if (indexDoc.getIdentifier().equals(entry.getIdentifier())) {

                    for (String documentedBy : entry.getDocumentedBy()) {

                        // Using removeOneFieldWithValue in-case same

                        // documents

                        // are in more than one data package. just

                        // remove

                        // one

                        // instance of data package info.

                        indexDoc.removeOneFieldWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY,

                                documentedBy);

                    }

                    for (String documents : entry.getDocuments()) {

                        indexDoc.removeOneFieldWithValue(SolrElementField.FIELD_DOCUMENTS,

                                documents);

                    }

                    break;

                }

            }

            removeFromIndex(indexDoc.getIdentifier());

            insertToIndex(indexDoc);

            //docsToUpdate.add(indexDoc);

        }

        //SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate);

        //httpService.sendUpdate(solrIndexUri, addCommand);

    }

    private void removeFromDataPackage(String pid) throws XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException, SAXException  {

        SolrDoc indexedDoc = ResourceMapSubprocessor.getSolrDoc(pid);

        removeFromIndex(pid);

        List<SolrDoc> docsToUpdate = new ArrayList<SolrDoc>();

        List<String> documents = indexedDoc.getAllFieldValues(SolrElementField.FIELD_DOCUMENTS);

        for (String documentsValue : documents) {

            SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentsValue);

            solrDoc.removeFieldsWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY, pid);

            removeFromIndex(documentsValue);

            insertToIndex(solrDoc);

        }

        List<String> documentedBy = indexedDoc

                .getAllFieldValues(SolrElementField.FIELD_ISDOCUMENTEDBY);

        for (String documentedByValue : documentedBy) {

            SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentedByValue);

            solrDoc.removeFieldsWithValue(SolrElementField.FIELD_DOCUMENTS, documentedByValue);

            //docsToUpdate.add(solrDoc);

            removeFromIndex(documentedByValue);

            insertToIndex(solrDoc);

        }

        //SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate);

        //httpService.sendUpdate(solrIndexUri, addCommand);

    }

    /*

     * Remove a pid from the solr index

     */

    private void removeFromIndex(String pid) throws SolrServerException, IOException {

            event.setDate(Calendar.getInstance().getTime());

            Identifier identifier = new Identifier();

            identifier.setValue(pid);

            event.setPid(identifier);

            try {

                solrServer.commit();

                event.setType(IndexEvent.SUCCESSDELETE);

                event.setDescription("Successfully remove the solr index for the id "+identifier.getValue());

                try {

                    EventlogFactory.createIndexEventLog().write(event);

                } catch (Exception e) {

                    log.error("SolrIndex.removeFromIndex - IndexEventLog can't insert the solr doc to the solr server :"+e.getMessage());

                }

            } catch (SolrServerException e) {

                event.setType(IndexEvent.FAILUREDELETE);

                event.setDescription("Failurely remove the solr index for the id "+identifier.getValue()+" since "+e.getMessage());

                try {

                    EventlogFactory.createIndexEventLog().write(event);

                } catch (Exception ee) {

                    log.error("SolrIndex.removeFromIndex - IndexEventLog can't insert the solr doc to the solr server :"+ee.getMessage());

                }

                throw e;

            } catch (IOException e) {

                event.setType(IndexEvent.FAILUREDELETE);

                event.setDescription("Failurely remove the solr index for the id "+identifier.getValue()+" since "+e.getMessage());

                try {

                    EventlogFactory.createIndexEventLog().write(event);

                } catch (Exception ee) {

                    log.error("SolrIndex.removeFromIndex - IndexEventLog can't insert the solr doc to the solr server :"+ee.getMessage());

                }

                throw e;

     * Is the pid a resource map

     */

    private boolean isDataPackage(String pid) throws FileNotFoundException, ServiceFailure {

        boolean isDataPackage = false;

        SystemMetadata sysmeta = DistributedMapsFactory.getSystemMetadata(pid);

        if(sysmeta != null) {

            isDataPackage = IndexGenerator.isResourceMap(sysmeta.getFormatId());

        }

        return isDataPackage;

    }

    private boolean isPartOfDataPackage(String pid) throws XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException, SAXException {

        SolrDoc dataPackageIndexDoc = ResourceMapSubprocessor.getSolrDoc(pid);

        if (dataPackageIndexDoc != null) {

            String resourceMapId = dataPackageIndexDoc

                    .getFirstFieldValue(SolrElementField.FIELD_RESOURCEMAP);

            return StringUtils.isNotEmpty(resourceMapId);

        } else {

            return false;

        }

    }

     * Get the solrServer

     * @return

     */

    }

    /**

     */

    }

    /**

     * @return an empty list if there is no index.

     */

    public List<String> getSolrIds() throws SolrServerException {