Project

General

Profile

« Previous | Next » 

Revision 8464

Added by ben leinfelder over 10 years ago

Unify solr indexing with an IndexTask that is added to the queue -- allows us to send more than just the systemMetadata to the indexer. Initially this is for READ event counts for each document. https://projects.ecoinformatics.org/ecoinfo/issues/6346

View differences:

metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/IndexGeneratorTimerTaskIT.java
35 35 
    public void testGenerateAll() throws Exception {
36 36 
        SolrIndex solrIndex = generateSolrIndex();
37 37 
        SystemMetadataEventListener systeMetaListener = new SystemMetadataEventListener(solrIndex);
38 
        systeMetaListener.start();
38 
        systeMetaListener.run();
39 39 
        IndexGeneratorTimerTask generator = new IndexGeneratorTimerTask(solrIndex);
40 40 
        generator.indexAll();
41 41 
        String result = SolrIndexIT.doQuery(solrIndex.getSolrServer());
metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/SolrIndexIT.java
1 1 
package edu.ucsb.nceas.metacat.index;
2 2 

  		




  
  3
  
    
import static org.junit.Assert.assertFalse;


  		




  3
  4
  
    
import static org.junit.Assert.assertTrue;


  		




  4
  5
  
    

  		




  5
  6
  
    
import edu.ucsb.nceas.metacat.common.SolrServerFactory;


  		




  ......




  10
  11
  
    
import java.io.StringWriter;


  		




  11
  12
  
    
import java.io.Writer;


  		




  12
  13
  
    
import java.util.ArrayList;


  		




  
  14
  
    
import java.util.HashMap;


  		




  13
  15
  
    
import java.util.List;


  		




  
  16
  
    
import java.util.Map;


  		




  14
  17
  
    

  		




  15
  18
  
    
import org.apache.solr.client.solrj.SolrServer;


  		




  16
  19
  
    
import org.apache.solr.client.solrj.SolrServerException;


  		




  ......




  128
  131
  
    
    }


  		




  129
  132
  
    
    


  		




  130
  133
  
    
    


  		




  131
  
  
    
    /*


  		




  132
  
  
    
     * Do query


  		




  
  134
  
    
    /**


  		




  
  135
  
    
     * Test building index for dynamic fields.


  		




  133
  136
  
    
     */


  		




  
  137
  
    
    @Test


  		




  
  138
  
    
    public void testDynamicFields() throws Exception {


  		




  
  139
  
    
    	


  		




  
  140
  
    
       SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAFILEPATH);


  		




  
  141
  
    
       InputStream emlInputStream = new FileInputStream(new File(EMLFILEPATH)); 


  		




  
  142
  
    
       Identifier pid = new Identifier();


  		




  
  143
  
    
       pid.setValue(id);


  		




  
  144
  
    
       solrIndex.update(pid, systemMetadata, emlInputStream);


  		




  
  145
  
    
       String result = doQuery(solrIndex.getSolrServer());


  		




  
  146
  
    
       List<String> ids = solrIndex.getSolrIds();


  		




  
  147
  
    
       boolean foundId = false;


  		




  
  148
  
    
       for(String identifiers :ids) {


  		




  
  149
  
    
           if (id.equals(identifiers)) {


  		




  
  150
  
    
               foundId = true;


  		




  
  151
  
    
           }


  		




  
  152
  
    
       }


  		




  
  153
  
    
       assertTrue(foundId);


  		




  
  154
  
    
       assertTrue(result.contains("version1"));


  		




  
  155
  
    
       


  		




  
  156
  
    
       // augment with the dynamic field


  		




  
  157
  
    
       String fieldName = "test_count_i";


  		




  
  158
  
    
       Map<String, List<Object>> fields = new HashMap<String, List<Object>>();


  		




  
  159
  
    
       List<Object> values = new ArrayList<Object>();


  		




  
  160
  
    
       values.add(6);


  		




  
  161
  
    
       fields.put(fieldName, values);


  		




  
  162
  
    
       solrIndex.insertFields(pid, fields);


  		




  
  163
  
    
       result = doQuery(solrIndex.getSolrServer(), "&fq=" + fieldName + ":[0 TO 5]");


  		




  
  164
  
    
       assertFalse(result.contains(id));


  		




  
  165
  
    
       result = doQuery(solrIndex.getSolrServer(), "&fq=" + fieldName + ":[6 TO 6]");


  		




  
  166
  
    
       assertTrue(result.contains(id));


  		




  
  167
  
    
       


  		




  
  168
  
    
       // now update the value


  		




  
  169
  
    
       values.clear();


  		




  
  170
  
    
       values.add(7);


  		




  
  171
  
    
       fields.put(fieldName, values);


  		




  
  172
  
    
       solrIndex.insertFields(pid, fields);


  		




  
  173
  
    
       result = doQuery(solrIndex.getSolrServer(), "&fq=" + fieldName + ":[7 TO 7]");


  		




  
  174
  
    
       assertTrue(result.contains(id));


  		




  
  175
  
    
       


  		




  
  176
  
    
    }


  		




  
  177
  
    
    


  		




  
  178
  
    
    /**


  		




  
  179
  
    
     * Do query - with no additional params


  		




  
  180
  
    
     */


  		




  134
  181
  
    
    public static String doQuery(SolrServer server)


  		




  135
  182
  
    
                    throws SolrServerException {


  		




  
  183
  
    
    	return doQuery(server, null);


  		




  
  184
  
    
    }


  		




  
  185
  
    
    


  		




  
  186
  
    
    /**


  		




  
  187
  
    
     * Do query, allowing additional parameters


  		




  
  188
  
    
     */


  		




  
  189
  
    
    public static String doQuery(SolrServer server, String moreParams)


  		




  
  190
  
    
                    throws SolrServerException {


  		




  136
  191
  
    
                StringBuffer request = new StringBuffer();


  		




  137
  192
  
    
                request.append("q=" + "*:*");


  		




  
  193
  
    
                if (moreParams != null) {


  		




  
  194
  
    
                    request.append(moreParams);


  		




  
  195
  
    
                }


  		




  138
  196
  
    
                SolrParams solrParams = SolrRequestParsers.parseQueryString(request


  		




  139
  197
  
    
                        .toString());


  		




  140
  198
  
    
                QueryResponse reponse = server.query(solrParams);


  		




  ......




  146
  204
  
    
                return result;


  		




  147
  205
  
    
    }


  		




  148
  206
  
    
    


  		




  149
  
  
    
    /*


  		




  
  207
  
    
    /**


  		




  150
  208
  
    
     * Transform the query response to the xml format.


  		




  151
  209
  
    
     */


  		




  152
  210
  
    
    private static String toXML(SolrParams request, QueryResponse response) {


  		












  

    
      metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/IndexGeneratorTimerTask.java
    
  





  57
  57
  
    
import com.hazelcast.core.ISet;


  		




  58
  58
  
    

  		




  59
  59
  
    
import edu.ucsb.nceas.metacat.common.SolrServerFactory;


  		




  
  60
  
    
import edu.ucsb.nceas.metacat.common.index.IndexTask;


  		




  60
  61
  
    
import edu.ucsb.nceas.metacat.common.index.event.IndexEvent;


  		




  61
  62
  
    
import edu.ucsb.nceas.metacat.index.event.EventlogFactory;


  		




  62
  63
  
    
import edu.ucsb.nceas.metacat.index.event.IndexEventLogException;


  		




  ......




  90
  91
  
    
    //private SystemMetadataEventListener systemMetadataListener = null;


  		




  91
  92
  
    
    private IMap<Identifier, SystemMetadata> systemMetadataMap;


  		




  92
  93
  
    
    private IMap<Identifier, String> objectPathMap;


  		




  93
  
  
    
    private ISet<SystemMetadata> indexQueue;


  		




  
  94
  
    
    private IMap<Identifier, IndexTask> indexQueue;


  		




  94
  95
  
    
    private Log log = LogFactory.getLog(IndexGeneratorTimerTask.class);


  		




  95
  96
  
    
    //private MNode mNode = null;


  		




  96
  97
  
    
    private static List<String> resourceMapNamespaces = null;


  		












  

    
      metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/resourcemap/ResourceMapSubprocessor.java
    
  





  15
  15
  
    
 */


  		




  16
  16
  
    
package edu.ucsb.nceas.metacat.index.resourcemap;


  		




  17
  17
  
    

  		




  18
  
  
    
import java.io.ByteArrayInputStream;


  		




  19
  
  
    
import java.io.ByteArrayOutputStream;


  		




  20
  18
  
    
import java.io.IOException;


  		




  21
  
  
    
import java.io.InputStream;


  		




  22
  
  
    
import java.io.StringWriter;


  		




  23
  
  
    
import java.io.Writer;


  		




  24
  
  
    
import java.net.MalformedURLException;


  		




  25
  19
  
    
import java.util.ArrayList;


  		




  26
  
  
    
import java.util.Collections;


  		




  27
  20
  
    
import java.util.Date;


  		




  28
  21
  
    
import java.util.HashMap;


  		




  29
  22
  
    
import java.util.List;


  		




  30
  23
  
    
import java.util.Map;


  		




  31
  
  
    
import java.util.Set;


  		




  32
  24
  
    

  		




  33
  
  
    
import javax.xml.parsers.DocumentBuilder;


  		




  34
  
  
    
import javax.xml.parsers.DocumentBuilderFactory;


  		




  35
  25
  
    
import javax.xml.parsers.ParserConfigurationException;


  		




  36
  
  
    
import javax.xml.xpath.XPathConstants;


  		




  37
  26
  
    
import javax.xml.xpath.XPathExpressionException;


  		




  38
  
  
    
import javax.xml.xpath.XPathFactory;


  		




  39
  27
  
    

  		




  40
  28
  
    
import org.apache.commons.codec.EncoderException;


  		




  41
  29
  
    
import org.apache.commons.logging.Log;


  		




  ......




  44
  32
  
    
import org.apache.solr.client.solrj.SolrServerException;


  		




  45
  33
  
    
import org.apache.solr.client.solrj.response.QueryResponse;


  		




  46
  34
  
    
import org.apache.solr.common.SolrDocument;


  		




  47
  
  
    
import org.apache.solr.common.SolrDocumentList;


  		




  48
  35
  
    
import org.apache.solr.common.params.SolrParams;


  		




  49
  
  
    
import org.apache.solr.core.CoreContainer;


  		




  50
  
  
    
import org.apache.solr.core.SolrCore;


  		




  51
  
  
    
import org.apache.solr.request.LocalSolrQueryRequest;


  		




  52
  
  
    
import org.apache.solr.response.QueryResponseWriter;


  		




  53
  
  
    
import org.apache.solr.response.SolrQueryResponse;


  		




  54
  
  
    
import org.apache.solr.schema.DateField;


  		




  55
  36
  
    
import org.apache.solr.schema.IndexSchema;


  		




  56
  
  
    
import org.apache.solr.schema.SchemaField;


  		




  57
  37
  
    
import org.apache.solr.servlet.SolrRequestParsers;


  		




  
  38
  
    
import org.dataone.cn.indexer.convert.SolrDateConverter;


  		




  58
  39
  
    
import org.dataone.cn.indexer.parser.AbstractDocumentSubprocessor;


  		




  59
  40
  
    
import org.dataone.cn.indexer.parser.IDocumentSubprocessor;


  		




  60
  41
  
    
import org.dataone.cn.indexer.resourcemap.ResourceMap;


  		




  ......




  64
  45
  
    
import org.dataone.service.exceptions.NotFound;


  		




  65
  46
  
    
import org.dataone.service.exceptions.NotImplemented;


  		




  66
  47
  
    
import org.dataone.service.exceptions.UnsupportedType;


  		




  67
  
  
    
import org.dataone.service.types.v1.Subject;


  		




  
  48
  
    
import org.dataone.service.util.DateTimeMarshaller;


  		




  68
  49
  
    
import org.dspace.foresite.OREParserException;


  		




  69
  50
  
    
import org.w3c.dom.Document;


  		




  70
  
  
    
import org.w3c.dom.Element;


  		




  71
  
  
    
import org.w3c.dom.NodeList;


  		




  72
  51
  
    
import org.xml.sax.SAXException;


  		




  73
  52
  
    

  		




  74
  
  
    
import edu.ucsb.nceas.metacat.common.query.SolrQueryResponseTransformer;


  		




  75
  
  
    
import edu.ucsb.nceas.metacat.common.query.SolrQueryResponseWriterFactory;


  		




  
  53
  
    
import edu.ucsb.nceas.metacat.common.SolrServerFactory;


  		




  76
  54
  
    
import edu.ucsb.nceas.metacat.common.query.SolrQueryServiceController;


  		




  77
  
  
    
import edu.ucsb.nceas.metacat.common.SolrServerFactory;


  		




  78
  55
  
    
import edu.ucsb.nceas.metacat.index.SolrIndex;


  		




  79
  56
  
    

  		




  80
  57
  
    

  		




  ......




  88
  65
  
    
    private static final String QUERY ="q=id:";


  		




  89
  66
  
    
    private static Log log = LogFactory.getLog(SolrIndex.class);


  		




  90
  67
  
    
    private static SolrServer solrServer =  null;


  		




  91
  
  
    
    private static SolrCore solrCore = null;


  		




  92
  
  
    
    private static CoreContainer solrCoreContainer = null;


  		




  93
  68
  
    
    static {


  		




  94
  69
  
    
        try {


  		




  95
  70
  
    
            solrServer = SolrServerFactory.createSolrServer();


  		




  96
  
  
    
            CoreContainer solrCoreContainer = SolrServerFactory.getCoreContainer();


  		




  97
  
  
    
            String coreName = SolrServerFactory.getCollectionName();


  		




  98
  
  
    
            solrCore = solrCoreContainer.getCore(coreName);


  		




  99
  71
  
    
        } catch (Exception e) {


  		




  100
  72
  
    
            log.error("ResourceMapSubprocessor - can't generate the SolrServer since - "+e.getMessage());


  		




  101
  73
  
    
        }


  		




  ......




  139
  111
  
    
        List<SolrDoc> list = new ArrayList<SolrDoc>();


  		




  140
  112
  
    
        if(ids != null) {


  		




  141
  113
  
    
            for(String id : ids) {


  		




  142
  
  
    
                SolrDoc doc = getSolrDoc(id);


  		




  
  114
  
    
            	SolrDoc doc = getSolrDoc(id);


  		




  143
  115
  
    
                if(doc != null) {


  		




  144
  116
  
    
                    list.add(doc);


  		




  145
  117
  
    
                } else if ( !id.equals(resourceMapId)) {


  		




  ......




  157
  129
  
    
        List<SolrDoc> list = new ArrayList<SolrDoc>();


  		




  158
  130
  
    
        if(ids != null) {


  		




  159
  131
  
    
            for(String id : ids) {


  		




  160
  
  
    
                SolrDoc doc = getSolrDoc(id);


  		




  
  132
  
    
            	SolrDoc doc = getSolrDoc(id);


  		




  161
  133
  
    
                if(doc != null) {


  		




  162
  134
  
    
                    list.add(doc);


  		




  163
  135
  
    
                }


  		




  ......




  166
  138
  
    
        return list;


  		




  167
  139
  
    
    }


  		




  168
  140
  
    
    


  		




  169
  
  
    
    /*


  		




  170
  
  
    
     * Get the SolrDoc for the specified id 


  		




  171
  
  
    
     */


  		




  172
  
  
    
    public static SolrDoc getSolrDoc(String id) throws SolrServerException, IOException, ParserConfigurationException, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType {


  		




  173
  
  
    
        SolrDoc solrDoc = null;


  		




  174
  
  
    
        if(solrServer != null) {


  		




  175
  
  
    
           String query = QUERY+"\""+id+"\"";


  		




  176
  
  
    
           SolrParams solrParams = SolrRequestParsers.parseQueryString(query);


  		




  177
  
  
    
           Set<Subject>subjects = null;//when subjects are null, there will not be any access rules.


  		




  178
  
  
    
           InputStream response = SolrQueryServiceController.getInstance().query(solrParams, subjects);


  		




  179
  
  
    
           solrDoc = transformQueryResponseToSolrDoc(solrParams, response);


  		




  180
  
  
    
           


  		




  181
  
  
    
           /*if(solrDoc != null) {


  		




  182
  
  
    
               ByteArrayOutputStream out = new ByteArrayOutputStream();


  		




  183
  
  
    
               solrDoc.serialize(out, "UTF-8");


  		




  184
  
  
    
               String result = new String(out.toByteArray(), "UTF-8");


  		




  185
  
  
    
               System.out.println("need to be updated document ===========================");


  		




  186
  
  
    
               System.out.println(result);


  		




  187
  
  
    
           }*/


  		




  188
  
  
    
           


  		




  189
  
  
    
        }


  		




  190
  
  
    
        return solrDoc;


  		




  191
  
  
    
    }


  		




  192
  
  
    
    


  		




  193
  
  
    
    /*


  		




  194
  
  
    
     * Transform a Solr QueryReponse to a SolrDoc. The QueryReponse contains a list of


  		




  195
  
  
    
     * SolrDocuments. This method will transform the first SolrDocuments (in the Solr lib) to


  		




  196
  
  
    
     * the SolrDoc (in the d1_cn_index_processor lib).


  		




  197
  
  
    
     * @param reponse


  		




  198
  
  
    
     * @return


  		




  199
  
  
    
     */


  		




  200
  
  
    
    private static SolrDoc transformQueryResponseToSolrDoc(SolrParams solrParams, InputStream response) throws SolrServerException, IOException, ParserConfigurationException, SAXException, XPathExpressionException, UnsupportedType, NotFound {


  		




  201
  
  
    
        SolrDoc solrDoc = null;


  		




  202
  
  
    
        if(response != null) {


  		




  203
  
  
    
            DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();


  		




  204
  
  
    
            DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();


  		




  205
  
  
    
            Document doc = dBuilder.parse(response);


  		




  206
  
  
    
            solrDoc = parseResults(doc);


  		




  207
  
  
    
        }


  		




  208
  
  
    
        return solrDoc;


  		




  209
  
  
    
    }


  		




  210
  
  
    
    


  		




  211
  
  
    
   


  		




  212
  
  
    
    


  		




  213
  
  
    
    /*


  		




  214
  
  
    
     * Parse the query result document. This method only choose the first one from a list.


  		




  215
  
  
    
     */


  		




  216
  
  
    
    private static SolrDoc parseResults(Document document) throws XPathExpressionException, MalformedURLException, UnsupportedType, NotFound, ParserConfigurationException, IOException, SAXException {


  		




  217
  
  
    
        SolrDoc solrDoc = null;


  		




  218
  
  
    
        NodeList nodeList = (NodeList) XPathFactory.newInstance().newXPath()


  		




  219
  
  
    
                .evaluate("/response/result/doc", document, XPathConstants.NODESET);


  		




  220
  
  
    
        if(nodeList != null && nodeList.getLength() >0) {


  		




  221
  
  
    
            Element docElement = (Element) nodeList.item(0);


  		




  222
  
  
    
            solrDoc = parseDoc(docElement);


  		




  223
  
  
    
        }


  		




  224
  
  
    
        return solrDoc;


  		




  225
  
  
    
    }


  		




  
  141
  
    
	/*


  		




  
  142
  
    
	 * Get the SolrDoc for the specified id


  		




  
  143
  
    
	 */


  		




  
  144
  
    
	public static SolrDoc getSolrDoc(String id) throws SolrServerException,


  		




  
  145
  
    
			IOException, ParserConfigurationException, SAXException,


  		




  
  146
  
    
			XPathExpressionException, NotImplemented, NotFound, UnsupportedType {


  		




  
  147
  
    
		SolrDoc doc = new SolrDoc();


  		




  226
  148
  
    

  		




  227
  
  
    
    


  		




  228
  
  
    
    /*


  		




  229
  
  
    
     * Parse an element


  		




  230
  
  
    
     */


  		




  231
  
  
    
    private static SolrDoc parseDoc(Element docElement) throws MalformedURLException, UnsupportedType, NotFound, ParserConfigurationException, IOException, SAXException {


  		




  232
  
  
    
        List<String> validSolrFieldNames = SolrQueryServiceController.getInstance().getValidSchemaFields();


  		




  233
  
  
    
        SolrDoc doc = new SolrDoc();


  		




  234
  
  
    
        doc.LoadFromElement(docElement, validSolrFieldNames);


  		




  235
  
  
    
        return doc;


  		




  236
  
  
    
    }


  		




  237
  
  
    
    


  		




  238
  
  
    
    


  		




  239
  
  
    
    /**


  		




  240
  
  
    
     * Get the valid schema fields from the solr server.


  		




  241
  
  
    
     * @return


  		




  242
  
  
    
     */


  		




  243
  
  
    
    /*private static List<String> getValidSchemaField() {


  		




  244
  
  
    
        List<String> validSolrFieldNames = new ArrayList<String>();


  		




  245
  
  
    
        IndexSchema schema = solrCore.getSchema();


  		




  246
  
  
    
        Map<String, SchemaField> fieldMap = schema.getFields();


  		




  247
  
  
    
        Set<String> fieldNames = fieldMap.keySet();


  		




  248
  
  
    
        for(String fieldName : fieldNames) {


  		




  249
  
  
    
            SchemaField field = fieldMap.get(fieldName);


  		




  250
  
  
    
            //remove the field which is the target field of a CopyField.


  		




  251
  
  
    
            if(field != null && !schema.isCopyFieldTarget(field)) {


  		




  252
  
  
    
                 validSolrFieldNames.add(fieldName);


  		




  253
  
  
    
            }


  		




  254
  
  
    
        }


  		




  255
  
  
    
        //System.out.println("the valid file name is\n"+validSolrFieldNames);


  		




  256
  
  
    
        return validSolrFieldNames;


  		




  257
  
  
    
    }*/


  		




  
  149
  
    
		if (solrServer != null) {


  		




  
  150
  
    
			String query = QUERY + "\"" + id + "\"";


  		




  
  151
  
    
			SolrParams solrParams = SolrRequestParsers.parseQueryString(query);


  		




  
  152
  
    
			QueryResponse qr = solrServer.query(solrParams);


  		




  
  153
  
    
			SolrDocument orig = qr.getResults().get(0);


  		




  
  154
  
    
			IndexSchema indexSchema = SolrQueryServiceController.getInstance().getSchema();


  		




  
  155
  
    
			for (String fieldName : orig.getFieldNames()) {


  		




  
  156
  
    
				// don't transfer the copyTo fields, otherwise there are errors


  		




  
  157
  
    
				if (indexSchema.isCopyFieldTarget(indexSchema.getField(fieldName))) {


  		




  
  158
  
    
					continue;


  		




  
  159
  
    
				}


  		




  
  160
  
    
				for (Object value : orig.getFieldValues(fieldName)) {


  		




  
  161
  
    
					String stringValue = value.toString();


  		




  
  162
  
    
					// special handling for dates in ISO 8601


  		




  
  163
  
    
					if (value instanceof Date) {


  		




  
  164
  
    
						stringValue = DateTimeMarshaller.serializeDateToUTC((Date) value);


  		




  
  165
  
    
						SolrDateConverter converter = new SolrDateConverter();


  		




  
  166
  
    
						stringValue = converter.convert(stringValue);


  		




  
  167
  
    
					}


  		




  
  168
  
    
					SolrElementField field = new SolrElementField(fieldName, stringValue);


  		




  
  169
  
    
					log.debug("Adding field: " + fieldName);


  		




  
  170
  
    
					doc.addField(field);


  		




  
  171
  
    
				}


  		




  
  172
  
    
			}


  		




  258
  173
  
    

  		




  
  174
  
    
		}


  		




  
  175
  
    
		return doc;


  		




  
  176
  
    
	}


  		




  
  177
  
    

  		




  
  178
  
    

  		




  259
  179
  
    
}


  		












  

    
      metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/ApplicationController.java
    
  





  32
  32
  
    
import org.dataone.service.exceptions.ServiceFailure;


  		




  33
  33
  
    
import org.springframework.context.ApplicationContext;


  		




  34
  34
  
    
import org.springframework.context.support.ClassPathXmlApplicationContext;


  		




  35
  
  
    
import org.springframework.context.support.FileSystemXmlApplicationContext;


  		




  36
  35
  
    

  		




  37
  
  
    
import com.ibm.icu.util.Calendar;


  		




  38
  36
  
    

  		




  39
  37
  
    
import edu.ucsb.nceas.metacat.common.SolrServerFactory;


  		




  40
  38
  
    
import edu.ucsb.nceas.metacat.common.query.EnabledQueryEngines;


  		




  ......




  51
  49
  
    
    private static short FIRST = 0;


  		




  52
  50
  
    

  		




  53
  51
  
    
    private List<SolrIndex> solrIndexes = null;


  		




  54
  
  
    
    private List<SystemMetadataEventListener> sysmetaListeners = new ArrayList<SystemMetadataEventListener>();


  		




  
  52
  
    
    private List<Runnable> sysmetaListeners = new ArrayList<Runnable>();


  		




  55
  53
  
    
    private static ApplicationContext context = null;


  		




  56
  54
  
    
    private String springConfigFileURL = "/index-processor-context.xml";


  		




  57
  55
  
    
    private String metacatPropertiesFile = null;


  		




  ......




  165
  163
  
    
        	SystemMetadataEventListener smel = new SystemMetadataEventListener();


  		




  166
  164
  
    
        	smel.setSolrIndex(solrIndex);


  		




  167
  165
  
    
        	sysmetaListeners.add(smel);


  		




  168
  
  
    
        	//smel.start();


  		




  169
  166
  
    
        }


  		




  170
  167
  
    
        


  		




  171
  168
  
    
    }


  		




  ......




  225
  222
  
    
    private void startSysmetaListener() throws FileNotFoundException, ServiceFailure {


  		




  226
  223
  
    
        if(sysmetaListeners != null) {


  		




  227
  224
  
    
            //only expects one listener.


  		




  228
  
  
    
            for(SystemMetadataEventListener listener : sysmetaListeners) {


  		




  
  225
  
    
            for(Runnable listener : sysmetaListeners) {


  		




  229
  226
  
    
                if(listener != null) {


  		




  230
  
  
    
                    listener.start();


  		




  
  227
  
    
                    listener.run();


  		




  231
  228
  
    
                }


  		




  232
  229
  
    
            }


  		




  233
  230
  
    
        }


  		












  

    
      metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java
    
  





  23
  23
  
    
import java.io.FileNotFoundException;


  		




  24
  24
  
    
import java.io.IOException;


  		




  25
  25
  
    
import java.io.InputStream;


  		




  
  26
  
    
import java.net.MalformedURLException;


  		




  26
  27
  
    
import java.util.ArrayList;


  		




  27
  28
  
    
import java.util.Calendar;


  		




  
  29
  
    
import java.util.Date;


  		




  28
  30
  
    
import java.util.HashMap;


  		




  29
  31
  
    
import java.util.Iterator;


  		




  30
  32
  
    
import java.util.List;


  		




  ......




  51
  53
  
    
import org.apache.solr.common.SolrDocument;


  		




  52
  54
  
    
import org.apache.solr.common.SolrDocumentList;


  		




  53
  55
  
    
import org.apache.solr.common.SolrInputDocument;


  		




  
  56
  
    
import org.apache.solr.schema.IndexSchema;


  		




  54
  57
  
    
import org.dataone.cn.indexer.XMLNamespaceConfig;


  		




  
  58
  
    
import org.dataone.cn.indexer.convert.SolrDateConverter;


  		




  55
  59
  
    
import org.dataone.cn.indexer.parser.IDocumentSubprocessor;


  		




  56
  60
  
    
import org.dataone.cn.indexer.parser.SolrField;


  		




  57
  61
  
    
import org.dataone.cn.indexer.resourcemap.ResourceEntry;


  		




  ......




  66
  70
  
    
import org.dataone.service.types.v1.Event;


  		




  67
  71
  
    
import org.dataone.service.types.v1.Identifier;


  		




  68
  72
  
    
import org.dataone.service.types.v1.SystemMetadata;


  		




  
  73
  
    
import org.dataone.service.util.DateTimeMarshaller;


  		




  69
  74
  
    
import org.dataone.service.util.TypeMarshaller;


  		




  70
  75
  
    
import org.dspace.foresite.OREParserException;


  		




  71
  76
  
    
import org.jibx.runtime.JiBXException;


  		




  ......




  73
  78
  
    
import org.xml.sax.SAXException;


  		




  74
  79
  
    

  		




  75
  80
  
    
import edu.ucsb.nceas.metacat.common.index.event.IndexEvent;


  		




  
  81
  
    
import edu.ucsb.nceas.metacat.common.query.SolrQueryServiceController;


  		




  76
  82
  
    
import edu.ucsb.nceas.metacat.index.event.EventlogFactory;


  		




  77
  83
  
    
import edu.ucsb.nceas.metacat.index.resourcemap.ResourceMapSubprocessor;


  		




  78
  84
  
    

  		




  ......




  261
  267
  
    
        List<SolrDoc> indexedDocuments = ResourceMapSubprocessor.getSolrDocs(ids);


  		




  262
  268
  
    
        SolrDoc indexedDocument = indexedDocuments == null || indexedDocuments.size() <= 0 ? null


  		




  263
  269
  
    
                : indexedDocuments.get(0);


  		




  
  270
  
    
        


  		




  
  271
  
    
        IndexSchema indexSchema = SolrQueryServiceController.getInstance().getSchema();


  		




  
  272
  
    

  		




  264
  273
  
    
        if (indexedDocument == null || indexedDocument.getFieldList().size() <= 0) {


  		




  265
  274
  
    
            return indexDocument;


  		




  266
  275
  
    
        } else {


  		




  ......




  270
  279
  
    
                        .getName().equals(SolrElementField.FIELD_RESOURCEMAP))


  		




  271
  280
  
    
                        && !indexDocument.hasFieldWithValue(field.getName(), field.getValue())) {


  		




  272
  281
  
    
                    indexDocument.addField(field);


  		




  
  282
  
    
                } else if (!indexSchema.isCopyFieldTarget(indexSchema.getField(field.getName())) && !indexDocument.hasField(field.getName())) {


  		




  
  283
  
    
                    indexDocument.addField(field);


  		




  273
  284
  
    
                }


  		




  274
  285
  
    
            }


  		




  275
  286
  
    

  		




  ......




  365
  376
  
    
        }


  		




  366
  377
  
    
    }


  		




  367
  378
  
    
    


  		




  
  379
  
    
    /**


  		




  
  380
  
    
     * Adds the given fields to the solr index for the given pid, preserving the index values


  		




  
  381
  
    
     * that previously existed


  		




  
  382
  
    
     * @param pid


  		




  
  383
  
    
     * @param fields


  		




  
  384
  
    
     */


  		




  
  385
  
    
    public void insertFields(Identifier pid, Map<String, List<Object>> fields) {


  		




  
  386
  
    
    	


  		




  
  387
  
    
    	try {


  		




  
  388
  
    
			// copy the original values already indexed for this document	


  		




  
  389
  
    
	    	SolrQuery query = new SolrQuery("id:\"" + pid.getValue() + "\"");


  		




  
  390
  
    
	    	QueryResponse res = solrServer.query(query);


  		




  
  391
  
    
	    	SolrDocument orig = res.getResults().get(0);


  		




  
  392
  
    
	    	SolrDoc doc = new SolrDoc();


  		




  
  393
  
    
	        IndexSchema indexSchema = SolrQueryServiceController.getInstance().getSchema();


  		




  
  394
  
    
	    	for (String fieldName: orig.getFieldNames()) {


  		




  
  395
  
    
	        	//  don't transfer the copyTo fields, otherwise there are errors


  		




  
  396
  
    
	        	if (indexSchema.isCopyFieldTarget(indexSchema.getField(fieldName))) {


  		




  
  397
  
    
	        		continue;


  		




  
  398
  
    
	        	}


  		




  
  399
  
    
	        	for (Object value: orig.getFieldValues(fieldName)) {


  		




  
  400
  
    
	        		String stringValue = value.toString();


  		




  
  401
  
    
	        		// special handling for dates in ISO 8601


  		




  
  402
  
    
	        		if (value instanceof Date) {


  		




  
  403
  
    
	        			stringValue = DateTimeMarshaller.serializeDateToUTC((Date)value);


  		




  
  404
  
    
	        			SolrDateConverter converter = new SolrDateConverter();


  		




  
  405
  
    
	        			stringValue = converter.convert(stringValue);


  		




  
  406
  
    
	        		}


  		




  
  407
  
    
					SolrElementField field = new SolrElementField(fieldName, stringValue);


  		




  
  408
  
    
					log.debug("Adding field: " + fieldName);


  		




  
  409
  
    
					doc.addField(field);


  		




  
  410
  
    
	        	}


  		




  
  411
  
    
	        }


  		




  
  412
  
    
	    	


  		




  
  413
  
    
	        // add the additional fields we are trying to include in the index


  		




  
  414
  
    
	        for (String fieldName: fields.keySet()) {


  		




  
  415
  
    
	    		List<Object> values = fields.get(fieldName);


  		




  
  416
  
    
	    		for (Object value: values) {


  		




  
  417
  
    
	    	    	doc.updateOrAddField(fieldName, value.toString());


  		




  
  418
  
    
	    		}


  		




  
  419
  
    
	    	}


  		




  
  420
  
    
	        


  		




  
  421
  
    
	        // insert the whole thing


  		




  
  422
  
    
	        insertToIndex(doc);


  		




  
  423
  
    
    	} catch (Exception e) {


  		




  
  424
  
    
    		String error = "SolrIndex.insetFields - could not update the solr index: " + e.getMessage();


  		




  
  425
  
    
            writeEventLog(null, pid, error);


  		




  
  426
  
    
            log.error(error, e);


  		




  
  427
  
    
    	}


  		




  
  428
  
    

  		




  
  429
  
    
    }


  		




  
  430
  
    
    


  		




  368
  431
  
    
    /*


  		




  369
  432
  
    
     * Insert a SolrDoc to the solr server.


  		




  370
  433
  
    
     */


  		












  

    
      metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/DistributedMapsFactory.java
    
  





  20
  20
  
    
import java.io.FileInputStream;


  		




  21
  21
  
    
import java.io.FileNotFoundException;


  		




  22
  22
  
    
import java.io.InputStream;


  		




  
  23
  
    
import java.util.List;


  		




  
  24
  
    
import java.util.Map;


  		




  23
  25
  
    

  		




  24
  26
  
    
import org.apache.commons.logging.Log;


  		




  25
  27
  
    
import org.apache.commons.logging.LogFactory;


  		




  ......




  35
  37
  
    
import com.hazelcast.core.IMap;


  		




  36
  38
  
    
import com.hazelcast.core.ISet;


  		




  37
  39
  
    

  		




  
  40
  
    
import edu.ucsb.nceas.metacat.common.index.IndexTask;


  		




  38
  41
  
    
import edu.ucsb.nceas.metacat.common.index.event.IndexEvent;


  		




  39
  42
  
    

  		




  40
  43
  
    

  		




  ......




  56
  59
  
    
    private static int maxAttempts = IndexGeneratorTimerTask.MAXWAITNUMBER;


  		




  57
  60
  
    
    private static IMap<Identifier, SystemMetadata> systemMetadataMap = null;


  		




  58
  61
  
    
    private static IMap<Identifier, String> objectPathMap = null;


  		




  59
  
  
    
    private static ISet<SystemMetadata> indexQueue = null;


  		




  
  62
  
    
    private static IMap<Identifier, IndexTask> indexQueue = null;


  		




  60
  63
  
    
    /* The name of the identifiers set */


  		




  61
  64
  
    
    private static String identifiersSetName = IDENTIFIERSETNAME;


  		




  62
  65
  
    
    /* The Hazelcast distributed identifiers set */


  		




  ......




  243
  246
  
    
     * @throws FileNotFoundException


  		




  244
  247
  
    
     * @throws ServiceFailure


  		




  245
  248
  
    
     */


  		




  246
  
  
    
    public static ISet<SystemMetadata> getIndexQueue() throws FileNotFoundException, ServiceFailure {


  		




  
  249
  
    
    public static IMap<Identifier, IndexTask> getIndexQueue() throws FileNotFoundException, ServiceFailure {


  		




  247
  250
  
    
        if(hzClient== null) {


  		




  248
  251
  
    
            startHazelCastClient();


  		




  249
  252
  
    
        }


  		




  250
  
  
    
        indexQueue = hzClient.getSet(hzIndexQueue);


  		




  
  253
  
    
        indexQueue = hzClient.getMap(hzIndexQueue);


  		




  251
  254
  
    
        return indexQueue;


  		




  252
  255
  
    
    }


  		




  253
  256
  
    
    


  		












  

    
      metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SystemMetadataEventListener.java
    
  





  18
  18
  
    
 */


  		




  19
  19
  
    
package edu.ucsb.nceas.metacat.index;


  		




  20
  20
  
    

  		




  21
  
  
    
import java.io.FileInputStream;


  		




  22
  21
  
    
import java.io.FileNotFoundException;


  		




  23
  
  
    
import java.io.InputStream;


  		




  24
  
  
    
import java.util.ArrayList;


  		




  25
  
  
    
import java.util.Calendar;


  		




  26
  22
  
    
import java.util.List;


  		




  
  23
  
    
import java.util.Map;


  		




  27
  24
  
    

  		




  28
  25
  
    
import org.apache.commons.logging.Log;


  		




  29
  26
  
    
import org.apache.commons.logging.LogFactory;


  		




  30
  27
  
    
import org.dataone.service.exceptions.ServiceFailure;


  		




  31
  
  
    
import org.dataone.service.types.v1.Event;


  		




  32
  28
  
    
import org.dataone.service.types.v1.Identifier;


  		




  33
  29
  
    
import org.dataone.service.types.v1.SystemMetadata;


  		




  34
  30
  
    

  		




  
  31
  
    
import com.hazelcast.core.EntryEvent;


  		




  
  32
  
    
import com.hazelcast.core.EntryListener;


  		




  35
  33
  
    
import com.hazelcast.core.IMap;


  		




  36
  
  
    
import com.hazelcast.core.ISet;


  		




  37
  
  
    
import com.hazelcast.core.ItemEvent;


  		




  38
  
  
    
import com.hazelcast.core.ItemListener;


  		




  39
  34
  
    

  		




  40
  
  
    
import edu.ucsb.nceas.metacat.common.index.event.IndexEvent;


  		




  41
  
  
    
import edu.ucsb.nceas.metacat.index.event.EventlogFactory;


  		




  
  35
  
    
import edu.ucsb.nceas.metacat.common.index.IndexTask;


  		




  42
  36
  
    

  		




  43
  
  
    
public class SystemMetadataEventListener implements ItemListener<SystemMetadata> {


  		




  
  37
  
    
public class SystemMetadataEventListener implements EntryListener<Identifier, IndexTask>, Runnable {


  		




  44
  38
  
    
	


  		




  45
  39
  
    
	private static Log log = LogFactory.getLog(SystemMetadataEventListener.class);


  		




  46
  40
  
    
	


  		




  47
  41
  
    
	private SolrIndex solrIndex = null;


  		




  48
  42
  
    
	


  		




  49
  
  
    
	private ISet<SystemMetadata> source = null;


  		




  
  43
  
    
	private IMap<Identifier, IndexTask> source = null;


  		




  50
  44
  
    
	        


  		




  51
  45
  
    
    /**


  		




  52
  46
  
    
     * Default constructor - caller needs to initialize manually


  		




  ......




  63
  57
  
    
    public SystemMetadataEventListener(SolrIndex solrIndex) {


  		




  64
  58
  
    
    	this.solrIndex = solrIndex;


  		




  65
  59
  
    
    	try {


  		




  66
  
  
    
			start();


  		




  
  60
  
    
			run();


  		




  67
  61
  
    
		} catch (Exception e) {


  		




  68
  62
  
    
			log.error(e.getMessage(), e);


  		




  69
  63
  
    
		}


  		




  ......




  90
  84
  
    
	 * @throws ServiceFailure 


  		




  91
  85
  
    
	 * @throws FileNotFoundException 


  		




  92
  86
  
    
     */


  		




  93
  
  
    
    public void start() throws FileNotFoundException, ServiceFailure {


  		




  
  87
  
    
    public void run() {


  		




  94
  88
  
    
    	


  		




  95
  89
  
    
        // get shared structures and add listener


  		




  96
  
  
    
        IMap<Identifier, String> objectPathMap = DistributedMapsFactory.getObjectPathMap();


  		




  97
  
  
    
        ISet<SystemMetadata> indexQueue = DistributedMapsFactory.getIndexQueue();


  		




  98
  
  
    
        indexQueue.addItemListener(this, true);


  		




  99
  
  
    
        this.source = indexQueue;


  		




  100
  
  
    
        log.info("System Metadata size: " + indexQueue.size());


  		




  101
  
  
    
        log.info("Object path size:" + objectPathMap.size());


  		




  
  90
  
    
    	try {


  		




  
  91
  
    
	        IMap<Identifier, String> objectPathMap = DistributedMapsFactory.getObjectPathMap();


  		




  
  92
  
    
	        IMap<Identifier, IndexTask> indexQueue = DistributedMapsFactory.getIndexQueue();


  		




  
  93
  
    
	        indexQueue.addEntryListener(this, true);


  		




  
  94
  
    
	        this.source = indexQueue;


  		




  
  95
  
    
	        log.info("System Metadata size: " + indexQueue.size());


  		




  
  96
  
    
	        log.info("Object path size:" + objectPathMap.size());


  		




  
  97
  
    
    	} catch (Exception e) {


  		




  
  98
  
    
    		log.error("Cannot start SystemMetadata listener" , e);


  		




  
  99
  
    
    	}


  		




  102
  100
  
    
    }


  		




  103
  101
  
    

  		




  104
  102
  
    
    /**


  		




  ......




  108
  106
  
    
     */


  		




  109
  107
  
    
    public void stop() throws FileNotFoundException, ServiceFailure {


  		




  110
  108
  
    
    	log.info("stopping index entry listener...");


  		




  111
  
  
    
    	DistributedMapsFactory.getIndexQueue().removeItemListener(this);


  		




  
  109
  
    
    	DistributedMapsFactory.getIndexQueue().removeEntryListener(this);


  		




  112
  110
  
    
    }


  		




  113
  
  
    
    


  		




  114
  
  
    
    /**


  		




  115
  
  
    
     * Get the obsoletes chain of the specified id. The returned list doesn't include


  		




  116
  
  
    
     * the specified id itself. The newer version has the lower index number in the list.


  		




  117
  
  
    
     * Empty list will be returned if there is no document to be obsoleted by this id.


  		




  118
  
  
    
     * @param id


  		




  119
  
  
    
     * @return


  		




  120
  
  
    
     * @throws ServiceFailure


  		




  121
  
  
    
     * @throws FileNotFoundException 


  		




  122
  
  
    
     */


  		




  123
  
  
    
    private List<String> getObsoletes(String id) throws FileNotFoundException, ServiceFailure {


  		




  124
  
  
    
        List<String> obsoletes = new ArrayList<String>();


  		




  125
  
  
    
        while (id != null) {


  		




  126
  
  
    
            SystemMetadata metadata = DistributedMapsFactory.getSystemMetadata(id);


  		




  127
  
  
    
            id = null;//set it to be null in order to stop the while loop if the id can't be assinged to a new value in the following code.


  		




  128
  
  
    
            if(metadata != null) {


  		




  129
  
  
    
                Identifier identifier = metadata.getObsoletes();


  		




  130
  
  
    
                if(identifier != null && identifier.getValue() != null && !identifier.getValue().trim().equals("")) {


  		




  131
  
  
    
                    obsoletes.add(identifier.getValue());


  		




  132
  
  
    
                    id = identifier.getValue();


  		




  133
  
  
    
                } 


  		




  134
  
  
    
            } 


  		




  135
  
  
    
        }


  		




  136
  
  
    
        return obsoletes;


  		




  
  111
  
    

  		




  
  112
  
    
    public void entryRemoved(EntryEvent<Identifier, IndexTask> entryEvent) {


  		




  
  113
  
    
    	// do nothing


  		




  137
  114
  
    
    }


  		




  
  115
  
    
    public void entryEvicted(EntryEvent<Identifier, IndexTask> entryEvent) {


  		




  
  116
  
    
    	// do nothing


  		




  
  117
  
    
    }


  		




  
  118
  
    
    public void entryAdded(EntryEvent<Identifier, IndexTask> entryEvent) {


  		




  
  119
  
    
    	entryUpdated(entryEvent);


  		




  
  120
  
    
    }


  		




  138
  121
  
    
    


  		




  139
  
  
    

  		




  140
  
  
    
	public void itemRemoved(ItemEvent<SystemMetadata> entryEvent) {


  		




  141
  
  
    
		// do nothing - indexing acts on added objects, even if they need to be deleted


  		




  142
  
  
    
	}


  		




  143
  
  
    

  		




  144
  
  
    
	public void itemAdded(ItemEvent<SystemMetadata> entryEvent) {


  		




  
  122
  
    
	public void entryUpdated(EntryEvent<Identifier, IndexTask> entryEvent) {


  		




  145
  123
  
    
	    //System.out.println("===================================calling entryUpdated method ");


  		




  146
  124
  
    
	    log.info("===================================calling SystemMetadataEventListener.itemAdded method ");


  		




  147
  125
  
    
		// add to the index


  		




  148
  
  
    
		Identifier pid = entryEvent.getItem().getIdentifier();


  		




  
  126
  
    
		Identifier pid = entryEvent.getKey();


  		




  149
  127
  
    
		//System.out.println("===================================update the document "+pid.getValue());


  		




  150
  
  
    
		log.info("===================================adding the document "+pid.getValue());


  		




  151
  
  
    
		SystemMetadata systemMetadata = entryEvent.getItem();


  		




  
  128
  
    
		log.info("===================================adding the document " + pid.getValue());


  		




  152
  129
  
    
		


  		




  
  130
  
    
		// what do we have to index?


  		




  
  131
  
    
		IndexTask task = entryEvent.getValue();


  		




  
  132
  
    
		SystemMetadata systemMetadata = task.getSystemMetadata();


  		




  
  133
  
    
		Map<String, List<Object>> fields = task.getFields();


  		




  
  134
  
    
		


  		




  153
  135
  
    
		/*if(systemMetadata == null) {


  		




  154
  136
  
    
		    writeEventLog(systemMetadata, pid, "SystemMetadataEventListener.itemAdded -could not get the SystemMetadata");


  		




  155
  137
  
    
		    return;


  		




  156
  138
  
    
		}*/


  		




  157
  139
  
    
		


  		




  158
  
  
    
		// make sure we remove this object so that it can be re-added in the future


  		




  159
  
  
    
		if (source != null && systemMetadata != null) {


  		




  160
  
  
    
			source.remove(systemMetadata);


  		




  
  140
  
    
		// make sure we remove this task so that it can be re-added in the future


  		




  
  141
  
    
		if (source != null && pid != null) {


  		




  
  142
  
    
			source.remove(pid);


  		




  161
  143
  
    
		}


  		




  162
  
  
    
		solrIndex.update(pid, systemMetadata);


  		




  163
  
  
    
				


  		




  164
  
  
    
		//Identifier obsoletes = systemMetadata.getObsoletes();


  		




  165
  
  
    
		//List<String> obsoletesChain = null;


  		




  166
  
  
    
		/*if (obsoletes != null) {


  		




  167
  
  
    
		    try {


  		




  168
  
  
    
				obsoletesChain = getObsoletes(pid.getValue());


  		




  169
  
  
    
			} catch (Exception e) {


  		




  170
  
  
    
			    String error = "SystemMetadataEventListener.itemAdded -could not look up revision history " + e.getMessage();


  		




  171
  
  
    
			    writeEventLog(systemMetadata, pid, error);


  		




  172
  
  
    
	            log.error(error, e);


  		




  173
  
  
    
	            return;


  		




  174
  
  
    
			}


  		




  175
  
  
    
		}*/


  		




  176
  144
  
    
		


  		




  177
  
  
    
		/*if(objectPath != null) {


  		




  178
  
  
    
		    InputStream data = null;


  		




  179
  
  
    
	        try {


  		




  180
  
  
    
	            data = new FileInputStream(objectPath);


  		




  181
  
  
    
	            solrIndex.update(pid.getValue(), systemMetadata, data);


  		




  182
  
  
    
	            EventlogFactory.createIndexEventLog().remove(pid);


  		




  183
  
  
    
	        } catch (Exception e) {


  		




  184
  
  
    
	            String error = "SystemMetadataEventListener.itemAdded - could not comit the index into the solr server since " + e.getMessage();


  		




  185
  
  
    
	            writeEventLog(systemMetadata, pid, error);


  		




  186
  
  
    
	            log.error(error, e);


  		




  187
  
  
    
	        }


  		




  188
  
  
    
		}*/


  		




  
  145
  
    
		if (systemMetadata != null) {


  		




  
  146
  
    
			solrIndex.update(pid, systemMetadata);


  		




  
  147
  
    
		}


  		




  
  148
  
    
		if (fields != null) {


  		




  
  149
  
    
			solrIndex.insertFields(pid, fields);


  		




  
  150
  
    
		}


  		




  189
  151
  
    
		


  		




  190
  152
  
    
	}


  		




  191
  153
  
    
	


  		












  

    
      metacat-common/src/test/java/edu/ucsb/nceas/metacat/common/query/SolrQueryServiceControllerTest.java
    
  





  4
  4
  
    

  		




  5
  5
  
    
import java.io.IOException;


  		




  6
  6
  
    
import java.io.InputStream;


  		




  7
  
  
    
import java.net.MalformedURLException;


  		




  8
  
  
    
import java.util.List;


  		




  9
  7
  
    
import java.util.Map;


  		




  10
  8
  
    

  		




  11
  9
  
    
import javax.xml.parsers.ParserConfigurationException;


  		




  ......




  47
  45
  
    
     * @throws UnsupportedType 


  		




  48
  46
  
    
     */


  		




  49
  47
  
    
    @Test


  		




  50
  
  
    
    public void testGetValidSchemaFields() throws Exception {


  		




  51
  
  
    
       List<String> fields = SolrQueryServiceController.getInstance().getValidSchemaFields();


  		




  52
  
  
    
       assertTrue(fields != null);


  		




  53
  
  
    
       assertTrue("The number of valid schema fields should be 85 rather than "+fields.size(), fields.size() ==85);


  		




  54
  
  
    
    }


  		




  55
  
  
    
    


  		




  56
  
  
    
    /**


  		




  57
  
  
    
     * Test get get valid schema fields.


  		




  58
  
  
    
     * @throws SAXException 


  		




  59
  
  
    
     * @throws IOException 


  		




  60
  
  
    
     * @throws ParserConfigurationException 


  		




  61
  
  
    
     * @throws NotFound 


  		




  62
  
  
    
     * @throws UnsupportedType 


  		




  63
  
  
    
     */


  		




  64
  
  
    
    @Test


  		




  65
  48
  
    
    public void testgetIndexSchemaFields() throws Exception {


  		




  66
  49
  
    
       Map<String, SchemaField> fields = SolrQueryServiceController.getInstance().getIndexSchemaFields();


  		




  67
  50
  
    
       assertTrue(fields != null);


  		












  

    
      metacat-common/src/main/java/edu/ucsb/nceas/metacat/common/query/HttpSolrQueryService.java
    
  





  202
  202
  
    
    private void getIndexSchemaFieldFromServer() throws MalformedURLException, ParserConfigurationException, IOException, SAXException {


  		




  203
  203
  
    
        //System.out.println("get filed map from server (downloading files) ==========================");


  		




  204
  204
  
    
        SolrConfig config = new SolrConfig("dataone", new InputSource(getSolrConfig())); 


  		




  205
  
  
    
        schema = new IndexSchema(config, "dataone", new InputSource(getSchema()));


  		




  
  205
  
    
        schema = new IndexSchema(config, "dataone", new InputSource(lookupSchema()));


  		




  206
  206
  
    
        fieldMap = schema.getFields();


  		




  207
  207
  
    
    }


  		




  208
  208
  
    
    


  		




  ......




  255
  255
  
    
    /*


  		




  256
  256
  
    
     * Get the schema InputStream from the url which is specified in the metacat.properties and transform it to a Document.


  		




  257
  257
  
    
     */


  		




  258
  
  
    
    private InputStream getSchema() throws MalformedURLException, IOException {


  		




  
  258
  
    
    private InputStream lookupSchema() throws MalformedURLException, IOException {


  		




  259
  259
  
    
        String schemaURLAppendix = Settings.getConfiguration().getString(SOLR_SCHEMA_URLAPPENDIX);


  		




  260
  260
  
    
        String schemaURL = solrServerBaseURL+schemaURLAppendix;


  		




  261
  261
  
    
        return (new URL(schemaURL)).openStream();


  		












  

    
      metacat-common/src/main/java/edu/ucsb/nceas/metacat/common/query/SolrQueryService.java
    
  





  96
  96
  
    
     */


  		




  97
  97
  
    
    public abstract Map<String, SchemaField> getIndexSchemaFields() throws Exception;


  		




  98
  98
  
    
    


  		




  99
  
  
    
    /**


  		




  
  99
  
    
    public IndexSchema getSchema() {


  		




  
  100
  
    
		return schema;


  		




  
  101
  
    
	}


  		




  
  102
  
    

  		




  
  103
  
    
	/**


  		




  100
  104
  
    
     * Get the version of the solr server.


  		




  101
  105
  
    
     * @return


  		




  102
  106
  
    
     */


  		












  

    
      metacat-common/src/main/java/edu/ucsb/nceas/metacat/common/query/SolrQueryServiceController.java
    
  





  33
  33
  
    
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;


  		




  34
  34
  
    
import org.apache.solr.common.params.SolrParams;


  		




  35
  35
  
    
import org.apache.solr.core.CoreContainer;


  		




  
  36
  
    
import org.apache.solr.schema.IndexSchema;


  		




  36
  37
  
    
import org.apache.solr.schema.SchemaField;


  		




  37
  38
  
    
import org.dataone.service.exceptions.NotFound;


  		




  38
  39
  
    
import org.dataone.service.exceptions.NotImplemented;


  		




  ......




  141
  142
  
    
            return httpQueryService.getIndexSchemaFields();


  		




  142
  143
  
    
        }


  		




  143
  144
  
    
    }


  		




  144
  
  
    
    


  		




  145
  
  
    
   


  		




  146
  
  
    
    


  		




  
  145
  
    

  		




  147
  146
  
    
    /**


  		




  148
  
  
    
     * Get the list of the valid field name (moved the fields names of the CopyFieldTarget).


  		




  
  147
  
    
     * Access the SOLR index schema


  		




  149
  148
  
    
     * @return


  		




  150
  149
  
    
     * @throws SAXException 


  		




  151
  150
  
    
     * @throws IOException 


  		




  152
  151
  
    
     * @throws ParserConfigurationException 


  		




  153
  152
  
    
     * @throws MalformedURLException 


  		




  154
  153
  
    
     */


  		




  155
  
  
    
    public List<String> getValidSchemaFields() throws MalformedURLException, ParserConfigurationException, IOException, SAXException {


  		




  
  154
  
    
    public IndexSchema getSchema() throws MalformedURLException, ParserConfigurationException, IOException, SAXException {


  		




  156
  155
  
    
        if(isEmbeddedSolrServer) {


  		




  157
  
  
    
            return embeddedQueryService.getValidSchemaFields();


  		




  
  156
  
    
            return embeddedQueryService.getSchema();


  		




  158
  157
  
    
        } else{


  		




  159
  
  
    
            return httpQueryService.getValidSchemaField();


  		




  
  158
  
    
            return httpQueryService.getSchema();


  		




  160
  159
  
    
        }


  		




  161
  160
  
    
       


  		




  162
  161
  
    
    }


  		












  

    
      metacat-common/src/main/java/edu/ucsb/nceas/metacat/common/index/IndexTask.java
    
  





  
  1
  
    
package edu.ucsb.nceas.metacat.common.index;


  		




  
  2
  
    

  		




  
  3
  
    
import java.io.Serializable;


  		




  
  4
  
    
import java.util.List;


  		




  
  5
  
    
import java.util.Map;


  		




  
  6
  
    

  		




  
  7
  
    
import org.dataone.service.types.v1.SystemMetadata;


  		




  
  8
  
    

  		




  
  9
  
    
public class IndexTask implements Serializable {


  		




  
  10
  
    
	


  		




  
  11
  
    
	private SystemMetadata systemMetadata;


  		




  
  12
  
    
	


  		




  
  13
  
    
	private Map<String, List<Object>> fields;


  		




  
  14
  
    
	


  		




  
  15
  
    
	public SystemMetadata getSystemMetadata() {


  		




  
  16
  
    
		return systemMetadata;


  		




  
  17
  
    
	}


  		




  
  18
  
    

  		




  
  19
  
    
	public void setSystemMetadata(SystemMetadata systemMetadata) {


  		




  
  20
  
    
		this.systemMetadata = systemMetadata;


  		




  
  21
  
    
	}


  		




  
  22
  
    

  		




  
  23
  
    
	public Map<String, List<Object>> getFields() {


  		




  
  24
  
    
		return fields;


  		




  
  25
  
    
	}


  		




  
  26
  
    

  		




  
  27
  
    
	public void setFields(Map<String, List<Object>> fields) {


  		




  
  28
  
    
		this.fields = fields;


  		




  
  29
  
    
	}


  		




  
  30
  
    

  		




  
  31
  
    
}


  		




  0
  32
  
    

  		












  

    
      src/edu/ucsb/nceas/metacat/EventLog.java
    
  





  29
  29
  
    
import java.sql.Timestamp;


  		




  30
  30
  
    
import java.util.ArrayList;


  		




  31
  31
  
    
import java.util.Date;


  		




  
  32
  
    
import java.util.HashMap;


  		




  32
  33
  
    
import java.util.List;


  		




  
  34
  
    
import java.util.Map;


  		




  33
  35
  
    
import java.util.Vector;


  		




  34
  36
  
    

  		




  35
  37
  
    
import org.apache.log4j.Logger;


  		




  ......




  44
  46
  
    
import edu.ucsb.nceas.metacat.database.DBConnection;


  		




  45
  47
  
    
import edu.ucsb.nceas.metacat.database.DBConnectionPool;


  		




  46
  48
  
    
import edu.ucsb.nceas.metacat.database.DatabaseService;


  		




  
  49
  
    
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;


  		




  47
  50
  
    
import edu.ucsb.nceas.metacat.properties.PropertyService;


  		




  48
  51
  
    
import edu.ucsb.nceas.metacat.util.DocumentUtil;


  		




  49
  52
  
    
import edu.ucsb.nceas.utilities.PropertyNotFoundException;


  		




  ......




  114
  117
  
    
    public void log(String ipAddress, String userAgent, String principal, String docid, String event) {


  		




  115
  118
  
    
        EventLogData logData = new EventLogData(ipAddress, principal, docid, event);


  		




  116
  119
  
    
        insertLogEntry(logData);


  		




  
  120
  
    
        


  		




  
  121
  
    
        // update the event information in the index


  		




  
  122
  
    
        try {


  		




  
  123
  
    
	        String localId = DocumentUtil.getSmartDocId(docid);


  		




  
  124
  
    
			int rev = DocumentUtil.getRevisionFromAccessionNumber(docid);


  		




  
  125
  
    
			


  		




  
  126
  
    
	        String guid = IdentifierManager.getInstance().getGUID(localId, rev);


  		




  
  127
  
    
	        Identifier pid = new Identifier();


  		




  
  128
  
    
	        pid.setValue(guid);


  		




  
  129
  
    
	        


  		




  
  130
  
    
	        // submit for indexing


  		




  
  131
  
    
	        MetacatSolrIndex.getInstance().submit(pid, null, this.getIndexFields(pid, event));


  		




  
  132
  
    
	        


  		




  
  133
  
    
        } catch (Exception e) {


  		




  
  134
  
    
        	logMetacat.error("Could not update event index information", e);


  		




  
  135
  
    
        }


  		




  117
  136
  
    
    }


  		




  118
  137
  
    
    


  		




  
  138
  
    
    public Map<String, List<Object>> getIndexFields(Identifier pid, String event) {


  		




  
  139
  
    
    	// update the search index for the event


  		




  
  140
  
    
        try {


  		




  
  141
  
    
        	


  		




  
  142
  
    
        	Event d1Event = Event.convert(event);


  		




  
  143
  
    
        	String fieldName = d1Event.xmlValue() + "_count_i";


  		




  
  144
  
    
        	int eventCount = 0;


  		




  
  145
  
    
        	


  		




  
  146
  
    
        	String docid = IdentifierManager.getInstance().getLocalId(pid.getValue());


  		




  
  147
  
    
        	Log eventLog = this.getD1Report(null, null, new String[] {docid}, d1Event, null, null, false, 0, 0);


  		




  
  148
  
    
        	eventCount = eventLog.getTotal();


  		




  
  149
  
    

  		




  
  150
  
    
	        List<Object> values = new ArrayList<Object>();


  		




  
  151
  
    
			values.add(eventCount);


  		




  
  152
  
    
	        Map<String, List<Object>> fields = new HashMap<String, List<Object>>();


  		




  
  153
  
    
	        fields.put(fieldName, values);


  		




  
  154
  
    
	        


  		




  
  155
  
    
	        return fields;


  		




  
  156
  
    
	        


  		




  
  157
  
    
        } catch (Exception e) {


  		




  
  158
  
    
        	logMetacat.error("Could not update event index information", e);


  		




  
  159
  
    
        	return null;


  		




  
  160
  
    
        }


  		




  
  161
  
    
    }


  		




  
  162
  
    
    


  		




  119
  163
  
    
    /**


  		




  120
  164
  
    
     * Insert a single log event record to the database.


  		




  121
  165
  
    
     * 


  		












  

    
      src/edu/ucsb/nceas/metacat/DocumentImpl.java
    
  





  68
  68
  
    
import edu.ucsb.nceas.metacat.database.DBConnectionPool;


  		




  69
  69
  
    
import edu.ucsb.nceas.metacat.database.DatabaseService;


  		




  70
  70
  
    
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;


  		




  
  71
  
    
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;


  		




  71
  72
  
    
import edu.ucsb.nceas.metacat.properties.PropertyService;


  		




  72
  73
  
    
import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;


  		




  73
  74
  
    
import edu.ucsb.nceas.metacat.replication.ReplicationService;


  		




  ......




  3445
  3446
  
    
				sysMeta.setArchived(true);


  		




  3446
  3447
  
    
            	sysMeta.setDateSysMetadataModified(Calendar.getInstance().getTime());


  		




  3447
  3448
  
    
				HazelcastService.getInstance().getSystemMetadataMap().put(guid, sysMeta);


  		




  3448
  
  
    
				// submit for indexing


  		




  3449
  
  
    
                HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  3449
  
    
                MetacatSolrIndex.getInstance().submit(guid, sysMeta, null);


  		




  3450
  3450
  
    
            }


  		




  3451
  3451
  
    
            


  		




  3452
  3452
  
    
            // clear cache after inserting or updating a document


  		












  

    
      src/edu/ucsb/nceas/metacat/MetacatHandler.java
    
  





  68
  68
  
    
import org.apache.commons.io.IOUtils;


  		




  69
  69
  
    
import org.apache.commons.io.input.XmlStreamReader;


  		




  70
  70
  
    
import org.apache.log4j.Logger;


  		




  
  71
  
    
import org.dataone.service.types.v1.Event;


  		




  71
  72
  
    
import org.dataone.service.types.v1.Identifier;


  		




  72
  73
  
    
import org.dataone.service.types.v1.SystemMetadata;


  		




  73
  74
  
    
import org.ecoinformatics.eml.EMLParser;


  		




  ......




  93
  94
  
    
import edu.ucsb.nceas.metacat.dataquery.DataQuery;


  		




  94
  95
  
    
import edu.ucsb.nceas.metacat.event.MetacatDocumentEvent;


  		




  95
  96
  
    
import edu.ucsb.nceas.metacat.event.MetacatEventService;


  		




  
  97
  
    
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;


  		




  96
  98
  
    
import edu.ucsb.nceas.metacat.properties.PropertyService;


  		




  97
  99
  
    
import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;


  		




  98
  100
  
    
import edu.ucsb.nceas.metacat.service.SessionService;


  		




  ......




  1829
  1831
  
    
                    HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);


  		




  1830
  1832
  
    
                    


  		




  1831
  1833
  
    
                    // submit for indexing


  		




  1832
  
  
    
                    HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  1834
  
    
                    MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null);


  		




  1833
  1835
  
    
                    


  		




  1834
  1836
  
    
                  } catch ( McdbDocNotFoundException dnfe ) {


  		




  1835
  1837
  
    
                    logMetacat.debug(


  		




  ......




  2655
  2657
  
    
                    Identifier identifier = new Identifier();


  		




  2656
  2658
  
    
                    identifier.setValue(id);


  		




  2657
  2659
  
    
					SystemMetadata sysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(identifier);


  		




  2658
  
  
    
					if(sysMeta == null) {


  		




  
  2660
  
    
					if (sysMeta == null) {


  		




  2659
  2661
  
    
					     failedList.add(id);


  		




  2660
  2662
  
    
					     logMetacat.info("no system metadata was found for pid " + id);


  		




  2661
  2663
  
    
					} else {


  		




  2662
  
  
    
					    HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  2664
  
    
						try {


  		




  
  2665
  
    
							// submit for indexing


  		




  
  2666
  
    
						    Map<String, List<Object>> fields = EventLog.getInstance().getIndexFields(identifier, Event.READ.xmlValue());


  		




  
  2667
  
    
	                        MetacatSolrIndex.getInstance().submit(identifier, sysMeta, fields);


  		




  
  2668
  
    
						} catch (Exception e) {


  		




  
  2669
  
    
							failedList.add(id);


  		




  
  2670
  
    
						    logMetacat.info("Error submitting to index for pid " + id);


  		




  
  2671
  
    
						    continue;


  		




  
  2672
  
    
						}


  		




  2663
  2673
  
    
					    successList.add(id);


  		




  2664
  2674
  
    
					    logMetacat.info("done queueing doc index for pid " + id);


  		




  2665
  2675
  
    
					}


  		




  ......




  2749
  2759
  
    
                        Identifier identifier = new Identifier();


  		




  2750
  2760
  
    
                        identifier.setValue(id);


  		




  2751
  2761
  
    
                        SystemMetadata sysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(identifier);


  		




  2752
  
  
    
                        if(sysMeta != null) {


  		




  2753
  
  
    
                            HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  2754
  
  
    
                            results.append("<pid>" + id + "</pid>\n");


  		




  
  2762
  
    
                        if (sysMeta != null) {


  		




  
  2763
  
    
                        	


  		




  
  2764
  
    
                            // submit for indexing


  		




  
  2765
  
    
    					    Map<String, List<Object>> fields = EventLog.getInstance().getIndexFields(identifier, Event.READ.xmlValue());


  		




  
  2766
  
    
                            MetacatSolrIndex.getInstance().submit(identifier, sysMeta, fields);


  		




  
  2767
  
    

  		




  
  2768
  
    
    					    results.append("<pid>" + id + "</pid>\n");


  		




  2755
  2769
  
    
                            logMetacat.debug("queued SystemMetadata for index on pid: " + id);


  		




  2756
  2770
  
    
                        }


  		




  2757
  2771
  
    
                        


  		




  ......




  3214
  3228
  
    
                            HazelcastService.getInstance().getSystemMetadataMap().put(sm.getIdentifier(), sm);


  		




  3215
  3229
  
    
                            


  		




  3216
  3230
  
    
                            // submit for indexing


  		




  3217
  
  
    
                            HazelcastService.getInstance().getIndexQueue().add(sm);


  		




  
  3231
  
    
                            MetacatSolrIndex.getInstance().submit(sm.getIdentifier(), sm, null);


  		




  3218
  3232
  
    
					        


  		




  3219
  3233
  
    
                        } catch (Exception ee) {


  		




  3220
  3234
  
    
                            // If the file did not exist before this method was 


  		












  

    
      src/edu/ucsb/nceas/metacat/dataone/MNodeService.java
    
  





  55
  55
  
    
import org.apache.commons.beanutils.BeanUtils;


  		




  56
  56
  
    
import org.apache.commons.io.IOUtils;


  		




  57
  57
  
    
import org.apache.log4j.Logger;


  		




  58
  
  
    
import org.apache.wicket.protocol.http.mock.MockHttpServletRequest;


  		




  59
  58
  
    
import org.dataone.client.CNode;


  		




  60
  59
  
    
import org.dataone.client.D1Client;


  		




  61
  60
  
    
import org.dataone.client.MNode;


  		




  ......




  126
  125
  
    
import edu.ucsb.nceas.ezid.EZIDException;


  		




  127
  126
  
    
import edu.ucsb.nceas.metacat.DBQuery;


  		




  128
  127
  
    
import edu.ucsb.nceas.metacat.DBTransform;


  		




  129
  
  
    
import edu.ucsb.nceas.metacat.DocumentImpl;


  		




  130
  128
  
    
import edu.ucsb.nceas.metacat.EventLog;


  		




  131
  129
  
    
import edu.ucsb.nceas.metacat.IdentifierManager;


  		




  132
  130
  
    
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;


  		




  133
  
  
    
import edu.ucsb.nceas.metacat.McdbException;


  		




  134
  131
  
    
import edu.ucsb.nceas.metacat.MetaCatServlet;


  		




  135
  132
  
    
import edu.ucsb.nceas.metacat.MetacatHandler;


  		




  136
  133
  
    
import edu.ucsb.nceas.metacat.common.query.EnabledQueryEngines;


  		




  ......




  1257
  1254
  
    
            // update the local copy of system metadata for the pid


  		




  1258
  1255
  
    
            try {


  		




  1259
  1256
  
    
                HazelcastService.getInstance().getSystemMetadataMap().put(newSysMeta.getIdentifier(), newSysMeta);


  		




  1260
  
  
    
                // submit for indexing


  		




  1261
  
  
    
                HazelcastService.getInstance().getIndexQueue().add(newSysMeta);


  		




  1262
  1257
  
    
                logMetacat.info("Updated local copy of system metadata for pid " +


  		




  1263
  1258
  
    
                    pid.getValue() + " after change notification from the CN.");


  		




  1264
  1259
  
    
                


  		




  ......




  1271
  1266
  
    
                sf.initCause(e);


  		




  1272
  1267
  
    
                throw sf;


  		




  1273
  1268
  
    
            }


  		




  
  1269
  
    
            


  		




  
  1270
  
    
            // submit for indexing


  		




  
  1271
  
    
            try {


  		




  
  1272
  
    
				MetacatSolrIndex.getInstance().submit(newSysMeta.getIdentifier(), newSysMeta, null);


  		




  
  1273
  
    
			} catch (Exception e) {


  		




  
  1274
  
    
                logMetacat.error("Could not submit changed systemMetadata for indexing, pid: " + newSysMeta.getIdentifier().getValue(), e);


  		




  
  1275
  
    
			}


  		




  1274
  1276
  
    
        }


  		




  1275
  1277
  
    
        


  		




  1276
  1278
  
    
        return true;


  		












  

    
      src/edu/ucsb/nceas/metacat/dataone/D1NodeService.java
    
  





  85
  85
  
    
import edu.ucsb.nceas.metacat.database.DBConnection;


  		




  86
  86
  
    
import edu.ucsb.nceas.metacat.database.DBConnectionPool;


  		




  87
  87
  
    
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;


  		




  
  88
  
    
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;


  		




  88
  89
  
    
import edu.ucsb.nceas.metacat.properties.PropertyService;


  		




  89
  90
  
    
import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;


  		




  90
  91
  
    
import edu.ucsb.nceas.utilities.PropertyNotFoundException;


  		




  ......




  427
  428
  
    
    	// lock and unlock of the pid happens in the subclass


  		




  428
  429
  
    
    	HazelcastService.getInstance().getSystemMetadataMap().put(sysmeta.getIdentifier(), sysmeta);


  		




  429
  430
  
    
    	// submit for indexing


  		




  430
  
  
    
        HazelcastService.getInstance().getIndexQueue().add(sysmeta);


  		




  
  431
  
    
        MetacatSolrIndex.getInstance().submit(sysmeta.getIdentifier(), sysmeta, null);


  		




  
  432
  
    
        


  		




  431
  433
  
    
    } catch (Exception e) {


  		




  432
  434
  
    
    	logMetacat.error("Problem creating system metadata: " + pid.getValue(), e);


  		




  433
  435
  
    
        throw new ServiceFailure("1190", e.getMessage());


  		




  ......




  1320
  1322
  
    
        // note: the calling subclass handles the map hazelcast lock/unlock


  		




  1321
  1323
  
    
      	HazelcastService.getInstance().getSystemMetadataMap().put(sysmeta.getIdentifier(), sysmeta);


  		




  1322
  1324
  
    
      	// submit for indexing


  		




  1323
  
  
    
        HazelcastService.getInstance().getIndexQueue().add(sysmeta);


  		




  
  1325
  
    
        MetacatSolrIndex.getInstance().submit(sysmeta.getIdentifier(), sysmeta, null);


  		




  1324
  1326
  
    
      } catch (Exception e) {


  		




  1325
  1327
  
    
          throw new ServiceFailure("1190", e.getMessage());


  		




  1326
  1328
  
    
          


  		




  ......




  1341
  1343
  
    
            HazelcastService.getInstance().getSystemMetadataMap().lock(sysMeta.getIdentifier());


  		




  1342
  1344
  
    
            HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);


  		




  1343
  1345
  
    
            // submit for indexing


  		




  1344
  
  
    
            HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  1346
  
    
            MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null);


  		




  1345
  1347
  
    
        } catch (Exception e) {


  		




  1346
  1348
  
    
            throw new ServiceFailure("4862", e.getMessage());


  		




  1347
  1349
  
    

  		












  

    
      src/edu/ucsb/nceas/metacat/dataone/hazelcast/HazelcastService.java
    
  





  62
  62
  
    

  		




  63
  63
  
    
import edu.ucsb.nceas.metacat.IdentifierManager;


  		




  64
  64
  
    
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;


  		




  
  65
  
    
import edu.ucsb.nceas.metacat.common.index.IndexTask;


  		




  65
  66
  
    
import edu.ucsb.nceas.metacat.common.index.event.IndexEvent;


  		




  66
  67
  
    
import edu.ucsb.nceas.metacat.properties.PropertyService;


  		




  67
  68
  
    
import edu.ucsb.nceas.metacat.shared.BaseService;


  		




  ......




  75
  76
  
    
public class HazelcastService extends BaseService


  		




  76
  77
  
    
  implements EntryListener<Identifier, SystemMetadata>, MembershipListener, LifecycleListener, ItemListener<Identifier> {


  		




  77
  78
  
    
  


  		




  78
  
  
    
  private static final String SINCE_PROPERTY = "dateSysMetadataModified";


  		




  79
  
  
    

  		




  80
  79
  
    
  private static final String MISSING_PID_PREFIX = "missing-";


  		




  81
  80
  
    

  		




  82
  81
  
    
/* The instance of the logging class */


  		




  ......




  105
  104
  
    
  


  		




  106
  105
  
    
  /* The Hazelcast distributed index queue */


  		




  107
  106
  
    
  private String hzIndexQueue;


  		




  108
  
  
    
  private ISet<SystemMetadata> indexQueue;


  		




  
  107
  
    
  private IMap<Identifier, IndexTask> indexQueue;


  		




  109
  108
  
    
  


  		




  110
  109
  
    
  /* The Hazelcast distributed index event map */


  		




  111
  110
  
    
  private String hzIndexEventMap;


  		




  ......




  202
  201
  
    

  		




  203
  202
  
    
      // for index tasks


  		




  204
  203
  
    
      hzIndexQueue = PropertyService.getProperty("index.hazelcast.indexqueue");


  		




  205
  
  
    
      indexQueue = this.hzInstance.getSet(hzIndexQueue);


  		




  
  204
  
    
      indexQueue = this.hzInstance.getMap(hzIndexQueue);


  		




  206
  205
  
    

  		




  207
  206
  
    
      // for index events (failures)


  		




  208
  207
  
    
      hzIndexEventMap = PropertyService.getProperty("index.hazelcast.indexeventmap");


  		




  ......




  259
  258
  
    
   * Get the index queue


  		




  260
  259
  
    
   * @return the set of SystemMetadata to be indexed


  		




  261
  260
  
    
   */


  		




  262
  
  
    
  public ISet<SystemMetadata> getIndexQueue() {


  		




  
  261
  
    
  public IMap<Identifier, IndexTask> getIndexQueue() {


  		




  263
  262
  
    
      return indexQueue;


  		




  264
  263
  
    
  }


  		




  265
  264
  
    
  


  		












  

    
      src/edu/ucsb/nceas/metacat/index/MetacatSolrIndex.java
    
  





  30
  30
  
    
import java.io.Writer;


  		




  31
  31
  
    
import java.sql.SQLException;


  		




  32
  32
  
    
import java.util.Hashtable;


  		




  
  33
  
    
import java.util.List;


  		




  
  34
  
    
import java.util.Map;


  		




  33
  35
  
    

  		




  34
  36
  
    
import java.util.Set;


  		




  35
  37
  
    

  		




  ......




  46
  48
  
    
import org.dataone.service.exceptions.NotFound;


  		




  47
  49
  
    
import org.dataone.service.exceptions.NotImplemented;


  		




  48
  50
  
    
import org.dataone.service.exceptions.UnsupportedType;


  		




  
  51
  
    
import org.dataone.service.types.v1.Identifier;


  		




  49
  52
  
    
import org.dataone.service.types.v1.Subject;


  		




  
  53
  
    
import org.dataone.service.types.v1.SystemMetadata;


  		




  50
  54
  
    
import org.xml.sax.SAXException;


  		




  51
  55
  
    

  		




  52
  56
  
    
import edu.ucsb.nceas.metacat.DBTransform;


  		




  
  57
  
    
import edu.ucsb.nceas.metacat.common.index.IndexTask;


  		




  53
  58
  
    
import edu.ucsb.nceas.metacat.common.query.SolrQueryResponseWriterFactory;


  		




  54
  59
  
    
import edu.ucsb.nceas.metacat.common.query.SolrQueryService;


  		




  55
  60
  
    
import edu.ucsb.nceas.metacat.common.query.SolrQueryServiceController;


  		




  56
  61
  
    
import edu.ucsb.nceas.metacat.common.query.stream.ContentTypeByteArrayInputStream;


  		




  
  62
  
    
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;


  		




  57
  63
  
    
import edu.ucsb.nceas.utilities.PropertyNotFoundException;


  		




  58
  64
  
    

  		




  59
  65
  
    

  		




  ......




  162
  168
  
    
        return inputStream;


  		




  163
  169
  
    
     


  		




  164
  170
  
    
    }


  		




  
  171
  
    
    


  		




  
  172
  
    
    public void submit(Identifier pid, SystemMetadata systemMetadata, Map<String, List<Object>> fields) {


  		




  
  173
  
    
    	IndexTask task = new IndexTask();


  		




  
  174
  
    
    	task.setSystemMetadata(systemMetadata);


  		




  
  175
  
    
    	task.setFields(fields);


  		




  
  176
  
    
		HazelcastService.getInstance().getIndexQueue().put(pid, task );


  		




  
  177
  
    
    }


  		




  
  178
  
    
    


  		




  165
  179
  
    

  		




  166
  180
  
    
}


  		












  

    
      src/edu/ucsb/nceas/metacat/replication/ReplicationHandler.java
    
  





  69
  69
  
    
import edu.ucsb.nceas.metacat.database.DBConnection;


  		




  70
  70
  
    
import edu.ucsb.nceas.metacat.database.DBConnectionPool;


  		




  71
  71
  
    
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;


  		




  
  72
  
    
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;


  		




  72
  73
  
    
import edu.ucsb.nceas.metacat.properties.PropertyService;


  		




  73
  74
  
    
import edu.ucsb.nceas.metacat.shared.HandlerException;


  		




  74
  75
  
    
import edu.ucsb.nceas.metacat.util.DocumentUtil;


  		




  ......




  402
  403
  
    
    	  logReplication.debug("Saving SystemMetadata to shared map: " + sysMeta.getIdentifier().getValue());


  		




  403
  404
  
    
      	  HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);


  		




  404
  405
  
    
      	  // submit for indexing


  		




  405
  
  
    
          HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  406
  
    
          MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null);


  		




  406
  407
  
    
      }


  		




  407
  408
  
    
   	  


  		




  408
  409
  
    
      docinfoParser.parse(new InputSource(new StringReader(docInfoStr)));


  		




  ......




  581
  582
  
    
    	  // save the system metadata


  		




  582
  583
  
    
    	  HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);


  		




  583
  584
  
    
    	  // submit for indexing


  		




  584
  
  
    
          HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  585
  
    
          MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null);


  		




  585
  586
  
    

  		




  586
  587
  
    
      }


  		




  587
  588
  
    
   	  


  		




  ......




  880
  881
  
    
										.getBytes("UTF-8")));


  		




  881
  882
  
    
				HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);


  		




  882
  883
  
    
				// submit for indexing


  		




  883
  
  
    
                HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  884
  
    
                MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null);


  		




  884
  885
  
    
			}


  		




  885
  886
  
    

  		




  886
  887
  
    
			logReplication.info("ReplicationHandler.handleSystemMetadata - Successfully replicated system metadata for guid: "


  		












  

    
      src/edu/ucsb/nceas/metacat/replication/ReplicationService.java
    
  





  87
  87
  
    
import edu.ucsb.nceas.metacat.database.DBConnectionPool;


  		




  88
  88
  
    
import edu.ucsb.nceas.metacat.database.DatabaseService;


  		




  89
  89
  
    
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;


  		




  
  90
  
    
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;


  		




  90
  91
  
    
import edu.ucsb.nceas.metacat.properties.PropertyService;


  		




  91
  92
  
    
import edu.ucsb.nceas.metacat.shared.BaseService;


  		




  92
  93
  
    
import edu.ucsb.nceas.metacat.shared.HandlerException;


  		




  ......




  591
  592
  
    
				// save the system metadata


  		




  592
  593
  
    
				HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);


  		




  593
  594
  
    
				// submit for indexing


  		




  594
  
  
    
                HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  595
  
    
                MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null);


  		




  595
  596
  
    
			}


  		




  596
  597
  
    
      


  		




  597
  598
  
    
			// dates


  		




  ......




  888
  889
  
    
	      	  // save the system metadata


  		




  889
  890
  
    
	      	  HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);


  		




  890
  891
  
    
	      	  // submit for indexing


  		




  891
  
  
    
              HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  892
  
    
              MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null);


  		




  892
  893
  
    
	        }


  		




  893
  894
  
    
	        


  		




  894
  895
  
    
	        // process the access control


  		




  ......




  1160
  1161
  
    
							new ByteArrayInputStream(systemMetadataXML.getBytes("UTF-8")));


  		




  1161
  1162
  
    
				HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);


  		




  1162
  1163
  
    
				// submit for indexing


  		




  1163
  
  
    
                HazelcastService.getInstance().getIndexQueue().add(sysMeta);


  		




  
  1164
  
    
                MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null);


  		




  1164
  1165
  
    
			}


  		




  1165
  1166
  
    
      


  		




  1166
  1167
  
    
			logReplication.info("ReplicationService.handleForceReplicateSystemMetadataRequest - processed guid: " + guid);


  		












Also available in:  Unified diff