Project

General

Profile

« Previous | Next » 

Revision 7627

Added by Jing Tao over 11 years ago

Use an update method to replace the insert and update methods in the SolrIndex class.

View differences:

metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/SolrIndexIT.java
33 33 
    private static final String EMLFILEPATH = "src/test/resources/eml-example.xml";
34 34 
    private static final String SYSTEMMETAUPDATEFILEPATH = "src/test/resources/eml-updating-system-meta-example.xml";
35 35 
    private static final String EMLUPDATEFILEPATH = "src/test/resources/eml-updating-example.xml";
36 
    private static final String SYSTEMMETAARCHIVEFILEPATH = "src/test/resources/eml-archive-system-meta-example.xml";
36 37 
    private static final String id = "urn:uuid:606a19dd-b531-4bf4-b5a5-6d06c3d39098";
37 38 
    private static final String newId = "urn:uuid:606a19dd-b531-4bf4-b5a5-6d06c3d39099";
38 39
......
63 64
64 65 
       //InputStream systemInputStream = new FileInputStream(new File(SYSTEMMETAFILEPATH));
65 66 
       SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAFILEPATH);
66 
       InputStream emlInputStream = new FileInputStream(new File(EMLFILEPATH));
67 
       solrIndex.insert(id, systemMetadata, emlInputStream);
67 
       InputStream emlInputStream = new FileInputStream(new File(EMLFILEPATH));
68 
       List<String> chain = null;
69 
       solrIndex.update(id, chain, systemMetadata, emlInputStream);
68 70 
       String result = doQuery(solrIndex.getSolrServer());
69 71 
       List<String> ids = solrIndex.getSolrIds();
70 
       assertTrue(ids.size() == 1);
72 
       //assertTrue(ids.size() == 1);
73 
       boolean foundId = false;
71 74 
       for(String identifiers :ids) {
72 
           assertTrue(id.equals(identifiers));
75 
           if(id.equals(identifiers)) {
76 
               foundId = true;
77 
           }
73 78 
       }
79 
       assertTrue(foundId);
74 80 
       assertTrue(result.contains("version1"));
75 81
76 82 
    }
......
95 101 
     * Test building index for an insert.
96 102 
     */
97 103 
    @Test
98 
    public void testDelete() throws Exception {
104 
    public void testArchive() throws Exception {
99 105 
       SolrIndex solrIndex = generateSolrIndex();
100 
       solrIndex.remove(newId);
106 
       //InputStream systemInputStream = new FileInputStream(new File(SYSTEMMETAFILEPATH));
107 
       //System metadata's archive is true.
108 
       SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAARCHIVEFILEPATH);
109 
       InputStream emlInputStream = new FileInputStream(new File(EMLUPDATEFILEPATH));
110 
       ArrayList<String> obsoletes = new ArrayList<String>();
111 
       obsoletes.add(id);
112 
       obsoletes.add("tao");
113 
       solrIndex.update(newId, obsoletes, systemMetadata, emlInputStream);
101 114 
       String result = doQuery(solrIndex.getSolrServer());
102 115 
       assertTrue(!result.contains("version1"));
103 116 
       assertTrue(!result.contains("version2"));
metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java
275 275 
    }
276 276
277 277 
    /**
278 
     * Generate indexes for a newly inserted document.
278 
     * Insert the indexes for a document.
279 279 
     * @param pid  the id of this document
280 280 
     * @param systemMetadata  the system metadata associated with the data object
281 281 
     * @param data  the data object itself
282 282 
     * @throws SolrServerException
283 283 
     * @throws JiBXException
284 284 
     */
285 
    public void insert(String pid, SystemMetadata systemMetadata, InputStream data)
285 
    private void insert(String pid, SystemMetadata systemMetadata, InputStream data)
286 286 
                    throws IOException, SAXException, ParserConfigurationException,
287 287 
                    XPathExpressionException, SolrServerException, JiBXException {
288 288 
        checkParams(pid, systemMetadata, data);
......
322 322 
    }
323 323
324 324 
    /**
325 
     * Update an existed document. First, remove the index of the old one. Second,
326 
     * insert the new document
327 
     * @param newPid  the new id of the document
325 
     * Update the solr index. This method handles the three scenarios:
326 
     * 1. Archive (or delete) - if the the system metadata shows the value of the archive is true,
327 
     *    remove the index for the document and its previous versions if it has.
328 
     * 2. Update an existing doc - if the the system metadata shows the value of the archive is false and it has an obsoletes,
329 
     *    remove the index for the previous version(s) and generate new index for the doc.
330 
     * 3. Add a new doc - if the system metadata shows the value of the archive is false and it hasn't an obsoletes, generate the
331 
     *    index for the doc.
332 
     * @param pid  the id of the document
328 333 
     * @param obsoleteIds  the chain of the obsoletes by this id
329 334 
     * @param systemMetadata  the system metadata associated with the data object
330 335 
     * @param data  the data object itself
331 336 
     * @throws SolrServerException
332 337 
     * @throws JiBXException
333 338 
     */
334 
    public void update(String newPid, List<String> obsoleteIds, SystemMetadata systemMetadata, InputStream data)
339 
    public void update(String pid, List<String> obsoleteIds, SystemMetadata systemMetadata, InputStream data)
335 340 
                    throws IOException, SAXException, ParserConfigurationException,
336 341 
                    XPathExpressionException, SolrServerException, JiBXException {
337 
        checkParams(newPid, systemMetadata, data);
338 
        Identifier oldIdentifier = systemMetadata.getObsoletes();
339 
        if(oldIdentifier == null) {
340 
            throw new SolrServerException("The system metadata of the new document doesn't have the obsoletes element in the update operation.");
341 
        }
342
343 
        if(obsoleteIds == null || obsoleteIds.isEmpty()) {
344 
            throw new SolrServerException("The obsoletes chain can't be null in the update operation.");
342 
        checkParams(pid, systemMetadata, data);
343 
        boolean isArchive = systemMetadata.getArchived();
344 
        if(isArchive) {
345 
            //archive(delete)
346 
            Identifier obsolete = systemMetadata.getObsoletes();
347 
            if(obsolete != null) {
348 
                removeObsoletesChain(obsolete.getValue(), obsoleteIds);
349 
            }
350 
            remove(pid);
351 
        } else {
352 
            Identifier obsolete = systemMetadata.getObsoletes();
353 
            if(obsolete != null) {
354 
                removeObsoletesChain(obsolete.getValue(), obsoleteIds);
355 
            }
356 
            //generate index for either add or update.
357 
            insert(pid, systemMetadata, data);
345 358 
        }
346 
        if(!obsoleteIds.contains(oldIdentifier.getValue())) {
347 
            throw new SolrServerException("The obsoletes elment in thesystem metadata is not in the obsoleteIds list in the update operation.");
348 
        }
349 
        remove(obsoleteIds);
350 
        insert(newPid, systemMetadata, data);
351 359 
    }
352 360
361
362 
    private void removeObsoletesChain(String obsoleteId, List<String> obsoleteIdChain) throws SolrServerException, IOException {
363 
        if(obsoleteId != null && !obsoleteId.trim().equals("")) {
364 
            if(obsoleteIdChain == null || obsoleteIdChain.isEmpty()) {
365 
                throw new SolrServerException("SolrIndex.removeObsoletesChain - The obsoletes chain can't be null or empty since the system metadata already has the obsoletes element.");
366 
            }
367 
            if(!obsoleteIdChain.contains(obsoleteId)) {
368 
                throw new SolrServerException("SolrIndex.removeObsoletesChain - The obsoletes elment in the system metadata is not in the obsoleteId chain");
369 
            }
370 
            remove(obsoleteIdChain);
371 
        } else {
372 
            throw new SolrServerException("SolrIndex.removeObsoletesChain - The obsolete id should be null.");
373 
        }
374 
    }
375
353 376 
    /**
354 377 
     * Remove all the indexes associated with the pids in the list.
355 378 
     * @param pidList
356 379 
     * @throws IOException
357 380 
     * @throws SolrServerException
358 381 
     */
359 
    public void remove(List<String> pidList) throws IOException, SolrServerException {
382 
    private void remove(List<String> pidList) throws IOException, SolrServerException {
360 383 
        if(pidList != null) {
361 384 
            for(String id : pidList) {
362 385 
                remove(id);
......
371 394 
     * @throws SolrServerException
372 395 
     */
373 396 
    public void remove(String pid) throws IOException, SolrServerException {
374 
        solrServer.deleteById(pid);
375 
        solrServer.commit();
397 
        if(pid != null && !pid.trim().equals("")) {
398 
            solrServer.deleteById(pid);
399 
            solrServer.commit();
400 
        }
376 401
402
377 403 
    }
378 404 

  		




  379
  405
  
    
    /**


  		












  

    
      metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SystemMetadataEventListener.java
    
  





  208
  208
  
    
    }


  		




  209
  209
  
    

  		




  210
  210
  
    
	public void entryAdded(EntryEvent<Identifier, SystemMetadata> entryEvent) {


  		




  
  211
  
    
	    System.out.println("===================================calling entryAdded method ");


  		




  211
  212
  
    
		// use the same implementation for insert/update for now


  		




  212
  213
  
    
		this.entryUpdated(entryEvent);


  		




  213
  214
  
    

  		




  ......




  232
  233
  
    
	}


  		




  233
  234
  
    

  		




  234
  235
  
    
	public void entryUpdated(EntryEvent<Identifier, SystemMetadata> entryEvent) {


  		




  
  236
  
    
	    System.out.println("===================================calling entryUpdated method ");


  		




  235
  237
  
    
		// add to the index


  		




  236
  238
  
    
		Identifier pid = entryEvent.getKey();


  		




  237
  239
  
    
		SystemMetadata systemMetadata = entryEvent.getValue();


  		




  238
  
  
    
		


  		




  
  240
  
    
		Identifier obsoletes = systemMetadata.getObsoletes();


  		




  
  241
  
    
		List<String> obsoletesChain = null;


  		




  
  242
  
    
		if(obsoletes != null) {


  		




  
  243
  
    
		    obsoletesChain= getObsoletes(pid.getValue());


  		




  
  244
  
    
		}


  		




  239
  245
  
    
		String objectPath = objectPathMap.get(pid);


  		




  240
  246
  
    
		InputStream data = null;


  		




  241
  247
  
    
		try {


  		




  242
  248
  
    
			data = new FileInputStream(objectPath);


  		




  243
  
  
    
			solrIndex.insert(pid.getValue(), systemMetadata, data);


  		




  
  249
  
    
			solrIndex.update(pid.getValue(), obsoletesChain, systemMetadata, data);


  		




  244
  250
  
    
		} catch (Exception e) {


  		




  245
  251
  
    
			// TODO: need to track errors, retry later


  		




  246
  252
  
    
			log.error(e.getMessage(), e);


  		












  

    
      metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/IndexGenerator.java
    
  





  257
  257
  
    
                if(sysmeta != null) {


  		




  258
  258
  
    
                        InputStream data = systemMetadataListener.getDataObject(id);


  		




  259
  259
  
    
                        Identifier obsolete = sysmeta.getObsoletes();


  		




  
  260
  
    
                        List<String> obsoleteChain = null;


  		




  260
  261
  
    
                        if(obsolete != null) {


  		




  261
  
  
    
                            List<String> obsoleteChain = systemMetadataListener.getObsoletes(id);


  		




  262
  
  
    
                            solrIndex.update(id, obsoleteChain, sysmeta, data);


  		




  263
  
  
    
                        } else {


  		




  264
  
  
    
                            solrIndex.insert(id, sysmeta, data);


  		




  265
  
  
    
                        }


  		




  
  262
  
    
                            obsoleteChain = systemMetadataListener.getObsoletes(id);


  		




  
  263
  
    
                        } 


  		




  
  264
  
    
                        solrIndex.update(id, obsoleteChain, sysmeta, data);


  		




  266
  265
  
    
                } else {


  		




  267
  266
  
    
                    throw new Exception("IndexGenerator.generate - there is no found SystemMetadata associated with the id "+id);


  		




  268
  267
  
    
                }


  		












Also available in:  Unified diff