Revision 8343
Added by Jing Tao about 11 years ago
| metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/SolrIndexIT.java | ||
|---|---|---|
| 23 | 23 |
import org.apache.solr.response.SolrQueryResponse; |
| 24 | 24 |
import org.apache.solr.response.XMLResponseWriter; |
| 25 | 25 |
import org.apache.solr.servlet.SolrRequestParsers; |
| 26 |
import org.dataone.service.types.v1.Identifier; |
|
| 26 | 27 |
import org.dataone.service.types.v1.SystemMetadata; |
| 27 | 28 |
import org.dataone.service.util.TypeMarshaller; |
| 28 | 29 |
import org.junit.Before; |
| ... | ... | |
| 70 | 71 |
SystemMetadata systemMetadata = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, SYSTEMMETAFILEPATH); |
| 71 | 72 |
InputStream emlInputStream = new FileInputStream(new File(EMLFILEPATH)); |
| 72 | 73 |
//List<String> chain = null; |
| 73 |
solrIndex.update(id, systemMetadata, emlInputStream); |
|
| 74 |
Identifier pid = new Identifier(); |
|
| 75 |
pid.setValue(id); |
|
| 76 |
solrIndex.update(pid, systemMetadata, emlInputStream); |
|
| 74 | 77 |
String result = doQuery(solrIndex.getSolrServer()); |
| 75 | 78 |
List<String> ids = solrIndex.getSolrIds(); |
| 76 | 79 |
//assertTrue(ids.size() == 1); |
| ... | ... | |
| 95 | 98 |
InputStream emlInputStream = new FileInputStream(new File(EMLUPDATEFILEPATH)); |
| 96 | 99 |
/*obsoletes.add(id); |
| 97 | 100 |
obsoletes.add("tao");*/
|
| 98 |
solrIndex.update(newId, systemMetadata, emlInputStream); |
|
| 101 |
Identifier pid = new Identifier(); |
|
| 102 |
pid.setValue(newId); |
|
| 103 |
solrIndex.update(pid, systemMetadata, emlInputStream); |
|
| 99 | 104 |
String result = doQuery(solrIndex.getSolrServer()); |
| 105 |
assertTrue(result.contains("version1"));
|
|
| 100 | 106 |
assertTrue(result.contains("version2"));
|
| 101 | 107 |
} |
| 102 | 108 |
|
| ... | ... | |
| 113 | 119 |
/*ArrayList<String> obsoletes = new ArrayList<String>(); |
| 114 | 120 |
obsoletes.add(id); |
| 115 | 121 |
obsoletes.add("tao");*/
|
| 116 |
solrIndex.update(newId, systemMetadata, emlInputStream); |
|
| 122 |
Identifier pid = new Identifier(); |
|
| 123 |
pid.setValue(newId); |
|
| 124 |
solrIndex.update(pid, systemMetadata, emlInputStream); |
|
| 117 | 125 |
String result = doQuery(solrIndex.getSolrServer()); |
| 118 | 126 |
assertTrue(result.contains("version1"));
|
| 119 |
assertTrue(result.contains("version2"));
|
|
| 127 |
assertTrue(!result.contains("version2"));
|
|
| 120 | 128 |
} |
| 121 | 129 |
|
| 122 | 130 |
|
| metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java | ||
|---|---|---|
| 19 | 19 |
package edu.ucsb.nceas.metacat.index; |
| 20 | 20 |
|
| 21 | 21 |
import java.io.ByteArrayInputStream; |
| 22 |
import java.io.FileInputStream; |
|
| 22 | 23 |
import java.io.FileNotFoundException; |
| 23 | 24 |
import java.io.IOException; |
| 24 | 25 |
import java.io.InputStream; |
| ... | ... | |
| 55 | 56 |
import org.dataone.cn.indexer.parser.SolrField; |
| 56 | 57 |
import org.dataone.cn.indexer.resourcemap.ResourceEntry; |
| 57 | 58 |
import org.dataone.cn.indexer.resourcemap.ResourceMap; |
| 59 |
import org.dataone.cn.indexer.resourcemap.ResourceMapFactory; |
|
| 58 | 60 |
import org.dataone.cn.indexer.solrhttp.SolrDoc; |
| 59 | 61 |
import org.dataone.cn.indexer.solrhttp.SolrElementField; |
| 60 | 62 |
import org.dataone.service.exceptions.NotFound; |
| ... | ... | |
| 320 | 322 |
* @param data |
| 321 | 323 |
* @throws SolrServerException |
| 322 | 324 |
*/ |
| 323 |
private void checkParams(String pid, SystemMetadata systemMetadata, InputStream data) throws SolrServerException {
|
|
| 324 |
if(pid == null || pid.trim().equals("")) {
|
|
| 325 |
private void checkParams(Identifier pid, SystemMetadata systemMetadata, InputStream data) throws SolrServerException {
|
|
| 326 |
if(pid == null || pid.getValue() == null || pid.getValue().trim().equals("")) {
|
|
| 325 | 327 |
throw new SolrServerException("The identifier of the indexed document should not be null or blank.");
|
| 326 | 328 |
} |
| 327 | 329 |
if(systemMetadata == null) {
|
| 328 |
throw new SolrServerException("The system metadata of the indexed document should not be null.");
|
|
| 330 |
throw new SolrServerException("The system metadata of the indexed document "+pid.getValue()+ " should not be null.");
|
|
| 329 | 331 |
} |
| 330 | 332 |
if(data == null) {
|
| 331 |
throw new SolrServerException("The indexed document itself should not be null.");
|
|
| 333 |
throw new SolrServerException("The indexed document itself for pid "+pid.getValue()+" should not be null.");
|
|
| 332 | 334 |
} |
| 333 | 335 |
} |
| 334 | 336 |
|
| ... | ... | |
| 344 | 346 |
* @throws NotFound |
| 345 | 347 |
* @throws NotImplemented |
| 346 | 348 |
*/ |
| 347 |
private synchronized void insert(String pid, SystemMetadata systemMetadata, InputStream data)
|
|
| 349 |
private synchronized void insert(Identifier pid, SystemMetadata systemMetadata, InputStream data)
|
|
| 348 | 350 |
throws IOException, SAXException, ParserConfigurationException, |
| 349 | 351 |
XPathExpressionException, SolrServerException, JiBXException, EncoderException, NotImplemented, NotFound, UnsupportedType {
|
| 350 | 352 |
checkParams(pid, systemMetadata, data); |
| 351 |
Map<String, SolrDoc> docs = process(pid, systemMetadata, data); |
|
| 353 |
Map<String, SolrDoc> docs = process(pid.getValue(), systemMetadata, data);
|
|
| 352 | 354 |
|
| 353 | 355 |
//transform the Map to the SolrInputDocument which can be used by the solr server |
| 354 | 356 |
if(docs != null) {
|
| ... | ... | |
| 426 | 428 |
|
| 427 | 429 |
/** |
| 428 | 430 |
* Update the solr index. This method handles the three scenarios: |
| 429 |
* 1. Update an existing doc - if the the system metadata shows the value of the archive is false and it has an obsoletes,
|
|
| 431 |
* 1. Remove an existing doc - if the the system metadata shows the value of the archive is true,
|
|
| 430 | 432 |
* remove the index for the previous version(s) and generate new index for the doc. |
| 431 |
* 2. Add a new doc - if the system metadata shows the value of the archive is false and it hasn't an obsoletes, generate the
|
|
| 433 |
* 2. Add a new doc - if the system metadata shows the value of the archive is false, generate the |
|
| 432 | 434 |
* index for the doc. |
| 433 |
* @param pid the id of the document |
|
| 434 |
* @param obsoleteIds the chain of the obsoletes by this id |
|
| 435 |
* @param systemMetadata the system metadata associated with the data object |
|
| 436 |
* @param data the data object itself |
|
| 437 |
* @throws SolrServerException |
|
| 438 |
* @throws JiBXException |
|
| 439 |
* @throws EncoderException |
|
| 440 |
* @throws UnsupportedType |
|
| 441 |
* @throws NotFound |
|
| 442 |
* @throws NotImplemented |
|
| 443 |
* @throws ServiceFailure |
|
| 444 |
* @throws OREParserException |
|
| 445 | 435 |
*/ |
| 446 |
public void update(String pid, SystemMetadata systemMetadata, InputStream data) |
|
| 447 |
throws IOException, SAXException, ParserConfigurationException, |
|
| 448 |
XPathExpressionException, SolrServerException, JiBXException, EncoderException, NotImplemented, NotFound, UnsupportedType, ServiceFailure, OREParserException {
|
|
| 449 |
checkParams(pid, systemMetadata, data); |
|
| 450 |
//generate index for either add or update. |
|
| 451 |
insert(pid, systemMetadata, data); |
|
| 452 |
log.info("============================= update index for the identifier "+pid);
|
|
| 453 |
|
|
| 436 |
public void update(Identifier pid, SystemMetadata systemMetadata) {
|
|
| 437 |
String objectPath = null; |
|
| 438 |
InputStream data = null; |
|
| 439 |
try {
|
|
| 440 |
objectPath = DistributedMapsFactory.getObjectPathMap().get(pid); |
|
| 441 |
data = new FileInputStream(objectPath); |
|
| 442 |
update(pid, systemMetadata, data); |
|
| 443 |
EventlogFactory.createIndexEventLog().remove(pid); |
|
| 444 |
} catch (Exception e) {
|
|
| 445 |
String error = "SolrIndex.update - could not update the solr index since " + e.getMessage(); |
|
| 446 |
writeEventLog(systemMetadata, pid, error); |
|
| 447 |
log.error(error, e); |
|
| 448 |
} |
|
| 454 | 449 |
} |
| 455 | 450 |
|
| 456 | 451 |
|
| 452 |
/** |
|
| 453 |
* Update the solr index. This method handles the three scenarios: |
|
| 454 |
* 1. Remove an existing doc - if the the system metadata shows the value of the archive is true, |
|
| 455 |
* remove the index for the previous version(s) and generate new index for the doc. |
|
| 456 |
* 2. Add a new doc - if the system metadata shows the value of the archive is false, generate the |
|
| 457 |
* index for the doc. |
|
| 458 |
* @param pid |
|
| 459 |
* @param systemMetadata |
|
| 460 |
* @param data |
|
| 461 |
* @throws SolrServerException |
|
| 462 |
* @throws ServiceFailure |
|
| 463 |
* @throws XPathExpressionException |
|
| 464 |
* @throws NotImplemented |
|
| 465 |
* @throws NotFound |
|
| 466 |
* @throws UnsupportedType |
|
| 467 |
* @throws IOException |
|
| 468 |
* @throws SAXException |
|
| 469 |
* @throws ParserConfigurationException |
|
| 470 |
* @throws OREParserException |
|
| 471 |
* @throws JiBXException |
|
| 472 |
* @throws EncoderException |
|
| 473 |
*/ |
|
| 474 |
void update(Identifier pid, SystemMetadata systemMetadata, InputStream data) throws SolrServerException, |
|
| 475 |
ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, |
|
| 476 |
IOException, SAXException, ParserConfigurationException, OREParserException, JiBXException, EncoderException {
|
|
| 477 |
checkParams(pid, systemMetadata, data); |
|
| 478 |
boolean isArchive = systemMetadata.getArchived(); |
|
| 479 |
if(isArchive ) {
|
|
| 480 |
//delete the index for the archived objects |
|
| 481 |
remove(pid.getValue(), systemMetadata); |
|
| 482 |
log.info("SolrIndex.update============================= archive the idex for the identifier "+pid);
|
|
| 483 |
} else {
|
|
| 484 |
//generate index for either add or update. |
|
| 485 |
insert(pid, systemMetadata, data); |
|
| 486 |
log.info("SolrIndex.update============================= insert index for the identifier "+pid);
|
|
| 487 |
} |
|
| 488 |
} |
|
| 457 | 489 |
|
| 458 | 490 |
|
| 459 | 491 |
|
| 460 | 492 |
/* |
| 461 | 493 |
* Is the pid a resource map |
| 462 | 494 |
*/ |
| 463 |
private boolean isDataPackage(String pid) throws FileNotFoundException, ServiceFailure {
|
|
| 495 |
private boolean isDataPackage(String pid, SystemMetadata sysmeta) throws FileNotFoundException, ServiceFailure {
|
|
| 464 | 496 |
boolean isDataPackage = false; |
| 465 |
SystemMetadata sysmeta = DistributedMapsFactory.getSystemMetadata(pid); |
|
| 497 |
//SystemMetadata sysmeta = DistributedMapsFactory.getSystemMetadata(pid);
|
|
| 466 | 498 |
if(sysmeta != null) {
|
| 467 | 499 |
isDataPackage = IndexGenerator.isResourceMap(sysmeta.getFormatId()); |
| 468 | 500 |
} |
| ... | ... | |
| 479 | 511 |
return false; |
| 480 | 512 |
} |
| 481 | 513 |
} |
| 514 |
/** |
|
| 515 |
* Remove the indexed associated with specified pid. |
|
| 516 |
* @param pid the pid which the indexes are associated with |
|
| 517 |
* @throws IOException |
|
| 518 |
* @throws SolrServerException |
|
| 519 |
* @throws ParserConfigurationException |
|
| 520 |
* @throws SAXException |
|
| 521 |
* @throws UnsupportedType |
|
| 522 |
* @throws NotFound |
|
| 523 |
* @throws NotImplemented |
|
| 524 |
* @throws XPathExpressionException |
|
| 525 |
* @throws ServiceFailure |
|
| 526 |
* @throws OREParserException |
|
| 527 |
*/ |
|
| 528 |
private void remove(String pid, SystemMetadata sysmeta) throws IOException, SolrServerException, ServiceFailure, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SAXException, ParserConfigurationException, OREParserException {
|
|
| 529 |
if (isDataPackage(pid, sysmeta)) {
|
|
| 530 |
removeDataPackage(pid); |
|
| 531 |
} else if (isPartOfDataPackage(pid)) {
|
|
| 532 |
removeFromDataPackage(pid); |
|
| 533 |
} else {
|
|
| 534 |
removeFromIndex(pid); |
|
| 535 |
} |
|
| 536 |
} |
|
| 537 |
|
|
| 538 |
/* |
|
| 539 |
* Remove a resource map pid |
|
| 540 |
*/ |
|
| 541 |
private void removeDataPackage(String pid) throws ServiceFailure, SAXException, XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException, OREParserException {
|
|
| 542 |
Document resourceMapDoc = generateXmlDocument(DistributedMapsFactory.getDataObject(pid)); |
|
| 543 |
//ResourceMap resourceMap = new ResourceMap(resourceMapDoc); |
|
| 544 |
ResourceMap resourceMap = ResourceMapFactory.buildResourceMap(resourceMapDoc); |
|
| 545 |
List<String> documentIds = resourceMap.getAllDocumentIDs(); |
|
| 546 |
List<SolrDoc> indexDocuments =ResourceMapSubprocessor.getSolrDocs(documentIds); |
|
| 547 |
removeFromIndex(pid); |
|
| 548 |
//List<SolrDoc> docsToUpdate = new ArrayList<SolrDoc>(); |
|
| 549 |
// for each document in data package: |
|
| 550 |
for (SolrDoc indexDoc : indexDocuments) {
|
|
| 482 | 551 |
|
| 552 |
if (indexDoc.getIdentifier().equals(pid)) {
|
|
| 553 |
continue; // skipping the resource map, no need update |
|
| 554 |
// it. |
|
| 555 |
// will |
|
| 556 |
// be removed. |
|
| 557 |
} |
|
| 558 |
|
|
| 559 |
// Remove resourceMap reference |
|
| 560 |
indexDoc.removeFieldsWithValue(SolrElementField.FIELD_RESOURCEMAP, |
|
| 561 |
resourceMap.getIdentifier()); |
|
| 562 |
|
|
| 563 |
// // Remove documents/documentedby values for this resource |
|
| 564 |
// map |
|
| 565 |
for (ResourceEntry entry : resourceMap.getMappedReferences()) {
|
|
| 566 |
if (indexDoc.getIdentifier().equals(entry.getIdentifier())) {
|
|
| 567 |
for (String documentedBy : entry.getDocumentedBy()) {
|
|
| 568 |
// Using removeOneFieldWithValue in-case same |
|
| 569 |
// documents |
|
| 570 |
// are in more than one data package. just |
|
| 571 |
// remove |
|
| 572 |
// one |
|
| 573 |
// instance of data package info. |
|
| 574 |
indexDoc.removeOneFieldWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY, |
|
| 575 |
documentedBy); |
|
| 576 |
} |
|
| 577 |
for (String documents : entry.getDocuments()) {
|
|
| 578 |
indexDoc.removeOneFieldWithValue(SolrElementField.FIELD_DOCUMENTS, |
|
| 579 |
documents); |
|
| 580 |
} |
|
| 581 |
break; |
|
| 582 |
} |
|
| 583 |
} |
|
| 584 |
removeFromIndex(indexDoc.getIdentifier()); |
|
| 585 |
insertToIndex(indexDoc); |
|
| 586 |
//docsToUpdate.add(indexDoc); |
|
| 587 |
} |
|
| 588 |
//SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate); |
|
| 589 |
//httpService.sendUpdate(solrIndexUri, addCommand); |
|
| 590 |
} |
|
| 591 |
|
|
| 592 |
private void removeFromDataPackage(String pid) throws XPathExpressionException, NotImplemented, NotFound, UnsupportedType, SolrServerException, IOException, ParserConfigurationException, SAXException {
|
|
| 593 |
SolrDoc indexedDoc = ResourceMapSubprocessor.getSolrDoc(pid); |
|
| 594 |
removeFromIndex(pid); |
|
| 595 |
List<SolrDoc> docsToUpdate = new ArrayList<SolrDoc>(); |
|
| 596 |
|
|
| 597 |
List<String> documents = indexedDoc.getAllFieldValues(SolrElementField.FIELD_DOCUMENTS); |
|
| 598 |
for (String documentsValue : documents) {
|
|
| 599 |
SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentsValue); |
|
| 600 |
solrDoc.removeFieldsWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY, pid); |
|
| 601 |
removeFromIndex(documentsValue); |
|
| 602 |
insertToIndex(solrDoc); |
|
| 603 |
} |
|
| 604 |
|
|
| 605 |
List<String> documentedBy = indexedDoc |
|
| 606 |
.getAllFieldValues(SolrElementField.FIELD_ISDOCUMENTEDBY); |
|
| 607 |
for (String documentedByValue : documentedBy) {
|
|
| 608 |
SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentedByValue); |
|
| 609 |
solrDoc.removeFieldsWithValue(SolrElementField.FIELD_DOCUMENTS, documentedByValue); |
|
| 610 |
//docsToUpdate.add(solrDoc); |
|
| 611 |
removeFromIndex(documentedByValue); |
|
| 612 |
insertToIndex(solrDoc); |
|
| 613 |
} |
|
| 614 |
|
|
| 615 |
//SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate); |
|
| 616 |
//httpService.sendUpdate(solrIndexUri, addCommand); |
|
| 617 |
} |
|
| 618 |
|
|
| 619 |
/* |
|
| 620 |
* Remove a pid from the solr index |
|
| 621 |
*/ |
|
| 622 |
private void removeFromIndex(String pid) throws SolrServerException, IOException {
|
|
| 623 |
if(pid != null && !pid.trim().equals("")) {
|
|
| 624 |
/*IndexEvent event = new IndexEvent(); |
|
| 625 |
event.setDate(Calendar.getInstance().getTime()); |
|
| 626 |
Identifier identifier = new Identifier(); |
|
| 627 |
identifier.setValue(pid); |
|
| 628 |
event.setIdentifier(identifier);*/ |
|
| 629 |
try {
|
|
| 630 |
solrServer.deleteById(pid); |
|
| 631 |
solrServer.commit(); |
|
| 632 |
/*event.setType(IndexEvent.SUCCESSDELETE); |
|
| 633 |
event.setDescription("Successfully remove the solr index for the id "+identifier.getValue());
|
|
| 634 |
try {
|
|
| 635 |
EventlogFactory.createIndexEventLog().write(event); |
|
| 636 |
} catch (Exception e) {
|
|
| 637 |
log.error("SolrIndex.removeFromIndex - IndexEventLog can't log the index deleting event :"+e.getMessage());
|
|
| 638 |
}*/ |
|
| 639 |
} catch (SolrServerException e) {
|
|
| 640 |
/*event.setAction(Event.DELETE); |
|
| 641 |
event.setDescription("Failurely remove the solr index for the id "+identifier.getValue()+" since "+e.getMessage());
|
|
| 642 |
try {
|
|
| 643 |
EventlogFactory.createIndexEventLog().write(event); |
|
| 644 |
} catch (Exception ee) {
|
|
| 645 |
log.error("SolrIndex.removeFromIndex - IndexEventLog can't log the index deleting event :"+ee.getMessage());
|
|
| 646 |
}*/ |
|
| 647 |
throw e; |
|
| 648 |
|
|
| 649 |
} catch (IOException e) {
|
|
| 650 |
/*event.setAction(Event.DELETE); |
|
| 651 |
event.setDescription("Failurely remove the solr index for the id "+identifier.getValue()+" since "+e.getMessage());
|
|
| 652 |
try {
|
|
| 653 |
EventlogFactory.createIndexEventLog().write(event); |
|
| 654 |
} catch (Exception ee) {
|
|
| 655 |
log.error("SolrIndex.removeFromIndex - IndexEventLog can't log the index deleting event :"+ee.getMessage());
|
|
| 656 |
}*/ |
|
| 657 |
throw e; |
|
| 658 |
} |
|
| 659 |
|
|
| 660 |
} |
|
| 661 |
} |
|
| 662 |
|
|
| 483 | 663 |
/** |
| 484 | 664 |
* Get the solrServer |
| 485 | 665 |
* @return |
| ... | ... | |
| 517 | 697 |
} |
| 518 | 698 |
return list; |
| 519 | 699 |
} |
| 700 |
|
|
| 701 |
private void writeEventLog(SystemMetadata systemMetadata, Identifier pid, String error) {
|
|
| 702 |
IndexEvent event = new IndexEvent(); |
|
| 703 |
event.setIdentifier(pid); |
|
| 704 |
event.setDate(Calendar.getInstance().getTime()); |
|
| 705 |
String action = null; |
|
| 706 |
if (systemMetadata == null ) {
|
|
| 707 |
action = Event.CREATE.xmlValue(); |
|
| 708 |
event.setAction(Event.CREATE); |
|
| 709 |
} |
|
| 710 |
else if(systemMetadata.getArchived()) {
|
|
| 711 |
action = Event.DELETE.xmlValue(); |
|
| 712 |
event.setAction(Event.DELETE); |
|
| 713 |
} else {
|
|
| 714 |
action = Event.CREATE.xmlValue(); |
|
| 715 |
event.setAction(Event.CREATE); |
|
| 716 |
} |
|
| 717 |
event.setDescription("Failed to "+action+"the solr index for the id "+pid.getValue()+" since "+error);
|
|
| 718 |
try {
|
|
| 719 |
EventlogFactory.createIndexEventLog().write(event); |
|
| 720 |
} catch (Exception ee) {
|
|
| 721 |
log.error("SolrIndex.insertToIndex - IndexEventLog can't log the index inserting event :"+ee.getMessage());
|
|
| 722 |
} |
|
| 723 |
} |
|
| 520 | 724 |
} |
| metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SystemMetadataEventListener.java | ||
|---|---|---|
| 149 | 149 |
//System.out.println("===================================update the document "+pid.getValue());
|
| 150 | 150 |
log.info("===================================adding the document "+pid.getValue());
|
| 151 | 151 |
SystemMetadata systemMetadata = entryEvent.getItem(); |
| 152 |
if(systemMetadata == null) {
|
|
| 152 |
|
|
| 153 |
/*if(systemMetadata == null) {
|
|
| 153 | 154 |
writeEventLog(systemMetadata, pid, "SystemMetadataEventListener.itemAdded -could not get the SystemMetadata"); |
| 154 | 155 |
return; |
| 155 |
} |
|
| 156 |
}*/
|
|
| 156 | 157 |
|
| 157 | 158 |
// make sure we remove this object so that it can be re-added in the future |
| 158 |
if (source != null) {
|
|
| 159 |
if (source != null && systemMetadata != null) {
|
|
| 159 | 160 |
source.remove(systemMetadata); |
| 160 | 161 |
} |
| 162 |
solrIndex.update(pid, systemMetadata); |
|
| 161 | 163 |
|
| 162 | 164 |
//Identifier obsoletes = systemMetadata.getObsoletes(); |
| 163 | 165 |
//List<String> obsoletesChain = null; |
| ... | ... | |
| 171 | 173 |
return; |
| 172 | 174 |
} |
| 173 | 175 |
}*/ |
| 174 |
String objectPath = null; |
|
| 175 |
try {
|
|
| 176 |
objectPath = DistributedMapsFactory.getObjectPathMap().get(pid); |
|
| 177 |
} catch (Exception e) {
|
|
| 178 |
String error = "SystemMetadataEventListener.itemAdded - could not look up object path" + e.getMessage(); |
|
| 179 |
writeEventLog(systemMetadata, pid, error); |
|
| 180 |
log.error(error, e); |
|
| 181 |
return; |
|
| 182 |
} |
|
| 183 |
if(objectPath != null) {
|
|
| 176 |
|
|
| 177 |
/*if(objectPath != null) {
|
|
| 184 | 178 |
InputStream data = null; |
| 185 | 179 |
try {
|
| 186 | 180 |
data = new FileInputStream(objectPath); |
| 187 | 181 |
solrIndex.update(pid.getValue(), systemMetadata, data); |
| 188 |
EventlogFactory.createIndexEventLog().remove(pid);
|
|
| 182 |
EventlogFactory.createIndexEventLog().remove(pid);
|
|
| 189 | 183 |
} catch (Exception e) {
|
| 190 | 184 |
String error = "SystemMetadataEventListener.itemAdded - could not comit the index into the solr server since " + e.getMessage(); |
| 191 | 185 |
writeEventLog(systemMetadata, pid, error); |
| 192 | 186 |
log.error(error, e); |
| 193 | 187 |
} |
| 194 |
} |
|
| 188 |
}*/
|
|
| 195 | 189 |
|
| 196 | 190 |
} |
| 197 | 191 |
|
| 198 |
private void writeEventLog(SystemMetadata systemMetadata, Identifier pid, String error) {
|
|
| 199 |
IndexEvent event = new IndexEvent(); |
|
| 200 |
event.setIdentifier(pid); |
|
| 201 |
event.setDate(Calendar.getInstance().getTime()); |
|
| 202 |
String action = null; |
|
| 203 |
if (systemMetadata == null ) {
|
|
| 204 |
action = Event.CREATE.xmlValue(); |
|
| 205 |
event.setAction(Event.CREATE); |
|
| 206 |
} |
|
| 207 |
else if(systemMetadata.getArchived() || systemMetadata.getObsoletedBy() != null) {
|
|
| 208 |
action = Event.UPDATE.xmlValue(); |
|
| 209 |
event.setAction(Event.UPDATE); |
|
| 210 |
} else {
|
|
| 211 |
action = Event.CREATE.xmlValue(); |
|
| 212 |
event.setAction(Event.CREATE); |
|
| 213 |
} |
|
| 214 |
event.setDescription("Failed to "+action+"the solr index for the id "+pid.getValue()+" since "+error);
|
|
| 215 |
try {
|
|
| 216 |
EventlogFactory.createIndexEventLog().write(event); |
|
| 217 |
} catch (Exception ee) {
|
|
| 218 |
log.error("SolrIndex.insertToIndex - IndexEventLog can't log the index inserting event :"+ee.getMessage());
|
|
| 219 |
} |
|
| 220 |
} |
|
| 192 |
|
|
| 221 | 193 |
|
| 222 | 194 |
} |
| metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/IndexGenerator.java | ||
|---|---|---|
| 108 | 108 |
|
| 109 | 109 |
} |
| 110 | 110 |
|
| 111 |
/** |
|
| 112 |
* Build the index for all documents in Metacat without overwriting. |
|
| 113 |
* @throws SolrServerException |
|
| 114 |
* @throws ServiceFailure |
|
| 115 |
* @throws NotImplemented |
|
| 116 |
* @throws NotAuthorized |
|
| 117 |
* @throws InvalidToken |
|
| 118 |
* @throws InvalidRequest |
|
| 119 |
* @throws IndexEventLogException |
|
| 120 |
* @throws IllegalAccessException |
|
| 121 |
* @throws InstantiationException |
|
| 122 |
* @throws ClassNotFoundException |
|
| 123 |
*/ |
|
| 124 |
/*public void indexAll() throws InvalidRequest, InvalidToken, NotAuthorized, |
|
| 125 |
NotImplemented, ServiceFailure, SolrServerException, FileNotFoundException, ClassNotFoundException, InstantiationException, IllegalAccessException, IndexEventLogException {
|
|
| 126 |
boolean force = false; |
|
| 127 |
indexAll(force); |
|
| 128 |
}*/ |
|
| 111 |
|
|
| 129 | 112 |
|
| 130 | 113 |
/** |
| 131 | 114 |
* Build the index for all documents. |
| ... | ... | |
| 392 | 375 |
}*/ |
| 393 | 376 |
|
| 394 | 377 |
public void run() {
|
| 395 |
/*IndexEvent event = new IndexEvent(); |
|
| 396 |
event.setDate(Calendar.getInstance().getTime()); |
|
| 397 |
event.setType(IndexEvent.STARTTIMEDINDEX); |
|
| 398 |
event.setDescription("Start the timed index job");
|
|
| 378 |
|
|
| 399 | 379 |
try {
|
| 400 |
EventlogFactory.createIndexEventLog().write(event); |
|
| 401 |
} catch (Exception e) {
|
|
| 402 |
log.error("IndexGenerator.run - IndexEventLog can't log the timed indexing start event :"+e.getMessage());
|
|
| 403 |
}*/ |
|
| 404 |
try {
|
|
| 405 | 380 |
Date since = EventlogFactory.createIndexEventLog().getLastProcessDate(); |
| 406 | 381 |
index(since); |
| 407 | 382 |
} catch (InvalidRequest e) {
|
| ... | ... | |
| 429 | 404 |
log.error("IndexGenerator.run - Metadata-Index couldn't generate indexes for those documents which haven't been indexed : "+e.getMessage());
|
| 430 | 405 |
} catch (FileNotFoundException e) {
|
| 431 | 406 |
log.error("IndexGenerator.run - Metadata-Index couldn't generate indexes for those documents which haven't been indexed : "+e.getMessage());
|
| 432 |
} |
|
| 433 |
/*event.setDate(Calendar.getInstance().getTime()); |
|
| 434 |
event.setType(IndexEvent.FINISHTIMEDINDEX); |
|
| 435 |
event.setDescription("Finish the timed index job");
|
|
| 436 |
try {
|
|
| 437 |
EventlogFactory.createIndexEventLog().write(event); |
|
| 438 |
} catch (Exception e) {
|
|
| 439 |
log.error("IndexGenerator.run - IndexEventLog can't log the timed indexing finish event :"+e.getMessage());
|
|
| 440 |
}*/ catch (ClassNotFoundException e) {
|
|
| 407 |
} catch (ClassNotFoundException e) {
|
|
| 441 | 408 |
// TODO Auto-generated catch block |
| 442 | 409 |
log.error("IndexGenerator.run - Metadata-Index couldn't generate indexes for those documents which haven't been indexed : "+e.getMessage());
|
| 443 | 410 |
} catch (InstantiationException e) {
|
| ... | ... | |
| 620 | 587 |
* Generate index for the id. |
| 621 | 588 |
*/ |
| 622 | 589 |
private void generateIndex(String id) {
|
| 623 |
if(id != null) {
|
|
| 624 |
SystemMetadata sysmeta = getSystemMetadata(id); |
|
| 625 |
//only update none-archived id. |
|
| 626 |
//if(sysmeta != null && !sysmeta.getArchived() && sysmeta.getObsoletedBy() == null) {
|
|
| 627 |
try {
|
|
| 628 |
if(sysmeta != null) {
|
|
| 629 |
InputStream data = getDataObject(id); |
|
| 630 |
/*Identifier obsolete = sysmeta.getObsoletes(); |
|
| 631 |
List<String> obsoleteChain = null; |
|
| 632 |
if(obsolete != null) {
|
|
| 633 |
obsoleteChain = getObsoletes(id); |
|
| 634 |
}*/ |
|
| 635 |
solrIndex.update(id, sysmeta, data); |
|
| 636 |
try {
|
|
| 637 |
Identifier identifier = new Identifier(); |
|
| 638 |
identifier.setValue(id); |
|
| 639 |
EventlogFactory.createIndexEventLog().remove(identifier); |
|
| 640 |
} catch (Exception ee) {
|
|
| 641 |
log.error("IndexGenerator.index - can't remove the id "+id +" from the index_event table since - "+ee.getMessage());
|
|
| 642 |
} |
|
| 643 |
} else {
|
|
| 644 |
throw new Exception("IndexGenerator.generate - there is no found SystemMetadata associated with the id "+id);
|
|
| 645 |
} |
|
| 646 |
} catch (Exception e) {
|
|
| 647 |
IndexEvent event = new IndexEvent(); |
|
| 648 |
Identifier pid = new Identifier(); |
|
| 649 |
pid.setValue(id); |
|
| 650 |
event.setIdentifier(pid); |
|
| 651 |
event.setDate(Calendar.getInstance().getTime()); |
|
| 652 |
event.setAction(Event.CREATE); |
|
| 653 |
String error = "IndexGenerator.index - Metacat Index couldn't generate the index for the id - "+id+" because "+e.getMessage(); |
|
| 654 |
event.setDescription(error); |
|
| 655 |
try {
|
|
| 656 |
EventlogFactory.createIndexEventLog().write(event); |
|
| 657 |
} catch (Exception ee) {
|
|
| 658 |
log.error("SolrIndex.insertToIndex - IndexEventLog can't log the index inserting event :"+ee.getMessage());
|
|
| 659 |
} |
|
| 660 |
log.error(error); |
|
| 661 |
} |
|
| 662 |
|
|
| 663 |
|
|
| 664 |
} |
|
| 590 |
//if id is null and sysmeta will be null. If sysmeta is null, it will be caught in solrIndex.update |
|
| 591 |
SystemMetadata sysmeta = getSystemMetadata(id); |
|
| 592 |
Identifier pid = new Identifier(); |
|
| 593 |
pid.setValue(id); |
|
| 594 |
solrIndex.update(pid, sysmeta); |
|
| 595 |
|
|
| 665 | 596 |
} |
| 666 | 597 |
|
| 667 | 598 |
/* |
Also available in: Unified diff
If an object was archived, the solr index will be removed for it.