Revision 10272
Added by Jing Tao over 7 years ago
| src/edu/ucsb/nceas/metacat/dataone/D1NodeService.java | ||
|---|---|---|
| 33 | 33 |
import java.io.OutputStreamWriter; |
| 34 | 34 |
import java.io.Writer; |
| 35 | 35 |
import java.math.BigInteger; |
| 36 |
import java.security.DigestOutputStream; |
|
| 37 |
import java.security.MessageDigest; |
|
| 38 |
import java.security.NoSuchAlgorithmException; |
|
| 36 | 39 |
import java.sql.PreparedStatement; |
| 37 | 40 |
import java.sql.ResultSet; |
| 38 | 41 |
import java.sql.SQLException; |
| ... | ... | |
| 48 | 51 |
import java.util.concurrent.locks.Lock; |
| 49 | 52 |
|
| 50 | 53 |
import javax.servlet.http.HttpServletRequest; |
| 54 |
import javax.xml.bind.DatatypeConverter; |
|
| 51 | 55 |
|
| 52 | 56 |
import org.apache.commons.io.IOUtils; |
| 53 | 57 |
import org.apache.log4j.Logger; |
| ... | ... | |
| 67 | 71 |
import org.dataone.service.exceptions.ServiceFailure; |
| 68 | 72 |
import org.dataone.service.exceptions.UnsupportedType; |
| 69 | 73 |
import org.dataone.service.types.v1.AccessRule; |
| 74 |
import org.dataone.service.types.v1.Checksum; |
|
| 70 | 75 |
import org.dataone.service.types.v1.DescribeResponse; |
| 71 | 76 |
import org.dataone.service.types.v1.Group; |
| 72 | 77 |
import org.dataone.service.types.v1.Identifier; |
| ... | ... | |
| 403 | 408 |
if(!allowed) {
|
| 404 | 409 |
throw new NotAuthorized("1100", "Provited Identity doesn't have the WRITE permission on the pid "+pid.getValue());
|
| 405 | 410 |
} |
| 406 |
// verify checksum, only if we can reset the inputstream |
|
| 407 |
if (object.markSupported()) {
|
|
| 408 |
logMetacat.debug("Checking checksum for: " + pid.getValue());
|
|
| 409 |
String checksumAlgorithm = sysmeta.getChecksum().getAlgorithm(); |
|
| 410 |
String checksumValue = sysmeta.getChecksum().getValue(); |
|
| 411 |
try {
|
|
| 412 |
String computedChecksumValue = ChecksumUtil.checksum(object, checksumAlgorithm).getValue(); |
|
| 413 |
// it's very important that we don't consume the stream |
|
| 414 |
object.reset(); |
|
| 415 |
if (!computedChecksumValue.equals(checksumValue)) {
|
|
| 416 |
logMetacat.error("Checksum for " + pid.getValue() + " does not match system metadata, computed = " + computedChecksumValue );
|
|
| 417 |
throw new InvalidSystemMetadata("4896", "Checksum given does not match that of the object");
|
|
| 418 |
} |
|
| 419 |
} catch (Exception e) {
|
|
| 420 |
String msg = "Error verifying checksum values"; |
|
| 421 |
logMetacat.error(msg, e); |
|
| 422 |
throw new ServiceFailure("1190", msg + ": " + e.getMessage());
|
|
| 423 |
} |
|
| 424 |
} else {
|
|
| 425 |
logMetacat.warn("mark is not supported on the object's input stream - cannot verify checksum without consuming stream");
|
|
| 426 |
} |
|
| 411 |
|
|
| 427 | 412 |
|
| 428 | 413 |
// we have the go ahead |
| 429 | 414 |
//if ( allowed ) {
|
| ... | ... | |
| 454 | 439 |
if(sysmeta.getFormatId() != null) {
|
| 455 | 440 |
formatId = sysmeta.getFormatId().getValue(); |
| 456 | 441 |
} |
| 457 |
localId = insertOrUpdateDocument(object,"UTF-8", pid, session, "insert", formatId); |
|
| 442 |
localId = insertOrUpdateDocument(object,"UTF-8", pid, session, "insert", formatId, sysmeta.getChecksum());
|
|
| 458 | 443 |
//localId = im.getLocalId(pid.getValue()); |
| 459 | 444 |
|
| 460 | 445 |
} catch (IOException e) {
|
| ... | ... | |
| 478 | 463 |
|
| 479 | 464 |
// DEFAULT CASE: DATA (needs to be checked and completed) |
| 480 | 465 |
try {
|
| 481 |
localId = insertDataObject(object, pid, session); |
|
| 466 |
localId = insertDataObject(object, pid, session, sysmeta.getChecksum());
|
|
| 482 | 467 |
} catch (ServiceFailure e) {
|
| 483 | 468 |
removeSystemMetaAndIdentifier(pid); |
| 484 | 469 |
throw e; |
| 470 |
} catch (InvalidSystemMetadata e) {
|
|
| 471 |
removeSystemMetaAndIdentifier(pid); |
|
| 472 |
throw e; |
|
| 485 | 473 |
} catch (Exception e) {
|
| 486 | 474 |
removeSystemMetaAndIdentifier(pid); |
| 487 | 475 |
throw new ServiceFailure("1190", "The node is unable to create the object "+pid.getValue()+" since " + e.getMessage());
|
| ... | ... | |
| 1445 | 1433 |
* |
| 1446 | 1434 |
*/ |
| 1447 | 1435 |
public String insertOrUpdateDocument(InputStream xmlStream, String encoding, Identifier pid, |
| 1448 |
Session session, String insertOrUpdate, String formatId) |
|
| 1436 |
Session session, String insertOrUpdate, String formatId, Checksum checksum)
|
|
| 1449 | 1437 |
throws ServiceFailure, IOException, PropertyNotFoundException{
|
| 1450 | 1438 |
|
| 1451 | 1439 |
logMetacat.debug("Starting to insert xml document...");
|
| ... | ... | |
| 1520 | 1508 |
// do the insert or update action |
| 1521 | 1509 |
handler = new MetacatHandler(new Timer()); |
| 1522 | 1510 |
String result = handler.handleInsertOrUpdateAction(request.getRemoteAddr(), request.getHeader("User-Agent"), null,
|
| 1523 |
null, params, username, groupnames, false, false, xmlBytes, formatId); |
|
| 1511 |
null, params, username, groupnames, false, false, xmlBytes, formatId, checksum);
|
|
| 1524 | 1512 |
boolean isScienceMetadata = true; |
| 1525 | 1513 |
if(result.indexOf("<error>") != -1 || !IdentifierManager.getInstance().objectFileExists(localId, isScienceMetadata)) {
|
| 1526 | 1514 |
String detailCode = ""; |
| ... | ... | |
| 1552 | 1540 |
* @returns localId of the data object inserted |
| 1553 | 1541 |
*/ |
| 1554 | 1542 |
public String insertDataObject(InputStream object, Identifier pid, |
| 1555 |
Session session) throws ServiceFailure {
|
|
| 1543 |
Session session, Checksum checksum) throws ServiceFailure, InvalidSystemMetadata {
|
|
| 1556 | 1544 |
|
| 1557 | 1545 |
String username = Constants.SUBJECT_PUBLIC; |
| 1558 | 1546 |
String[] groupnames = null; |
| ... | ... | |
| 1599 | 1587 |
File dataDirectory = new File(datafilepath); |
| 1600 | 1588 |
dataDirectory.mkdirs(); |
| 1601 | 1589 |
|
| 1602 |
File newFile = writeStreamToFile(dataDirectory, localId, object); |
|
| 1590 |
File newFile = writeStreamToFile(dataDirectory, localId, object, checksum, pid);
|
|
| 1603 | 1591 |
|
| 1604 | 1592 |
// TODO: Check that the file size matches SystemMetadata |
| 1605 | 1593 |
// long size = newFile.length(); |
| ... | ... | |
| 2093 | 2081 |
* |
| 2094 | 2082 |
* @throws ServiceFailure |
| 2095 | 2083 |
*/ |
| 2096 |
private File writeStreamToFile(File dir, String fileName, InputStream dataStream) |
|
| 2097 |
throws ServiceFailure {
|
|
| 2084 |
private File writeStreamToFile(File dir, String fileName, InputStream dataStream, Checksum checksum, Identifier pid)
|
|
| 2085 |
throws ServiceFailure, InvalidSystemMetadata {
|
|
| 2098 | 2086 |
|
| 2099 | 2087 |
File newFile = new File(dir, fileName); |
| 2100 |
logMetacat.debug("Filename for write is: " + newFile.getAbsolutePath());
|
|
| 2088 |
logMetacat.debug("Filename for write is: " + newFile.getAbsolutePath()+" for the data object pid "+pid.getValue());
|
|
| 2101 | 2089 |
|
| 2102 | 2090 |
try {
|
| 2103 | 2091 |
if (newFile.createNewFile()) {
|
| 2092 |
if(checksum == null) {
|
|
| 2093 |
logMetacat.error("D1NodeService.writeStreamToFile - the checksum object from the system metadata shouldn't be null for the data object "+pid.getValue());
|
|
| 2094 |
throw new InvalidSystemMetadata("1180", "The checksum object from the system metadata shouldn't be null.");
|
|
| 2095 |
} |
|
| 2096 |
String checksumValue = checksum.getValue(); |
|
| 2097 |
logMetacat.info("D1NodeService.writeStreamToFile - the checksum value from the system metadata is "+checksumValue+" for the data object "+pid.getValue());
|
|
| 2098 |
if(checksumValue == null || checksumValue.trim().equals("")) {
|
|
| 2099 |
logMetacat.error("D1NodeService.writeStreamToFile - the checksum value from the system metadata shouldn't be null or blank for the data object "+pid.getValue());
|
|
| 2100 |
throw new InvalidSystemMetadata("1180", "The checksum value from the system metadata shouldn't be null or blank.");
|
|
| 2101 |
} |
|
| 2102 |
String algorithm = checksum.getAlgorithm(); |
|
| 2103 |
logMetacat.info("D1NodeService.writeStreamToFile - the algorithm to calculate the checksum from the system metadata is "+algorithm+" for the data object "+pid.getValue());
|
|
| 2104 |
if(algorithm == null || algorithm.trim().equals("")) {
|
|
| 2105 |
logMetacat.error("D1NodeService.writeStreamToFile - the algorithm to calculate the checksum from the system metadata shouldn't be null or blank for the data object "+pid.getValue());
|
|
| 2106 |
throw new InvalidSystemMetadata("1180", "The algorithm to calculate the checksum from the system metadata shouldn't be null or blank.");
|
|
| 2107 |
} |
|
| 2108 |
MessageDigest md = MessageDigest.getInstance(algorithm); |
|
| 2104 | 2109 |
// write data stream to desired file |
| 2105 |
OutputStream os = new FileOutputStream(newFile);
|
|
| 2110 |
DigestOutputStream os = new DigestOutputStream( new FileOutputStream(newFile), md);
|
|
| 2106 | 2111 |
long length = IOUtils.copyLarge(dataStream, os); |
| 2107 | 2112 |
os.flush(); |
| 2108 | 2113 |
os.close(); |
| 2114 |
String localChecksum = DatatypeConverter.printHexBinary(md.digest()); |
|
| 2115 |
logMetacat.info("D1NodeService.writeStreamToFile - the check sum calculated from the saved local file is "+localChecksum);
|
|
| 2116 |
if(localChecksum == null || localChecksum.trim().equals("") || !localChecksum.equalsIgnoreCase(checksumValue)) {
|
|
| 2117 |
logMetacat.error("D1NodeService.writeStreamToFile - the check sum calculated from the saved local file is "+localChecksum+ ". But it doesn't match the value from the system metadata "+checksumValue+" for the object "+pid.getValue());
|
|
| 2118 |
boolean success = newFile.delete(); |
|
| 2119 |
logMetacat.info("delete the file "+newFile.getAbsolutePath()+" for the object "+pid.getValue()+" sucessfully?"+success);
|
|
| 2120 |
throw new InvalidSystemMetadata("1180", "The checksum calculated from the saved local file is "+localChecksum+ ". But it doesn't match the value from the system metadata "+checksumValue+".");
|
|
| 2121 |
} |
|
| 2122 |
|
|
| 2109 | 2123 |
} else {
|
| 2110 | 2124 |
logMetacat.debug("File creation failed, or file already exists.");
|
| 2111 | 2125 |
throw new ServiceFailure("1190", "File already exists: " + fileName);
|
| 2112 | 2126 |
} |
| 2113 | 2127 |
} catch (FileNotFoundException e) {
|
| 2114 |
logMetacat.debug("FNF: " + e.getMessage());
|
|
| 2128 |
logMetacat.error("FNF: " + e.getMessage()+" for the data object "+pid.getValue(), e);
|
|
| 2115 | 2129 |
throw new ServiceFailure("1190", "File not found: " + fileName + " "
|
| 2116 | 2130 |
+ e.getMessage()); |
| 2117 | 2131 |
} catch (IOException e) {
|
| 2118 |
logMetacat.debug("IOE: " + e.getMessage());
|
|
| 2132 |
logMetacat.error("IOE: " + e.getMessage()+" for the data object "+pid.getValue(), e);
|
|
| 2119 | 2133 |
throw new ServiceFailure("1190", "File was not written: " + fileName
|
| 2120 | 2134 |
+ " " + e.getMessage()); |
| 2135 |
} catch (NoSuchAlgorithmException e) {
|
|
| 2136 |
logMetacat.error("D1NodeService.writeStreamToFile - no such checksum algorithm exception " + e.getMessage()+" for the data object "+pid.getValue(), e);
|
|
| 2137 |
throw new ServiceFailure("1190", "No such checksum algorithm: "
|
|
| 2138 |
+ " " + e.getMessage()); |
|
| 2121 | 2139 |
} finally {
|
| 2122 | 2140 |
IOUtils.closeQuietly(dataStream); |
| 2123 | 2141 |
} |
Also available in: Unified diff
Add code to check the check sum by the DigestOutputStream.