Project

General

Profile

Revision 10272

Added by Jing Tao over 3 years ago

Add code to check the check sum by the DigestOutputStream.

View differences:

src/edu/ucsb/nceas/metacat/dataone/D1NodeService.java
33 33
import java.io.OutputStreamWriter;
34 34
import java.io.Writer;
35 35
import java.math.BigInteger;
36
import java.security.DigestOutputStream;
37
import java.security.MessageDigest;
38
import java.security.NoSuchAlgorithmException;
36 39
import java.sql.PreparedStatement;
37 40
import java.sql.ResultSet;
38 41
import java.sql.SQLException;
......
48 51
import java.util.concurrent.locks.Lock;
49 52

  
50 53
import javax.servlet.http.HttpServletRequest;
54
import javax.xml.bind.DatatypeConverter;
51 55

  
52 56
import org.apache.commons.io.IOUtils;
53 57
import org.apache.log4j.Logger;
......
67 71
import org.dataone.service.exceptions.ServiceFailure;
68 72
import org.dataone.service.exceptions.UnsupportedType;
69 73
import org.dataone.service.types.v1.AccessRule;
74
import org.dataone.service.types.v1.Checksum;
70 75
import org.dataone.service.types.v1.DescribeResponse;
71 76
import org.dataone.service.types.v1.Group;
72 77
import org.dataone.service.types.v1.Identifier;
......
403 408
    if(!allowed) {
404 409
        throw new NotAuthorized("1100", "Provited Identity doesn't have the WRITE permission on the pid "+pid.getValue());
405 410
    }
406
    // verify checksum, only if we can reset the inputstream
407
    if (object.markSupported()) {
408
        logMetacat.debug("Checking checksum for: " + pid.getValue());
409
	    String checksumAlgorithm = sysmeta.getChecksum().getAlgorithm();
410
	    String checksumValue = sysmeta.getChecksum().getValue();
411
	    try {
412
			String computedChecksumValue = ChecksumUtil.checksum(object, checksumAlgorithm).getValue();
413
			// it's very important that we don't consume the stream
414
			object.reset();
415
			if (!computedChecksumValue.equals(checksumValue)) {
416
			    logMetacat.error("Checksum for " + pid.getValue() + " does not match system metadata, computed = " + computedChecksumValue );
417
				throw new InvalidSystemMetadata("4896", "Checksum given does not match that of the object");
418
			}
419
		} catch (Exception e) {
420
			String msg = "Error verifying checksum values";
421
	      	logMetacat.error(msg, e);
422
	        throw new ServiceFailure("1190", msg + ": " + e.getMessage());
423
		}
424
    } else {
425
    	logMetacat.warn("mark is not supported on the object's input stream - cannot verify checksum without consuming stream");
426
    }
411
 
427 412
    	
428 413
    // we have the go ahead
429 414
    //if ( allowed ) {
......
454 439
            if(sysmeta.getFormatId() != null)  {
455 440
                formatId = sysmeta.getFormatId().getValue();
456 441
            }
457
	        localId = insertOrUpdateDocument(object,"UTF-8", pid, session, "insert", formatId);
442
	        localId = insertOrUpdateDocument(object,"UTF-8", pid, session, "insert", formatId, sysmeta.getChecksum());
458 443
	        //localId = im.getLocalId(pid.getValue());
459 444

  
460 445
        } catch (IOException e) {
......
478 463
	        
479 464
	      // DEFAULT CASE: DATA (needs to be checked and completed)
480 465
          try {
481
              localId = insertDataObject(object, pid, session);
466
              localId = insertDataObject(object, pid, session, sysmeta.getChecksum());
482 467
          } catch (ServiceFailure e) {
483 468
              removeSystemMetaAndIdentifier(pid);
484 469
              throw e;
470
          } catch (InvalidSystemMetadata e) {
471
              removeSystemMetaAndIdentifier(pid);
472
              throw e;
485 473
          } catch (Exception e) {
486 474
              removeSystemMetaAndIdentifier(pid);
487 475
              throw new ServiceFailure("1190", "The node is unable to create the object "+pid.getValue()+" since " + e.getMessage());
......
1445 1433
   * 
1446 1434
   */
1447 1435
  public String insertOrUpdateDocument(InputStream xmlStream, String encoding,  Identifier pid, 
1448
    Session session, String insertOrUpdate, String formatId) 
1436
    Session session, String insertOrUpdate, String formatId, Checksum checksum) 
1449 1437
    throws ServiceFailure, IOException, PropertyNotFoundException{
1450 1438
    
1451 1439
  	logMetacat.debug("Starting to insert xml document...");
......
1520 1508
    // do the insert or update action
1521 1509
    handler = new MetacatHandler(new Timer());
1522 1510
    String result = handler.handleInsertOrUpdateAction(request.getRemoteAddr(), request.getHeader("User-Agent"), null, 
1523
                        null, params, username, groupnames, false, false, xmlBytes, formatId);
1511
                        null, params, username, groupnames, false, false, xmlBytes, formatId, checksum);
1524 1512
    boolean isScienceMetadata = true;
1525 1513
    if(result.indexOf("<error>") != -1 || !IdentifierManager.getInstance().objectFileExists(localId, isScienceMetadata)) {
1526 1514
    	String detailCode = "";
......
1552 1540
   * @returns localId of the data object inserted
1553 1541
   */
1554 1542
  public String insertDataObject(InputStream object, Identifier pid, 
1555
          Session session) throws ServiceFailure {
1543
          Session session, Checksum checksum) throws ServiceFailure, InvalidSystemMetadata {
1556 1544
      
1557 1545
    String username = Constants.SUBJECT_PUBLIC;
1558 1546
    String[] groupnames = null;
......
1599 1587
          File dataDirectory = new File(datafilepath);
1600 1588
          dataDirectory.mkdirs();
1601 1589
  
1602
          File newFile = writeStreamToFile(dataDirectory, localId, object);
1590
          File newFile = writeStreamToFile(dataDirectory, localId, object, checksum, pid);
1603 1591
  
1604 1592
          // TODO: Check that the file size matches SystemMetadata
1605 1593
          // long size = newFile.length();
......
2093 2081
   * 
2094 2082
   * @throws ServiceFailure
2095 2083
   */
2096
  private File writeStreamToFile(File dir, String fileName, InputStream dataStream) 
2097
    throws ServiceFailure {
2084
  private File writeStreamToFile(File dir, String fileName, InputStream dataStream, Checksum checksum, Identifier pid) 
2085
    throws ServiceFailure, InvalidSystemMetadata {
2098 2086
    
2099 2087
    File newFile = new File(dir, fileName);
2100
    logMetacat.debug("Filename for write is: " + newFile.getAbsolutePath());
2088
    logMetacat.debug("Filename for write is: " + newFile.getAbsolutePath()+" for the data object pid "+pid.getValue());
2101 2089

  
2102 2090
    try {
2103 2091
        if (newFile.createNewFile()) {
2092
          if(checksum == null) {
2093
              logMetacat.error("D1NodeService.writeStreamToFile - the checksum object from the system metadata shouldn't be null for the data object "+pid.getValue());
2094
              throw new InvalidSystemMetadata("1180", "The checksum object from the system metadata shouldn't be null.");
2095
          } 
2096
          String checksumValue = checksum.getValue();
2097
          logMetacat.info("D1NodeService.writeStreamToFile - the checksum value from the system metadata is "+checksumValue+" for the data object "+pid.getValue());
2098
          if(checksumValue == null || checksumValue.trim().equals("")) {
2099
              logMetacat.error("D1NodeService.writeStreamToFile - the checksum value from the system metadata shouldn't be null or blank for the data object "+pid.getValue());
2100
              throw new InvalidSystemMetadata("1180", "The checksum value from the system metadata shouldn't be null or blank.");
2101
          }
2102
          String algorithm = checksum.getAlgorithm();
2103
          logMetacat.info("D1NodeService.writeStreamToFile - the algorithm to calculate the checksum from the system metadata is "+algorithm+" for the data object "+pid.getValue());
2104
          if(algorithm == null || algorithm.trim().equals("")) {
2105
              logMetacat.error("D1NodeService.writeStreamToFile - the algorithm to calculate the checksum from the system metadata shouldn't be null or blank for the data object "+pid.getValue());
2106
              throw new InvalidSystemMetadata("1180", "The algorithm to calculate the checksum from the system metadata shouldn't be null or blank.");
2107
          }
2108
          MessageDigest md = MessageDigest.getInstance(algorithm);
2104 2109
          // write data stream to desired file
2105
          OutputStream os = new FileOutputStream(newFile);
2110
          DigestOutputStream os = new DigestOutputStream( new FileOutputStream(newFile), md);
2106 2111
          long length = IOUtils.copyLarge(dataStream, os);
2107 2112
          os.flush();
2108 2113
          os.close();
2114
          String localChecksum = DatatypeConverter.printHexBinary(md.digest());
2115
          logMetacat.info("D1NodeService.writeStreamToFile - the check sum calculated from the saved local file is "+localChecksum);
2116
          if(localChecksum == null || localChecksum.trim().equals("") || !localChecksum.equalsIgnoreCase(checksumValue)) {
2117
              logMetacat.error("D1NodeService.writeStreamToFile - the check sum calculated from the saved local file is "+localChecksum+ ". But it doesn't match the value from the system metadata "+checksumValue+" for the object "+pid.getValue());
2118
              boolean success = newFile.delete();
2119
              logMetacat.info("delete the file "+newFile.getAbsolutePath()+" for the object "+pid.getValue()+" sucessfully?"+success);
2120
              throw new InvalidSystemMetadata("1180", "The checksum calculated from the saved local file is "+localChecksum+ ". But it doesn't match the value from the system metadata "+checksumValue+".");
2121
          }
2122
          
2109 2123
        } else {
2110 2124
          logMetacat.debug("File creation failed, or file already exists.");
2111 2125
          throw new ServiceFailure("1190", "File already exists: " + fileName);
2112 2126
        }
2113 2127
    } catch (FileNotFoundException e) {
2114
      logMetacat.debug("FNF: " + e.getMessage());
2128
      logMetacat.error("FNF: " + e.getMessage()+" for the data object "+pid.getValue(), e);
2115 2129
      throw new ServiceFailure("1190", "File not found: " + fileName + " " 
2116 2130
                + e.getMessage());
2117 2131
    } catch (IOException e) {
2118
      logMetacat.debug("IOE: " + e.getMessage());
2132
      logMetacat.error("IOE: " + e.getMessage()+" for the data object "+pid.getValue(), e);
2119 2133
      throw new ServiceFailure("1190", "File was not written: " + fileName 
2120 2134
                + " " + e.getMessage());
2135
    } catch (NoSuchAlgorithmException e) {
2136
        logMetacat.error("D1NodeService.writeStreamToFile - no such checksum algorithm exception " + e.getMessage()+" for the data object "+pid.getValue(), e);
2137
        throw new ServiceFailure("1190", "No such checksum algorithm: "  
2138
                + " " + e.getMessage());
2121 2139
    } finally {
2122 2140
        IOUtils.closeQuietly(dataStream);
2123 2141
    }

Also available in: Unified diff