Project

General

Profile

« Previous | Next » 

Revision 6102

transfer full System Metadata (as XML) during document and data replication

View differences:

src/edu/ucsb/nceas/metacat/util/ReplicationUtil.java
1
/**
2
 *  '$RCSfile$'
3
 *    Purpose: A Class that implements utility methods for a metadata catalog
4
 *  Copyright: 2009 Regents of the University of California and the
5
 *             National Center for Ecological Analysis and Synthesis
6
 *    Authors: Michael Daigle
7
 *
8
 *   '$Author: daigle $'
9
 *     '$Date: 2009-08-04 14:32:58 -0700 (Tue, 04 Aug 2009) $'
10
 * '$Revision: 5015 $'
11
 *
12
 * This program is free software; you can redistribute it and/or modify
13
 * it under the terms of the GNU General Public License as published by
14
 * the Free Software Foundation; either version 2 of the License, or
15
 * (at your option) any later version.
16
 *
17
 * This program is distributed in the hope that it will be useful,
18
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20
 * GNU General Public License for more details.
21
 *
22
 * You should have received a copy of the GNU General Public License
23
 * along with this program; if not, write to the Free Software
24
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
25
 */
26

  
27
package edu.ucsb.nceas.metacat.util;
28

  
29
/**
30
 * A suite of utility classes for the metadata replication functions
31
 */
32
public class ReplicationUtil {
33
	  
34
    public static String startTag = "<systemMetadata>";
35
    public static String endTag = "</systemMetadata>";
36
    
37
    /**
38
     * return the contents between start and end tag
39
     */
40
    public static String getSystemMetadataContent(String docInfoStr) {
41
    	// get the system metadata portion
42
        String systemMetadataXML = null;
43
        if (docInfoStr.indexOf(startTag) > -1) {
44
      	  systemMetadataXML = docInfoStr.substring(docInfoStr.indexOf(startTag) + startTag.length(), docInfoStr.lastIndexOf(endTag));
45
        }
46
        return systemMetadataXML;
47
    }
48
    
49
    /**
50
     * return the string WITHOUT the contents between start and end tag
51
     */
52
    public static String getContentWithoutSystemMetadata(String docInfoStr) {
53
    	// strip out the system metadata portion
54
        if (docInfoStr.indexOf(startTag) > -1) {      	  
55
      	  docInfoStr = docInfoStr.substring(0, docInfoStr.indexOf(startTag)) + docInfoStr.substring(docInfoStr.indexOf(endTag) + endTag.length());
56
        }
57
        return docInfoStr;
58
    }
59

  
60

  
61
}
src/edu/ucsb/nceas/metacat/replication/ReplicationHandler.java
42 42
import edu.ucsb.nceas.metacat.properties.PropertyService;
43 43
import edu.ucsb.nceas.metacat.shared.HandlerException;
44 44
import edu.ucsb.nceas.metacat.util.MetacatUtil;
45
import edu.ucsb.nceas.metacat.util.ReplicationUtil;
45 46
import edu.ucsb.nceas.metacat.IdentifierManager;
46 47
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
47 48

  
......
54 55

  
55 56
import org.apache.log4j.Logger;
56 57
import org.dataone.service.types.SystemMetadata;
58
import org.dataone.service.types.util.ServiceTypeUtil;
57 59
import org.xml.sax.ContentHandler;
58 60
import org.xml.sax.ErrorHandler;
59 61
import org.xml.sax.InputSource;
......
352 354
                       "&action=getdocumentinfo&docid="+accNumber;
353 355
      docInfoURLStr = MetacatUtil.replaceWhiteSpaceForURL(docInfoURLStr);
354 356
      URL docinfoUrl = new URL(docInfoURLStr);
355
      logReplication.info("ReplicationHandler.handleSingleXMLDocument - Sending message: " +
356
                                                  docinfoUrl.toString());
357
      logReplication.info("ReplicationHandler.handleSingleXMLDocument - Sending message: " + docinfoUrl.toString());
357 358
      String docInfoStr = ReplicationService.getURLContent(docinfoUrl);
359
      
360
      // strip out the system metadata portion
361
      String systemMetadataXML = ReplicationUtil.getSystemMetadataContent(docInfoStr);
362
   	  docInfoStr = ReplicationUtil.getContentWithoutSystemMetadata(docInfoStr);
363
      
358 364
      docinfoParser.parse(new InputSource(new StringReader(docInfoStr)));
359 365
      Hashtable<String, String> docinfoHash = dih.getDocInfo();
360 366
      // Get home server of the docid
......
418 424
          }
419 425
      }
420 426
      
421
      //process guid
422
      String guid = docinfoHash.get("guid");
423
      logMetacat.debug("Guid passed from docinfo hash: " + guid);
424
      if (guid != null) {
425
          // if the guid was passed in, put it in the identifiers table
427
      // process system metadata
428
      if (systemMetadataXML != null) {
429
    	  SystemMetadata sysMeta = 
430
    		  (SystemMetadata) ServiceTypeUtil.deserializeServiceType(
431
    				  SystemMetadata.class, 
432
    				  new ByteArrayInputStream(systemMetadataXML.getBytes("UTF-8")));
433
    	  String guid = sysMeta.getIdentifier().getValue();
426 434
    	  if (!IdentifierManager.getInstance().identifierExists(guid)) {
427
    		  logReplication.debug("Creating guid/docid mapping for docid " + docinfoHash.get("docid") + " and guid: " + guid);
435
    		  logReplication.debug("Creating system metadata and guid/docid mapping for docid " + docinfoHash.get("docid") + " and guid: " + guid);
428 436
    		  IdentifierManager.getInstance().createMapping(guid, docinfoHash.get("docid"));
437
    		  IdentifierManager.getInstance().createSystemMetadata(sysMeta);
429 438
    	  } else {
430 439
    		  logReplication.debug("Updating guid/docid mapping for docid " + docinfoHash.get("docid") + " and guid: " + guid);
431 440
    		  IdentifierManager.getInstance().updateMapping(guid, docinfoHash.get("docid"));
432 441
    	  }
433
    	  // process system metadata
434
    	  Date dateUploaded = new Date(new Long(docinfoHash.get("date_uploaded")));
435
          Date dateModified = new Date(new Long(docinfoHash.get("date_modified")));
436
          SystemMetadata sysMeta = IdentifierManager.getInstance().asSystemMetadata(
437
                  dateUploaded, 
438
                  docinfoHash.get("rights_holder"),
439
                  docinfoHash.get("checksum"), 
440
                  docinfoHash.get("checksum_algorithm"), 
441
                  docinfoHash.get("origin_member_node"),
442
                  docinfoHash.get("authoritive_member_node"), 
443
                  dateModified,
444
                  docinfoHash.get("submitter"),
445
                  docinfoHash.get("guid"),
446
                  docinfoHash.get("object_format"),
447
                  new Long(docinfoHash.get("size")).longValue());
448
          if (actions.equalsIgnoreCase("INSERT")) {
449
	    	  logMetacat.debug("Creating systemMetadata for guid: " + guid);
450
	    	  IdentifierManager.getInstance().createSystemMetadata(sysMeta);
451
          }
452 442
          IdentifierManager.getInstance().updateSystemMetadata(sysMeta);
453 443
      }
454 444
      
......
531 521
      URL docinfoUrl = new URL(docInfoURLString);
532 522

  
533 523
      String docInfoStr = ReplicationService.getURLContent(docinfoUrl);
524
      
525
      // strip out the system metadata portion
526
      String systemMetadataXML = ReplicationUtil.getSystemMetadataContent(docInfoStr);
527
   	  docInfoStr = ReplicationUtil.getContentWithoutSystemMetadata(docInfoStr);  
528
   	  
534 529
      docinfoParser.parse(new InputSource(new StringReader(docInfoStr)));
535 530
      Hashtable<String, String> docinfoHash = dih.getDocInfo();
536 531
      
......
547 542
      /*String accnum=docId+util.getProperty("document.accNumSeparator")+
548 543
                                              (String)docinfoHash.get("rev");*/
549 544

  
550

  
551 545
      String datafilePath = PropertyService.getProperty("application.datafilepath");
552 546
      // Get data file content
553 547
      String readDataURLString = "https://" + remoteserver + "?server="+
......
589 583
        }
590 584
        
591 585
        //process guid
592
        String guid = docinfoHash.get("guid");
593
        logMetacat.debug("Guid passed from docinfo hash: " + guid);
594
        if (guid != null) {
595
            // if the guid was passed in, put it in the identifiers table
586
     // process system metadata
587
        if (systemMetadataXML != null) {
588
      	  SystemMetadata sysMeta = 
589
      		  (SystemMetadata) ServiceTypeUtil.deserializeServiceType(
590
      				  SystemMetadata.class, 
591
      				  new ByteArrayInputStream(systemMetadataXML.getBytes("UTF-8")));
592
      	  String guid = sysMeta.getIdentifier().getValue();
596 593
      	  if (!IdentifierManager.getInstance().identifierExists(guid)) {
597
      		  logReplication.debug("Creating guid/docid mapping for docid " + docinfoHash.get("docid") + " and guid: " + guid);
594
      		  logReplication.debug("Creating system metadata and guid/docid mapping for docid " + docinfoHash.get("docid") + " and guid: " + guid);
598 595
      		  IdentifierManager.getInstance().createMapping(guid, docinfoHash.get("docid"));
596
      		  IdentifierManager.getInstance().createSystemMetadata(sysMeta);
599 597
      	  } else {
600 598
      		  logReplication.debug("Updating guid/docid mapping for docid " + docinfoHash.get("docid") + " and guid: " + guid);
601 599
      		  IdentifierManager.getInstance().updateMapping(guid, docinfoHash.get("docid"));
602 600
      	  }
603
      	  // process system metadata
604
      	  Date dateUploaded = new Date(new Long(docinfoHash.get("date_uploaded")));
605
            Date dateModified = new Date(new Long(docinfoHash.get("date_modified")));
606
            SystemMetadata sysMeta = IdentifierManager.getInstance().asSystemMetadata(
607
                    dateUploaded, 
608
                    docinfoHash.get("rights_holder"),
609
                    docinfoHash.get("checksum"), 
610
                    docinfoHash.get("checksum_algorithm"), 
611
                    docinfoHash.get("origin_member_node"),
612
                    docinfoHash.get("authoritive_member_node"), 
613
                    dateModified,
614
                    docinfoHash.get("submitter"),
615
                    docinfoHash.get("guid"),
616
                    docinfoHash.get("object_format"),
617
                    new Long(docinfoHash.get("size")).longValue());
618
            if (actions.equalsIgnoreCase("INSERT")) {
619
  	    	  logMetacat.debug("Creating systemMetadata for guid: " + guid);
620
  	    	  IdentifierManager.getInstance().createSystemMetadata(sysMeta);
621
            }
622 601
            IdentifierManager.getInstance().updateSystemMetadata(sysMeta);
623 602
        }
624 603
        
src/edu/ucsb/nceas/metacat/replication/ReplicationService.java
28 28

  
29 29
import java.io.BufferedInputStream;
30 30
import java.io.BufferedOutputStream;
31
import java.io.ByteArrayInputStream;
32
import java.io.ByteArrayOutputStream;
31 33
import java.io.File;
32 34
import java.io.FileInputStream;
33 35
import java.io.FileNotFoundException;
......
55 57

  
56 58
import org.apache.log4j.Logger;
57 59
import org.dataone.service.types.SystemMetadata;
60
import org.dataone.service.types.util.ServiceTypeUtil;
58 61
import org.xml.sax.InputSource;
59 62
import org.xml.sax.SAXException;
60 63
import org.xml.sax.XMLReader;
......
80 83
import edu.ucsb.nceas.metacat.shared.ServiceException;
81 84
import edu.ucsb.nceas.metacat.util.DocumentUtil;
82 85
import edu.ucsb.nceas.metacat.util.MetacatUtil;
86
import edu.ucsb.nceas.metacat.util.ReplicationUtil;
83 87
import edu.ucsb.nceas.metacat.util.SystemUtil;
84 88
import edu.ucsb.nceas.utilities.FileUtil;
85 89
import edu.ucsb.nceas.utilities.GeneralPropertyException;
......
550 554
			
551 555

  
552 556
			String docInfoStr = ReplicationService.getURLContent(docinfourl);
553

  
557
			// strip out the system metadata portion
558
			String systemMetadataXML = ReplicationUtil.getSystemMetadataContent(docInfoStr);
559
			docInfoStr = ReplicationUtil.getContentWithoutSystemMetadata(docInfoStr);
560
		   	  
554 561
			//dih is the parser for the docinfo xml format
555 562
			DocInfoHandler dih = new DocInfoHandler();
556 563
			XMLReader docinfoParser = ReplicationHandler.initParser(dih);
557 564
			docinfoParser.parse(new InputSource(new StringReader(docInfoStr)));
558 565
			//      Hashtable<String,Vector<AccessControlForSingleFile>> docinfoHash = dih.getDocInfo();
559 566
			Hashtable<String, String> docinfoHash = dih.getDocInfo();
560

  
561 567
			
562 568
			// Get home server of this docid
563 569
			String homeServer = (String) docinfoHash.get("home_server");
564
			String guid = (String) docinfoHash.get("guid");
565
			logReplication.debug("GUID found in docinfoHash: " + guid);
566 570
			
567
			logReplication.debug("Processing guid " + guid + 
568
			  " information from handleForceReplicationRequest: " + 
569
			  docinfoHash.toString());
570
			
571
			//if the guid was passed in, we have system metadata with the doc
572
			if (guid != null) {
573
				// map the guid to local identifiers
574
	        	if (dbaction.equalsIgnoreCase("UPDATE")) {
575
	        		logReplication.info("updating mapping: guid: " + guid + " localId: " + docinfoHash.get("docid"));
576
	        		IdentifierManager.getInstance().updateMapping(guid, docinfoHash.get("docid"));
577
	        	} else { // insert
578
		            logReplication.info("creating mapping: guid: " + guid + " localId: " + docinfoHash.get("docid"));
579
		            IdentifierManager.getInstance().createMapping(guid, docinfoHash.get("docid"));
580
	        	}
581
	        	// process system metadata
582
	        	logReplication.info("handleForceReplicateRequest: updating system metadata");
583
	        	Date dateUploaded = new Date(new Long(docinfoHash.get("date_uploaded")));
584
		        Date dateModified = new Date(new Long(docinfoHash.get("date_modified")));
585
	            SystemMetadata sysMeta = IdentifierManager.getInstance().asSystemMetadata(
586
	                    dateUploaded, 
587
	                    docinfoHash.get("rights_holder"),
588
	                    docinfoHash.get("checksum"), 
589
	                    docinfoHash.get("checksum_algorithm"), 
590
	                    docinfoHash.get("origin_member_node"),
591
	                    docinfoHash.get("authoritive_member_node"), 
592
	                    dateModified,
593
	                    docinfoHash.get("submitter"),
594
	                    docinfoHash.get("guid"),
595
	                    docinfoHash.get("object_format"),
596
	                    new Long(docinfoHash.get("size")).longValue());
597
	        	if (dbaction.equalsIgnoreCase("INSERT")) {
598
		            logReplication.info("Adding systemMetadata for guid: " + guid);
599
		            IdentifierManager.getInstance().createSystemMetadata(sysMeta);
600
	        	}
601
	            IdentifierManager.getInstance().updateSystemMetadata(sysMeta);
571
			// process system metadata
572
			if (systemMetadataXML != null) {
573
				SystemMetadata sysMeta = 
574
					(SystemMetadata) ServiceTypeUtil.deserializeServiceType(
575
							SystemMetadata.class,
576
							new ByteArrayInputStream(systemMetadataXML.getBytes("UTF-8")));
577
				String guid = sysMeta.getIdentifier().getValue();
578
				if (!IdentifierManager.getInstance().identifierExists(guid)) {
579
					logReplication.debug("Creating system metadata and guid/docid mapping for docid "
580
									+ docinfoHash.get("docid")
581
									+ " and guid: "
582
									+ guid);
583
					IdentifierManager.getInstance().createMapping(guid, docinfoHash.get("docid"));
584
					IdentifierManager.getInstance().createSystemMetadata(sysMeta);
585
				} else {
586
					logReplication.debug("Updating guid/docid mapping for docid "
587
									+ docinfoHash.get("docid") + " and guid: "
588
									+ guid);
589
					IdentifierManager.getInstance().updateMapping(guid, docinfoHash.get("docid"));
590
				}
591
				IdentifierManager.getInstance().updateSystemMetadata(sysMeta);
602 592
			}
603 593
      
604 594
        	// replicate doc contents
......
663 653
				logReplication.info("ReplicationService.handleForceReplicateRequest - document " + docid + " added to DB with "
664 654
						+ "action " + dbaction);
665 655
				
666
				if(guid != null)
667
                {
668
                    if(!docName.trim().equals("systemMetadata"))
669
                    {
670
                        logReplication.info("replicate D1GUID:" + guid + ":D1SCIMETADATA:" + 
671
                                docid + ":");
672
                    }
673
                    else
674
                    {
675
                        logReplication.info("replicate D1GUID:" + guid + ":D1SYSMETADATA:" + 
676
                                docid + ":");
677
                    }
678
                }
679 656
				EventLog.getInstance().log(request.getRemoteAddr(), REPLICATIONUSER, docid, dbaction);
680 657
			}
681 658
		} catch (SQLException sqle) {
......
808 785
					+ "&action=getdocumentinfo&docid=" + docid);
809 786

  
810 787
			String docInfoStr = ReplicationService.getURLContent(docinfourl);
788
			
789
			// strip out the system metadata portion
790
		    String systemMetadataXML = ReplicationUtil.getSystemMetadataContent(docInfoStr);
791
		   	docInfoStr = ReplicationUtil.getContentWithoutSystemMetadata(docInfoStr);
811 792

  
812 793
			//dih is the parser for the docinfo xml format
813 794
			DocInfoHandler dih = new DocInfoHandler();
......
868 849
		            }
869 850
		        }
870 851
		        
871
				String guid = (String) docinfoHash.get("guid");
872
				//if the guid was passed in, we have system metadata with the doc
873
				if (guid != null) {
874
					// map the guid to local identifiers
875
		        	if (dbaction.equalsIgnoreCase("UPDATE")) {
876
		        		logReplication.info("updating mapping: guid: " + guid + " localId: " + docinfoHash.get("docid"));
877
		        		IdentifierManager.getInstance().updateMapping(guid, docinfoHash.get("docid"));
878
		        	} else { // insert
879
			            logReplication.info("creating mapping: guid: " + guid + " localId: " + docinfoHash.get("docid"));
880
			            IdentifierManager.getInstance().createMapping(guid, docinfoHash.get("docid"));
881
		        	}
882
		        	// process system metadata
883
		        	logReplication.info("handleForceReplicateRequest: updating system metadata");
884
		        	Date dateUploaded = new Date(new Long(docinfoHash.get("date_uploaded")));
885
			        Date dateModified = new Date(new Long(docinfoHash.get("date_modified")));
886
		            SystemMetadata sysMeta = IdentifierManager.getInstance().asSystemMetadata(
887
		                    dateUploaded, 
888
		                    docinfoHash.get("rights_holder"),
889
		                    docinfoHash.get("checksum"), 
890
		                    docinfoHash.get("checksum_algorithm"), 
891
		                    docinfoHash.get("origin_member_node"),
892
		                    docinfoHash.get("authoritive_member_node"), 
893
		                    dateModified,
894
		                    docinfoHash.get("submitter"),
895
		                    docinfoHash.get("guid"),
896
		                    docinfoHash.get("object_format"),
897
		                    new Long(docinfoHash.get("size")).longValue());
898
		        	if (dbaction.equalsIgnoreCase("INSERT")) {
899
			            logReplication.info("Adding systemMetadata for guid: " + guid);
900
			            IdentifierManager.getInstance().createSystemMetadata(sysMeta);
901
		        	}
902
		            IdentifierManager.getInstance().updateSystemMetadata(sysMeta);
903
				}
852
		        // process system metadata
853
		        if (systemMetadataXML != null) {
854
		      	  SystemMetadata sysMeta = 
855
		      		  (SystemMetadata) ServiceTypeUtil.deserializeServiceType(
856
		      				  SystemMetadata.class, 
857
		      				  new ByteArrayInputStream(systemMetadataXML.getBytes("UTF-8")));
858
		      	  String guid = sysMeta.getIdentifier().getValue();
859
		      	  if (!IdentifierManager.getInstance().identifierExists(guid)) {
860
		      		  logReplication.debug("Creating system metadata and guid/docid mapping for docid " + docinfoHash.get("docid") + " and guid: " + guid);
861
		      		  IdentifierManager.getInstance().createMapping(guid, docinfoHash.get("docid"));
862
		      		  IdentifierManager.getInstance().createSystemMetadata(sysMeta);
863
		      	  } else {
864
		      		  logReplication.debug("Updating guid/docid mapping for docid " + docinfoHash.get("docid") + " and guid: " + guid);
865
		      		  IdentifierManager.getInstance().updateMapping(guid, docinfoHash.get("docid"));
866
		      	  }
867
		      	  IdentifierManager.getInstance().updateSystemMetadata(sysMeta);
868
		        }
904 869

  
905 870
				if (writeException != null) {
906 871
					throw writeException;
......
988 953
		StringBuffer sb = new StringBuffer();
989 954

  
990 955
		try {
991
		  IdentifierManager idman = IdentifierManager.getInstance();
992

  
993 956
			DocumentImpl doc = new DocumentImpl(docid);
994 957
			sb.append("<documentinfo><docid>").append(docid);
995 958
			sb.append("</docid>");
996
			try
997
			{
998
			  String guid = idman.getGUID(doc.getDocID(), doc.getRev());
999
			  sb.append("<guid>").append(guid).append("</guid>");
1000
			  Hashtable<String, String> sysmetaInfo = idman.getSystemMetadataInfo(guid);
1001
			  
1002
			  Enumeration<String> sysmetaKeys = sysmetaInfo.keys();
1003
			  while(sysmetaKeys.hasMoreElements())
1004
			  {
1005
			      String key = sysmetaKeys.nextElement();
1006
			      sb.append("<" + key + ">" + sysmetaInfo.get(key).toString() + "</" + key + ">");
1007
			  }
1008
			  
959
			
960
			try {
961
				// serialize the System Metadata as XML for docinfo
962
				String guid = IdentifierManager.getInstance().getGUID(doc.getDocID(), doc.getRev());
963
				SystemMetadata systemMetadata = IdentifierManager.getInstance().getSystemMetadata(guid);
964
				ByteArrayOutputStream baos = new ByteArrayOutputStream();
965
				ServiceTypeUtil.serializeServiceType(SystemMetadata.class, systemMetadata, baos);
966
				String systemMetadataXML = baos.toString("UTF-8");
967
				sb.append("<systemMetadata>");
968
				sb.append(systemMetadataXML);
969
				sb.append("</systemMetadata>");
970
			} catch(McdbDocNotFoundException e) {
971
			  logMetacat.warn("No SystemMetadata found for: " + docid);
1009 972
			}
1010
			catch(McdbDocNotFoundException e)
1011
			{
1012
			  //do nothing, there was no guid for this document
1013
			}
1014 973
			sb.append("<docname>").append(doc.getDocname());
1015 974
			sb.append("</docname><doctype>").append(doc.getDoctype());
1016 975
			sb.append("</doctype>");

Also available in: Unified diff