Project

General

Profile

« Previous | Next » 

Revision 7356

stream the replication "update" response rather than building up a complete list in a stringbuffer. prompted by findings on t he CN: https://redmine.dataone.org/issues/3141

View differences:

src/edu/ucsb/nceas/metacat/replication/ReplicationHandler.java
38 38
import java.sql.Timestamp;
39 39
import java.text.DateFormat;
40 40
import java.text.ParseException;
41
import java.util.Calendar;
42 41
import java.util.Date;
43 42
import java.util.Hashtable;
44 43
import java.util.TimerTask;
45 44
import java.util.Vector;
46 45

  
47
import javax.xml.bind.DatatypeConverter;
48 46

  
49 47
import org.apache.log4j.Logger;
50 48
import org.dataone.service.types.v1.SystemMetadata;
......
71 69
import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;
72 70
import edu.ucsb.nceas.metacat.database.DBConnection;
73 71
import edu.ucsb.nceas.metacat.database.DBConnectionPool;
74
import edu.ucsb.nceas.metacat.database.DatabaseService;
75 72
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;
76 73
import edu.ucsb.nceas.metacat.properties.PropertyService;
77 74
import edu.ucsb.nceas.metacat.shared.HandlerException;
......
168 165
                                        // Server list
169 166
    String server = null; // Variable to store server name
170 167
//    String update;
171
    Vector<String> responses = new Vector<String>();
168
    Vector<InputStream> responses = new Vector<InputStream>();
172 169
    URL u;
173 170
    long replicationStartTime = System.currentTimeMillis();
174 171
    long timeToGetServerList = 0;
......
182 179
        replServer = serverList.serverAt(i);
183 180
        // Get server name from ReplicationServer object
184 181
        server = replServer.getServerName().trim();
185
        String result = null;
182
        InputStream result = null;
186 183
        logReplication.info("ReplicationHandler.update - full update started to: " + server);
187 184
        // Send command to that server to get updated docid information
188 185
        try
......
190 187
          u = new URL("https://" + server + "?server="
191 188
          +MetacatUtil.getLocalReplicationServerName()+"&action=update");
192 189
          logReplication.info("ReplicationHandler.update - Sending infomation " +u.toString());
193
          result = ReplicationService.getURLContent(u);
190
          result = ReplicationService.getURLStream(u);
194 191
        }
195 192
        catch (Exception e)
196 193
        {
......
202 199

  
203 200
        //logReplication.info("ReplicationHandler.update - docid: "+server+" "+result);
204 201
        //check if result have error or not, if has skip it.
205
        if (result.indexOf("<error>")!=-1 && result.indexOf("</error>")!=-1)
206
        {
202
        // TODO: check for error in stream
203
        //if (result.indexOf("<error>") != -1 && result.indexOf("</error>") != -1) {
204
        if (result == null) {
207 205
          logMetacat.error("ReplicationHandler.update - " + ReplicationService.METACAT_REPL_ERROR_MSG);
208 206
          logReplication.error( "ReplicationHandler.update - Failed to get updated doc list "+
209 207
                       "for server " + server + " because "+result);
......
250 248
    	
251 249
        try
252 250
        {
253
          parser.parse(new InputSource(
254
                     new StringReader(
255
                     (String)(responses.elementAt(i)))));
251
          parser.parse(new InputSource(responses.elementAt(i)));
256 252
        }
257 253
        catch(Exception e)
258 254
        {
src/edu/ucsb/nceas/metacat/replication/ReplicationService.java
120 120

  
121 121
	public static final String REPLICATION_LOG_FILE_NAME = "metacatreplication.log";
122 122

  
123
	private static Object DATA_FILE_FLAG = null;
123
	private static String DATA_FILE_FLAG = null;
124 124
	public static String METACAT_REPL_ERROR_MSG = null;
125 125
	private static Logger logReplication = Logger.getLogger("ReplicationLogging");
126 126
	private static Logger logMetacat = Logger.getLogger(ReplicationService.class);
......
1395 1395
			// Store the sql command
1396 1396
			StringBuffer docsql = new StringBuffer();
1397 1397
			StringBuffer revisionSql = new StringBuffer();
1398
			// Stroe the docid list
1399
			StringBuffer doclist = new StringBuffer();
1398
			
1400 1399
			// Store the deleted docid list
1401 1400
			StringBuffer delsql = new StringBuffer();
1402 1401
			// Store the data set file
1403 1402
			Vector<Vector<String>> packageFiles = new Vector<Vector<String>>();
1404 1403

  
1405 1404
			// Append local server's name and replication servlet to doclist
1406
			doclist.append("<?xml version=\"1.0\"?><replication>");
1407
			doclist.append("<server>")
1405
			out.append("<?xml version=\"1.0\"?><replication>");
1406
			out.append("<server>")
1408 1407
					.append(MetacatUtil.getLocalReplicationServerName());
1409 1408
			//doclist.append(util.getProperty("replicationpath"));
1410
			doclist.append("</server><updates>");
1409
			out.append("</server><updates>");
1411 1410

  
1412 1411
			// Get correct docid that reside on this server according the requesting
1413 1412
			// server's replicate and data replicate value in xml_replication table
......
1456 1455
						//If this is not data file
1457 1456
						if (!recordDoctype.equals("BIN")) {
1458 1457
							//for non-data file document
1459
							doclist.append("<updatedDocument>");
1460
							doclist.append("<docid>").append(rs.getString(1));
1461
							doclist.append("</docid><rev>").append(rs.getInt(2));
1462
							doclist.append("</rev>");
1463
							doclist.append("</updatedDocument>");
1458
							out.append("<updatedDocument>");
1459
							out.append("<docid>").append(rs.getString(1));
1460
							out.append("</docid><rev>" + rs.getInt(2));
1461
							out.append("</rev>");
1462
							out.append("</updatedDocument>");
1464 1463
						}//if
1465 1464
						else {
1466 1465
							//for data file document, in datafile attributes
1467 1466
							//we put "datafile" value there
1468
							doclist.append("<updatedDocument>");
1469
							doclist.append("<docid>").append(rs.getString(1));
1470
							doclist.append("</docid><rev>").append(rs.getInt(2));
1471
							doclist.append("</rev>");
1472
							doclist.append("<datafile>");
1473
							doclist.append(DATA_FILE_FLAG);
1474
							doclist.append("</datafile>");
1475
							doclist.append("</updatedDocument>");
1467
							out.append("<updatedDocument>");
1468
							out.append("<docid>").append(rs.getString(1));
1469
							out.append("</docid><rev>" + rs.getInt(2));
1470
							out.append("</rev>");
1471
							out.append("<datafile>");
1472
							out.append(DATA_FILE_FLAG);
1473
							out.append("</datafile>");
1474
							out.append("</updatedDocument>");
1476 1475
						}//else
1477 1476
					}//if packagedoctpes
1478 1477
					else { //the package files are saved to be put into the xml later.
......
1496 1495
								&& !packagedoctypes.contains(recordDoctype)) { //if this is a package file, put it at the end
1497 1496
							//because if a package file is read before all of the files it
1498 1497
							//refers to are loaded then there is an error
1499
							doclist.append("<updatedDocument>");
1500
							doclist.append("<docid>").append(rs.getString(1));
1501
							doclist.append("</docid><rev>").append(rs.getInt(2));
1502
							doclist.append("</rev>");
1503
							doclist.append("</updatedDocument>");
1498
							out.append("<updatedDocument>");
1499
							out.append("<docid>" + rs.getString(1));
1500
							out.append("</docid><rev>" + rs.getInt(2));
1501
							out.append("</rev>");
1502
							out.append("</updatedDocument>");
1504 1503
						} else { //the package files are saved to be put into the xml later.
1505 1504
							Vector<String> v = new Vector<String>();
1506 1505
							v.add(rs.getString(1));
......
1520 1519
			rs = pstmt.getResultSet();
1521 1520
			tablehasrows = rs.next();
1522 1521
			while (tablehasrows) { //handle the deleted documents
1523
				doclist.append("<deletedDocument><docid>").append(rs.getString(1));
1524
				doclist.append("</docid><rev></rev></deletedDocument>");
1522
				out.append("<deletedDocument><docid>").append(rs.getString(1));
1523
				out.append("</docid><rev></rev></deletedDocument>");
1525 1524
				//note that rev is always empty for deleted docs
1526 1525
				tablehasrows = rs.next();
1527 1526
			}
......
1529 1528
			//now we can put the package files into the xml results
1530 1529
			for (int i = 0; i < packageFiles.size(); i++) {
1531 1530
				Vector<String> v = packageFiles.elementAt(i);
1532
				doclist.append("<updatedDocument>");
1533
				doclist.append("<docid>").append(v.elementAt(0));
1534
				doclist.append("</docid><rev>");
1535
				doclist.append(v.elementAt(1));
1536
				doclist.append("</rev>");
1537
				doclist.append("</updatedDocument>");
1531
				out.append("<updatedDocument>");
1532
				out.append("<docid>").append(v.elementAt(0));
1533
				out.append("</docid><rev>");
1534
				out.append(v.elementAt(1));
1535
				out.append("</rev>");
1536
				out.append("</updatedDocument>");
1538 1537
			}
1539 1538
			// add revision doc list  
1540
			doclist.append(prepareRevisionDoc(dbConn, revisionSql.toString(),
1539
			out.append(prepareRevisionDoc(dbConn, revisionSql.toString(),
1541 1540
					replicateData));
1542 1541

  
1543
			doclist.append("</updates></replication>");
1544
			logReplication.info("ReplicationService.handleUpdateRequest - doclist: " + doclist.toString());
1542
			out.append("</updates></replication>");
1543
			logReplication.info("ReplicationService.handleUpdateRequest - done writing to output stream.");
1545 1544
			pstmt.close();
1546 1545
			//conn.close();
1547
			out.write(doclist.toString());
1548 1546

  
1549 1547
		} catch (Exception e) {
1550 1548
			logMetacat.error("ReplicationService.handleUpdateRequest - " + ReplicationService.METACAT_REPL_ERROR_MSG);                         

Also available in: Unified diff