Revision 6814
Added by ben leinfelder almost 13 years ago
src/edu/ucsb/nceas/metacat/replication/ForceReplicationSystemMetadataHandler.java | ||
---|---|---|
1 |
/** |
|
2 |
* '$RCSfile$' |
|
3 |
* Purpose: A class to asyncronously force the replication of each server |
|
4 |
* that has an entry in the xml_replication table. When run, |
|
5 |
* this thread communicates with each server in the list and |
|
6 |
* solicites a read of an updated or newly inserted document |
|
7 |
* with a certain docid. |
|
8 |
* Copyright: 2000 Regents of the University of California and the |
|
9 |
* National Center for Ecological Analysis and Synthesis |
|
10 |
* Authors: Chad Berkley |
|
11 |
* |
|
12 |
* '$Author: daigle $' |
|
13 |
* '$Date: 2009-08-24 13:34:17 -0800 (Mon, 24 Aug 2009) $' |
|
14 |
* '$Revision: 5030 $' |
|
15 |
* |
|
16 |
* This program is free software; you can redistribute it and/or modify |
|
17 |
* it under the terms of the GNU General Public License as published by |
|
18 |
* the Free Software Foundation; either version 2 of the License, or |
|
19 |
* (at your option) any later version. |
|
20 |
* |
|
21 |
* This program is distributed in the hope that it will be useful, |
|
22 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
23 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
24 |
* GNU General Public License for more details. |
|
25 |
* |
|
26 |
* You should have received a copy of the GNU General Public License |
|
27 |
* along with this program; if not, write to the Free Software |
|
28 |
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
|
29 |
*/ |
|
30 |
|
|
31 |
package edu.ucsb.nceas.metacat.replication; |
|
32 |
|
|
33 |
import java.net.URL; |
|
34 |
|
|
35 |
import org.apache.log4j.Logger; |
|
36 |
|
|
37 |
import edu.ucsb.nceas.metacat.properties.PropertyService; |
|
38 |
import edu.ucsb.nceas.metacat.util.MetacatUtil; |
|
39 |
import edu.ucsb.nceas.utilities.PropertyNotFoundException; |
|
40 |
|
|
41 |
/** |
|
42 |
* A class to asyncronously force the replication of each server |
|
43 |
* that has an entry in the xml_replication table. When run, |
|
44 |
* this thread communicates with each server in the list and |
|
45 |
* forces replication of system metadata |
|
46 |
*/ |
|
47 |
public class ForceReplicationSystemMetadataHandler implements Runnable |
|
48 |
{ |
|
49 |
private Thread btThread; |
|
50 |
private String guid; |
|
51 |
|
|
52 |
private ReplicationServerList serverLists = null; |
|
53 |
|
|
54 |
private String notificationServer = null; |
|
55 |
private static Logger logReplication = Logger.getLogger("ReplicationLogging"); |
|
56 |
private static Logger logMetacat = Logger.getLogger(ForceReplicationSystemMetadataHandler.class); |
|
57 |
|
|
58 |
/** |
|
59 |
* Use this constructor when the action is implied. |
|
60 |
*/ |
|
61 |
public ForceReplicationSystemMetadataHandler(String guid, String myNotificationServer ) |
|
62 |
{ |
|
63 |
this.guid = guid; |
|
64 |
// Build a severLists from xml_replication table |
|
65 |
this.serverLists = new ReplicationServerList(); |
|
66 |
|
|
67 |
// Get notification server |
|
68 |
this.notificationServer = myNotificationServer; |
|
69 |
btThread = new Thread(this); |
|
70 |
btThread.setPriority(Thread.MIN_PRIORITY); |
|
71 |
btThread.start(); |
|
72 |
} |
|
73 |
|
|
74 |
/** |
|
75 |
* Method to send force replication command to other server to get |
|
76 |
* a new or updated docid |
|
77 |
*/ |
|
78 |
public void run() |
|
79 |
{ |
|
80 |
|
|
81 |
|
|
82 |
// URL for notification |
|
83 |
URL comeAndGetIt = null; |
|
84 |
// If no server in xml_replication table we are done |
|
85 |
if (serverLists.isEmpty()) { |
|
86 |
return; |
|
87 |
} |
|
88 |
|
|
89 |
// Thread sleeping for some seconds to make sure the document was insert |
|
90 |
// into the database before we send force replication request |
|
91 |
int sleepTime; |
|
92 |
try { |
|
93 |
sleepTime = Integer.parseInt(PropertyService.getProperty("replication.forcereplicationwaitingtime")); |
|
94 |
Thread.sleep(sleepTime); |
|
95 |
} catch (PropertyNotFoundException pnfe) { |
|
96 |
logMetacat.error(ReplicationService.METACAT_REPL_ERROR_MSG); |
|
97 |
logReplication.error("Property error: " + pnfe.getMessage()); |
|
98 |
} catch (InterruptedException ie) { |
|
99 |
logMetacat.error(ReplicationService.METACAT_REPL_ERROR_MSG); |
|
100 |
logReplication.error("Thread sleep error: " + ie.getMessage()); |
|
101 |
} |
|
102 |
logReplication.info("Notification server:" + notificationServer); |
|
103 |
// Check every server in the serverlists |
|
104 |
for (int i = 0; i < serverLists.size(); i++) { |
|
105 |
//Set comeAndGetIt null |
|
106 |
comeAndGetIt = null; |
|
107 |
// Get ReplicationServer object in index i |
|
108 |
ReplicationServer replicationServer = serverLists.serverAt(i); |
|
109 |
// Get this ReplicationServer 's name |
|
110 |
String server = replicationServer.getServerName(); |
|
111 |
try { |
|
112 |
|
|
113 |
// If the server is the notification server, we don't notify it back |
|
114 |
// again, if server is null we don't replicate to |
|
115 |
if (server != null && !server.equals(notificationServer)) { |
|
116 |
if (replicationServer.isSystemMetadataReplication()) { |
|
117 |
comeAndGetIt = new URL("https://" + server |
|
118 |
+ "?action=forcereplicatesystemmetadata" |
|
119 |
+ "&guid=" + guid + "&server=" |
|
120 |
+ MetacatUtil.getLocalReplicationServerName()); |
|
121 |
|
|
122 |
logReplication.warn("Sending message: " + comeAndGetIt.toString()); |
|
123 |
String message = ReplicationService.getURLContent(comeAndGetIt); |
|
124 |
} |
|
125 |
} |
|
126 |
} catch (Exception e) { |
|
127 |
logMetacat.error(ReplicationService.METACAT_REPL_ERROR_MSG, e); |
|
128 |
logReplication.error("Error forcing System Metadata replication"); |
|
129 |
} |
|
130 |
|
|
131 |
} |
|
132 |
|
|
133 |
logReplication.warn("Exiting Forced System Metadata Thread"); |
|
134 |
} |
|
135 |
} |
src/upgrade-db-to-2.0.0-oracle.sql | ||
---|---|---|
45 | 45 |
rev NUMBER(8) -- the revision part of the local identifier |
46 | 46 |
); |
47 | 47 |
|
48 |
/* |
|
49 |
* Replication changes to support DataONE System Metadata replication |
|
50 |
*/ |
|
51 |
ALTER TABLE xml_replication ADD COLUMN systemmetadatareplicate NUMBER(1); |
|
52 |
|
|
53 | 48 |
/** |
54 | 49 |
* track the user-agent for the request |
55 | 50 |
*/ |
src/xmltables-oracle.sql | ||
---|---|---|
80 | 80 |
last_checked DATE, |
81 | 81 |
replicate NUMBER(1), |
82 | 82 |
datareplicate NUMBER(1), |
83 |
systemmetadatareplicate NUMBER(1), |
|
84 | 83 |
hub NUMBER(1), |
85 | 84 |
CONSTRAINT xml_replication_pk PRIMARY KEY (serverid) |
86 | 85 |
); |
... | ... | |
95 | 94 |
END; |
96 | 95 |
/ |
97 | 96 |
|
98 |
INSERT INTO xml_replication (server, replicate, datareplicate, systemmetadatareplicate, hub)
|
|
99 |
VALUES ('localhost', '0', '0', '0', '0');
|
|
97 |
INSERT INTO xml_replication (server, replicate, datareplicate, hub) |
|
98 |
VALUES ('localhost', '0', '0', '0'); |
|
100 | 99 |
|
101 | 100 |
/* |
102 | 101 |
* Nodes -- table to store XML Nodes (both elements and attributes) |
src/upgrade-db-to-2.0.0-postgres.sql | ||
---|---|---|
58 | 58 |
ALTER TABLE xml_path_index ADD COLUMN nodedatadate TIMESTAMP; |
59 | 59 |
CREATE INDEX xml_path_index_idx5 ON xml_path_index (nodedatadate); |
60 | 60 |
|
61 |
/* |
|
62 |
* Replication changes to support DataONE System Metadata replication |
|
63 |
*/ |
|
64 |
ALTER TABLE xml_replication ADD COLUMN systemmetadatareplicate INT8; |
|
65 |
|
|
66 | 61 |
/** |
67 | 62 |
* track the user-agent for the request |
68 | 63 |
*/ |
src/xmltables-postgres.sql | ||
---|---|---|
39 | 39 |
last_checked DATE, |
40 | 40 |
replicate INT8, |
41 | 41 |
datareplicate INT8, |
42 |
systemmetadatareplicate INT8, |
|
43 | 42 |
hub INT8, |
44 | 43 |
CONSTRAINT xml_replication_pk PRIMARY KEY (serverid) |
45 | 44 |
); |
46 | 45 |
|
47 |
INSERT INTO xml_replication (server, replicate, datareplicate, systemmetadatareplicate, hub) VALUES ('localhost', '0', '0', '0', '0'); |
|
48 |
/* |
|
49 |
* ALTER TABLE xml_replication ADD COLUMN systemmetadatareplicate INT8; |
|
50 |
*/ |
|
46 |
INSERT INTO xml_replication (server, replicate, datareplicate, hub) VALUES ('localhost', '0', '0', '0'); |
|
51 | 47 |
|
48 |
|
|
52 | 49 |
/* |
53 | 50 |
* Nodes -- table to store XML Nodes (both elements and attributes) |
54 | 51 |
*/ |
src/edu/ucsb/nceas/metacat/replication/ReplicationServer.java | ||
---|---|---|
42 | 42 |
//it is relative to replcation |
43 | 43 |
//if replication is false, it should |
44 | 44 |
//be false |
45 |
private boolean systemMetadataReplication = false; //replicate system metadata or not |
|
46 | 45 |
private boolean hub = false; //it is hub or not. Hub means the localhost can |
47 | 46 |
//replcate documents to the server if the |
48 | 47 |
//document's home server is not localhost |
... | ... | |
126 | 125 |
this.dataReplication = myDataReplication; |
127 | 126 |
}//setDataReplication |
128 | 127 |
|
129 |
public boolean isSystemMetadataReplication() { |
|
130 |
return systemMetadataReplication; |
|
131 |
} |
|
132 | 128 |
|
133 |
public void setSystemMetadataReplication(boolean systemMetadataReplication) { |
|
134 |
this.systemMetadataReplication = systemMetadataReplication; |
|
135 |
} |
|
136 |
|
|
137 | 129 |
/** |
138 | 130 |
* Get hub option |
139 | 131 |
*/ |
src/edu/ucsb/nceas/metacat/replication/ReplicationService.java | ||
---|---|---|
299 | 299 |
String replicate = null; |
300 | 300 |
String server = null; |
301 | 301 |
String dataReplicate = null; |
302 |
String systemMetadataReplicate = null; |
|
303 | 302 |
String hub = null; |
304 | 303 |
Writer out = null; |
305 | 304 |
try { |
... | ... | |
318 | 317 |
|
319 | 318 |
//Get data replication value |
320 | 319 |
dataReplicate = ((String[]) params.get("datareplicate"))[0]; |
321 |
//Get system metadata replication value |
|
322 |
systemMetadataReplicate = ((String[]) params.get("systemmetadatareplicate"))[0]; |
|
320 |
|
|
323 | 321 |
//Get hub value |
324 | 322 |
hub = ((String[]) params.get("hub"))[0]; |
325 | 323 |
|
326 | 324 |
Calendar cal = Calendar.getInstance(); |
327 | 325 |
cal.set(1980, 1, 1); |
328 | 326 |
String sql = "INSERT INTO xml_replication " |
329 |
+ "(server, last_checked, replicate, datareplicate, systemmetadatareplicate, hub) "
|
|
330 |
+ "VALUES (?,?,?,?,?,?)";
|
|
327 |
+ "(server, last_checked, replicate, datareplicate, hub) " |
|
328 |
+ "VALUES (?,?,?,?,?)"; |
|
331 | 329 |
|
332 | 330 |
pstmt = dbConn.prepareStatement(sql); |
333 | 331 |
|
... | ... | |
335 | 333 |
pstmt.setTimestamp(2, new Timestamp(cal.getTimeInMillis())); |
336 | 334 |
pstmt.setInt(3, Integer.parseInt(replicate)); |
337 | 335 |
pstmt.setInt(4, Integer.parseInt(dataReplicate)); |
338 |
pstmt.setInt(5, Integer.parseInt(systemMetadataReplicate)); |
|
339 |
pstmt.setInt(6, Integer.parseInt(hub)); |
|
336 |
pstmt.setInt(5, Integer.parseInt(hub)); |
|
340 | 337 |
|
341 | 338 |
String sqlReport = "XMLAccessAccess.getXMLAccessForDoc - SQL: " + sql; |
342 | 339 |
sqlReport += " [" + server + "," + replicate + |
... | ... | |
354 | 351 |
out.write("<td><b>last_checked</b></td>"); |
355 | 352 |
out.write("<td><b>replicate</b></td>"); |
356 | 353 |
out.write("<td><b>datareplicate</b></td>"); |
357 |
out.write("<td><b>systemmetadatareplicate</b></td>"); |
|
358 | 354 |
out.write("<td><b>hub</b></td></tr>"); |
359 |
pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, systemmetadatareplicate, hub FROM xml_replication");
|
|
355 |
pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, hub FROM xml_replication"); |
|
360 | 356 |
//increase dbconnection usage |
361 | 357 |
dbConn.increaseUsageCount(1); |
362 | 358 |
|
... | ... | |
368 | 364 |
out.write(rs.getString(3) + "</td><td>"); |
369 | 365 |
out.write(rs.getString(4) + "</td><td>"); |
370 | 366 |
out.write(rs.getString(5) + "</td><td>"); |
371 |
out.write(rs.getString(6) + "</td><td>"); |
|
372 |
out.write(rs.getString(7) + "</td></tr>"); |
|
367 |
out.write(rs.getString(6) + "</td></tr>"); |
|
373 | 368 |
|
374 | 369 |
tablehasrows = rs.next(); |
375 | 370 |
} |
... | ... | |
391 | 386 |
out.write("<td><b>last_checked</b></td>"); |
392 | 387 |
out.write("<td><b>replicate</b></td>"); |
393 | 388 |
out.write("<td><b>datareplicate</b></td>"); |
394 |
out.write("<td><b>systemmetadatareplicate</b></td>"); |
|
395 | 389 |
out.write("<td><b>hub</b></td></tr>"); |
396 | 390 |
|
397 |
pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, systemmetadatareplicate, hub FROM xml_replication");
|
|
391 |
pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, hub FROM xml_replication"); |
|
398 | 392 |
//increase dbconnection usage |
399 | 393 |
dbConn.increaseUsageCount(1); |
400 | 394 |
pstmt.execute(); |
... | ... | |
405 | 399 |
out.write(rs.getString(3) + "</td><td>"); |
406 | 400 |
out.write(rs.getString(4) + "</td><td>"); |
407 | 401 |
out.write(rs.getString(5) + "</td><td>"); |
408 |
out.write(rs.getString(6) + "</td><td>"); |
|
409 |
out.write(rs.getString(7) + "</td></tr>"); |
|
402 |
out.write(rs.getString(6) + "</td></tr>"); |
|
410 | 403 |
tablehasrows = rs.next(); |
411 | 404 |
} |
412 | 405 |
out.write("</table></body></html>"); |
... | ... | |
419 | 412 |
out.write("<td><b>last_checked</b></td>"); |
420 | 413 |
out.write("<td><b>replicate</b></td>"); |
421 | 414 |
out.write("<td><b>datareplicate</b></td>"); |
422 |
out.write("<td><b>systemmetadatareplicate</b></td>"); |
|
423 | 415 |
out.write("<td><b>hub</b></td></tr>"); |
424 |
pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, systemmetadatareplicate, hub FROM xml_replication");
|
|
416 |
pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, hub FROM xml_replication"); |
|
425 | 417 |
pstmt.execute(); |
426 | 418 |
ResultSet rs = pstmt.getResultSet(); |
427 | 419 |
boolean tablehasrows = rs.next(); |
... | ... | |
430 | 422 |
out.write(rs.getString(3) + "</td><td>"); |
431 | 423 |
out.write(rs.getString(4) + "</td><td>"); |
432 | 424 |
out.write(rs.getString(5) + "</td><td>"); |
433 |
out.write(rs.getString(6) + "</td><td>"); |
|
434 |
out.write(rs.getString(7) + "</td></tr>"); |
|
425 |
out.write(rs.getString(6) + "</td></tr>"); |
|
435 | 426 |
|
436 | 427 |
tablehasrows = rs.next(); |
437 | 428 |
} |
... | ... | |
1509 | 1500 |
// add revision doc list |
1510 | 1501 |
doclist.append(prepareRevisionDoc(dbConn, revisionSql.toString(), |
1511 | 1502 |
replicateData)); |
1512 |
|
|
1513 |
// add the system metadata entries if configured to replicate them |
|
1514 |
boolean replicateSystemMetadata = serverList.getSystemMetadataReplicationValue(server); |
|
1515 |
if (replicateSystemMetadata) { |
|
1516 |
Date since = new Date(System.currentTimeMillis()); |
|
1517 |
since = serverList.getLastCheckedDate(server); |
|
1518 |
List<String> systemMetadataEntries = |
|
1519 |
IdentifierManager.getInstance().getUpdatedSystemMetadataIds(since); |
|
1520 |
for (int i = 0; i < systemMetadataEntries.size(); i++) { |
|
1521 |
String guid = systemMetadataEntries.get(i); |
|
1522 |
doclist.append("<updatedSystemMetadata>"); |
|
1523 |
doclist.append("<guid>"); |
|
1524 |
doclist.append(guid); |
|
1525 |
doclist.append("</guid>"); |
|
1526 |
doclist.append("</updatedSystemMetadata>"); |
|
1527 |
} |
|
1528 |
} |
|
1529 | 1503 |
|
1530 | 1504 |
doclist.append("</updates></replication>"); |
1531 | 1505 |
logReplication.info("ReplicationService.handleUpdateRequest - doclist: " + doclist.toString()); |
src/edu/ucsb/nceas/metacat/replication/ReplicationServerList.java | ||
---|---|---|
88 | 88 |
serialNumber=dbConn.getCheckOutSerialNumber(); |
89 | 89 |
//Select fields from xml_replication table |
90 | 90 |
pstmt = dbConn.prepareStatement("select server, last_checked, replicate,"+ |
91 |
" datareplicate, systemmetadatareplicate, hub from xml_replication");
|
|
91 |
" datareplicate, hub from xml_replication"); |
|
92 | 92 |
//Execute prepare statement |
93 | 93 |
pstmt.execute(); |
94 | 94 |
//Get result set |
... | ... | |
109 | 109 |
//Data replication value |
110 | 110 |
int dataReplication = rs.getInt(4); |
111 | 111 |
logMetacat.info("DataReplication value: "+dataReplication); |
112 |
//Data replication value |
|
113 |
int systemMetadataReplication = rs.getInt(5); |
|
114 |
logMetacat.info("DataReplication value: "+dataReplication); |
|
115 | 112 |
//Hub value |
116 |
int hubValue = rs.getInt(6);
|
|
113 |
int hubValue = rs.getInt(5);
|
|
117 | 114 |
logMetacat.info("Hub value: "+hubValue); |
118 | 115 |
//Get rid of local host |
119 | 116 |
if(!serverName.equals("localhost")) |
... | ... | |
155 | 152 |
}//else |
156 | 153 |
//Set data replciation value |
157 | 154 |
server.setDataReplication(hasDataReplication); |
158 |
if (systemMetadataReplication == 1) { |
|
159 |
hasSystemMetadataReplication = true; |
|
160 |
} |
|
161 |
server.setSystemMetadataReplication(hasSystemMetadataReplication); |
|
162 | 155 |
|
163 | 156 |
//Detemine isHub by hubValue |
164 | 157 |
if (hubValue ==1) |
... | ... | |
392 | 385 |
return server.getDataReplication(); |
393 | 386 |
}//getDataReplicationValue |
394 | 387 |
|
395 |
/** |
|
396 |
* To a given server name, try to get its data file replciation option |
|
397 |
* If couldn't find the server in the server list, false will return |
|
398 |
* @param givenServerName, the server's name which we want to get data file |
|
399 |
* replication value |
|
400 |
*/ |
|
401 |
public synchronized boolean getSystemMetadataReplicationValue(String givenServerName) |
|
402 |
{ |
|
403 |
int index = -1;//Variable to store the index |
|
404 |
ReplicationServer server = null;//Variable for replication server |
|
405 |
|
|
406 |
//Get the server's index in server list |
|
407 |
index = findIndexInServerList(givenServerName); |
|
408 |
//If index = -1, couldn't find this server, null will return |
|
409 |
if (index == -1) |
|
410 |
{ |
|
411 |
return false; |
|
412 |
}//if |
|
413 |
//Get Replication server object |
|
414 |
server = (ReplicationServer) serverList.elementAt(index); |
|
415 |
//Return this object's data replication value |
|
416 |
return server.isSystemMetadataReplication(); |
|
417 |
} |
|
418 | 388 |
|
419 | 389 |
/** |
420 | 390 |
* To a given server name, try to get its hub option |
Also available in: Unified diff
remove flag for independent system metadata replication -- these entries are replicated along with the data/metadata objects or via hazelcast when the actual object is not on the server.