Project

General

Profile

« Previous | Next » 

Revision 6814

remove flag for independent system metadata replication -- these entries are replicated along with the data/metadata objects or via hazelcast when the actual object is not on the server.

View differences:

src/edu/ucsb/nceas/metacat/replication/ForceReplicationSystemMetadataHandler.java
1
/**
2
 *  '$RCSfile$'
3
 *    Purpose: A class to asyncronously force the replication of each server
4
 *             that has an entry in the xml_replication table.  When run,
5
 *             this thread communicates with each server in the list and
6
 *             solicites a read of an updated or newly inserted document
7
 *             with a certain docid.
8
 *  Copyright: 2000 Regents of the University of California and the
9
 *             National Center for Ecological Analysis and Synthesis
10
 *    Authors: Chad Berkley
11
 *
12
 *   '$Author: daigle $'
13
 *     '$Date: 2009-08-24 13:34:17 -0800 (Mon, 24 Aug 2009) $'
14
 * '$Revision: 5030 $'
15
 *
16
 * This program is free software; you can redistribute it and/or modify
17
 * it under the terms of the GNU General Public License as published by
18
 * the Free Software Foundation; either version 2 of the License, or
19
 * (at your option) any later version.
20
 *
21
 * This program is distributed in the hope that it will be useful,
22
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
23
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
24
 * GNU General Public License for more details.
25
 *
26
 * You should have received a copy of the GNU General Public License
27
 * along with this program; if not, write to the Free Software
28
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
29
 */
30

  
31
package edu.ucsb.nceas.metacat.replication;
32

  
33
import java.net.URL;
34

  
35
import org.apache.log4j.Logger;
36

  
37
import edu.ucsb.nceas.metacat.properties.PropertyService;
38
import edu.ucsb.nceas.metacat.util.MetacatUtil;
39
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
40

  
41
/**
42
 * A class to asyncronously force the replication of each server
43
 * that has an entry in the xml_replication table.  When run,
44
 * this thread communicates with each server in the list and
45
 * forces replication of system metadata
46
 */
47
public class ForceReplicationSystemMetadataHandler implements Runnable
48
{
49
  private Thread btThread;
50
  private String guid;
51
    
52
  private ReplicationServerList serverLists = null;
53

  
54
  private String notificationServer = null;
55
  private static Logger logReplication = Logger.getLogger("ReplicationLogging");
56
  private static Logger logMetacat = Logger.getLogger(ForceReplicationSystemMetadataHandler.class);
57
 
58
  /**
59
   * Use this constructor when the action is implied.
60
   */
61
  public ForceReplicationSystemMetadataHandler(String guid, String myNotificationServer )
62
  {
63
    this.guid = guid;
64
    // Build a severLists from xml_replication table
65
    this.serverLists = new ReplicationServerList();
66
   
67
    // Get notification server
68
    this.notificationServer = myNotificationServer;
69
    btThread = new Thread(this);
70
    btThread.setPriority(Thread.MIN_PRIORITY);
71
    btThread.start();
72
  }
73

  
74
  /**
75
   * Method to send force replication command to other server to get
76
   * a new or updated docid
77
   */
78
  public void run()
79
  {
80

  
81
    
82
	    // URL for notification
83
	    URL comeAndGetIt = null;
84
		// If no server in xml_replication table we are done
85
		if (serverLists.isEmpty()) {
86
			return;
87
		}
88

  
89
		// Thread sleeping for some seconds to make sure the document was insert
90
		// into the database before we send force replication request
91
		int sleepTime;
92
		try {
93
			sleepTime = Integer.parseInt(PropertyService.getProperty("replication.forcereplicationwaitingtime"));
94
			Thread.sleep(sleepTime);
95
		} catch (PropertyNotFoundException pnfe) {
96
	    	logMetacat.error(ReplicationService.METACAT_REPL_ERROR_MSG);
97
			logReplication.error("Property error: " + pnfe.getMessage());
98
		} catch (InterruptedException ie) {
99
	    	logMetacat.error(ReplicationService.METACAT_REPL_ERROR_MSG);
100
			logReplication.error("Thread sleep error: " + ie.getMessage());
101
		}
102
		logReplication.info("Notification server:" + notificationServer);
103
		// Check every server in the serverlists
104
		for (int i = 0; i < serverLists.size(); i++) {
105
			//Set comeAndGetIt null
106
			comeAndGetIt = null;
107
			// Get ReplicationServer object in index i
108
			ReplicationServer replicationServer = serverLists.serverAt(i);
109
			// Get this ReplicationServer 's name
110
			String server = replicationServer.getServerName();
111
			try {
112

  
113
				// If the server is the notification server, we don't notify it back
114
				// again, if server is null we don't replicate to
115
				if (server != null && !server.equals(notificationServer)) {
116
					if (replicationServer.isSystemMetadataReplication()) {
117
						comeAndGetIt = new URL("https://" + server
118
								+ "?action=forcereplicatesystemmetadata"
119
								+ "&guid=" + guid + "&server="
120
								+ MetacatUtil.getLocalReplicationServerName());
121

  
122
						logReplication.warn("Sending message: " + comeAndGetIt.toString());
123
						String message = ReplicationService.getURLContent(comeAndGetIt);
124
					}
125
				}
126
			} catch (Exception e) {
127
		    	logMetacat.error(ReplicationService.METACAT_REPL_ERROR_MSG, e);
128
				logReplication.error("Error forcing System Metadata replication");
129
			}
130
			
131
		}
132

  
133
		logReplication.warn("Exiting Forced System Metadata Thread");
134
	}
135
}
src/upgrade-db-to-2.0.0-oracle.sql
45 45
   rev    NUMBER(8)       -- the revision part of the local identifier
46 46
);
47 47

  
48
/*
49
 * Replication changes to support DataONE System Metadata replication
50
 */
51
ALTER TABLE xml_replication ADD COLUMN systemmetadatareplicate NUMBER(1);
52

  
53 48
/**
54 49
 * track the user-agent for the request
55 50
 */
src/xmltables-oracle.sql
80 80
  last_checked  DATE,
81 81
  replicate     NUMBER(1),
82 82
  datareplicate NUMBER(1),
83
  systemmetadatareplicate NUMBER(1),
84 83
  hub NUMBER(1),
85 84
  CONSTRAINT xml_replication_pk PRIMARY KEY (serverid)
86 85
);
......
95 94
END;
96 95
/
97 96

  
98
INSERT INTO xml_replication (server, replicate, datareplicate, systemmetadatareplicate, hub)
99
 VALUES ('localhost', '0', '0', '0', '0');
97
INSERT INTO xml_replication (server, replicate, datareplicate, hub)
98
 VALUES ('localhost', '0', '0', '0');
100 99

  
101 100
/*
102 101
 * Nodes -- table to store XML Nodes (both elements and attributes)
src/upgrade-db-to-2.0.0-postgres.sql
58 58
ALTER TABLE xml_path_index ADD COLUMN nodedatadate TIMESTAMP;
59 59
CREATE INDEX xml_path_index_idx5 ON xml_path_index (nodedatadate);
60 60

  
61
/*
62
 * Replication changes to support DataONE System Metadata replication
63
 */
64
ALTER TABLE xml_replication ADD COLUMN systemmetadatareplicate INT8;
65

  
66 61
/**
67 62
 * track the user-agent for the request
68 63
 */
src/xmltables-postgres.sql
39 39
  last_checked DATE,
40 40
  replicate INT8,
41 41
  datareplicate INT8,
42
  systemmetadatareplicate INT8,
43 42
  hub INT8,
44 43
  CONSTRAINT xml_replication_pk PRIMARY KEY (serverid)
45 44
);
46 45

  
47
INSERT INTO xml_replication (server, replicate, datareplicate, systemmetadatareplicate, hub) VALUES ('localhost', '0', '0', '0', '0');
48
/*
49
 * ALTER TABLE xml_replication ADD COLUMN systemmetadatareplicate INT8;
50
 */
46
INSERT INTO xml_replication (server, replicate, datareplicate, hub) VALUES ('localhost', '0', '0', '0');
51 47

  
48

  
52 49
/*
53 50
 * Nodes -- table to store XML Nodes (both elements and attributes)
54 51
 */
src/edu/ucsb/nceas/metacat/replication/ReplicationServer.java
42 42
                                           //it is relative to replcation
43 43
                                           //if replication is false, it should
44 44
                                           //be false
45
  private boolean systemMetadataReplication = false; //replicate system metadata or not
46 45
  private boolean hub = false; //it is hub or not. Hub means the localhost can
47 46
                               //replcate documents to the server if the 
48 47
                               //document's home server is not localhost
......
126 125
    this.dataReplication = myDataReplication;
127 126
  }//setDataReplication   
128 127
   
129
  public boolean isSystemMetadataReplication() {
130
	return systemMetadataReplication;
131
}
132 128

  
133
public void setSystemMetadataReplication(boolean systemMetadataReplication) {
134
	this.systemMetadataReplication = systemMetadataReplication;
135
}
136

  
137 129
/**
138 130
   * Get hub option
139 131
   */
src/edu/ucsb/nceas/metacat/replication/ReplicationService.java
299 299
		String replicate = null;
300 300
		String server = null;
301 301
		String dataReplicate = null;
302
		String systemMetadataReplicate = null;
303 302
		String hub = null;
304 303
		Writer out = null;
305 304
		try {
......
318 317

  
319 318
				//Get data replication value
320 319
				dataReplicate = ((String[]) params.get("datareplicate"))[0];
321
				//Get system metadata replication value
322
				systemMetadataReplicate = ((String[]) params.get("systemmetadatareplicate"))[0];
320
				
323 321
				//Get hub value
324 322
				hub = ((String[]) params.get("hub"))[0];
325 323

  
326 324
				Calendar cal = Calendar.getInstance();
327 325
                cal.set(1980, 1, 1);
328 326
				String sql = "INSERT INTO xml_replication "
329
						+ "(server, last_checked, replicate, datareplicate, systemmetadatareplicate, hub) "
330
						+ "VALUES (?,?,?,?,?,?)";
327
						+ "(server, last_checked, replicate, datareplicate, hub) "
328
						+ "VALUES (?,?,?,?,?)";
331 329
				
332 330
				pstmt = dbConn.prepareStatement(sql);
333 331
						
......
335 333
				pstmt.setTimestamp(2, new Timestamp(cal.getTimeInMillis()));
336 334
				pstmt.setInt(3, Integer.parseInt(replicate));
337 335
				pstmt.setInt(4, Integer.parseInt(dataReplicate));
338
				pstmt.setInt(5, Integer.parseInt(systemMetadataReplicate));
339
				pstmt.setInt(6, Integer.parseInt(hub));
336
				pstmt.setInt(5, Integer.parseInt(hub));
340 337
				
341 338
				String sqlReport = "XMLAccessAccess.getXMLAccessForDoc - SQL: " + sql;
342 339
				sqlReport += " [" + server + "," + replicate + 
......
354 351
				out.write("<td><b>last_checked</b></td>");
355 352
				out.write("<td><b>replicate</b></td>");
356 353
				out.write("<td><b>datareplicate</b></td>");
357
				out.write("<td><b>systemmetadatareplicate</b></td>");
358 354
				out.write("<td><b>hub</b></td></tr>");
359
				pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, systemmetadatareplicate, hub FROM xml_replication");
355
				pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, hub FROM xml_replication");
360 356
				//increase dbconnection usage
361 357
				dbConn.increaseUsageCount(1);
362 358

  
......
368 364
					out.write(rs.getString(3) + "</td><td>");
369 365
					out.write(rs.getString(4) + "</td><td>");
370 366
					out.write(rs.getString(5) + "</td><td>");
371
					out.write(rs.getString(6) + "</td><td>");
372
					out.write(rs.getString(7) + "</td></tr>");
367
					out.write(rs.getString(6) + "</td></tr>");
373 368

  
374 369
					tablehasrows = rs.next();
375 370
				}
......
391 386
				out.write("<td><b>last_checked</b></td>");
392 387
				out.write("<td><b>replicate</b></td>");
393 388
				out.write("<td><b>datareplicate</b></td>");
394
				out.write("<td><b>systemmetadatareplicate</b></td>");
395 389
				out.write("<td><b>hub</b></td></tr>");
396 390

  
397
				pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, systemmetadatareplicate, hub FROM xml_replication");
391
				pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, hub FROM xml_replication");
398 392
				//increase dbconnection usage
399 393
				dbConn.increaseUsageCount(1);
400 394
				pstmt.execute();
......
405 399
					out.write(rs.getString(3) + "</td><td>");
406 400
					out.write(rs.getString(4) + "</td><td>");
407 401
					out.write(rs.getString(5) + "</td><td>");
408
					out.write(rs.getString(6) + "</td><td>");
409
					out.write(rs.getString(7) + "</td></tr>");
402
					out.write(rs.getString(6) + "</td></tr>");
410 403
					tablehasrows = rs.next();
411 404
				}
412 405
				out.write("</table></body></html>");
......
419 412
				out.write("<td><b>last_checked</b></td>");
420 413
				out.write("<td><b>replicate</b></td>");
421 414
				out.write("<td><b>datareplicate</b></td>");
422
				out.write("<td><b>systemmetadatareplicate</b></td>");
423 415
				out.write("<td><b>hub</b></td></tr>");
424
				pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, systemmetadatareplicate, hub FROM xml_replication");
416
				pstmt = dbConn.prepareStatement("SELECT serverid, server, last_checked, replicate, datareplicate, hub FROM xml_replication");
425 417
				pstmt.execute();
426 418
				ResultSet rs = pstmt.getResultSet();
427 419
				boolean tablehasrows = rs.next();
......
430 422
					out.write(rs.getString(3) + "</td><td>");
431 423
					out.write(rs.getString(4) + "</td><td>");
432 424
					out.write(rs.getString(5) + "</td><td>");
433
					out.write(rs.getString(6) + "</td><td>");
434
					out.write(rs.getString(7) + "</td></tr>");
425
					out.write(rs.getString(6) + "</td></tr>");
435 426

  
436 427
					tablehasrows = rs.next();
437 428
				}
......
1509 1500
			// add revision doc list  
1510 1501
			doclist.append(prepareRevisionDoc(dbConn, revisionSql.toString(),
1511 1502
					replicateData));
1512
			
1513
			// add the system metadata entries if configured to replicate them
1514
			boolean replicateSystemMetadata = serverList.getSystemMetadataReplicationValue(server);
1515
			if (replicateSystemMetadata) {
1516
				Date since = new Date(System.currentTimeMillis());
1517
				since = serverList.getLastCheckedDate(server);
1518
				List<String> systemMetadataEntries = 
1519
					IdentifierManager.getInstance().getUpdatedSystemMetadataIds(since);
1520
				for (int i = 0; i < systemMetadataEntries.size(); i++) {
1521
					String guid = systemMetadataEntries.get(i);
1522
					doclist.append("<updatedSystemMetadata>");
1523
					doclist.append("<guid>");
1524
					doclist.append(guid);
1525
					doclist.append("</guid>");
1526
					doclist.append("</updatedSystemMetadata>");
1527
				}
1528
			}
1529 1503

  
1530 1504
			doclist.append("</updates></replication>");
1531 1505
			logReplication.info("ReplicationService.handleUpdateRequest - doclist: " + doclist.toString());
src/edu/ucsb/nceas/metacat/replication/ReplicationServerList.java
88 88
      serialNumber=dbConn.getCheckOutSerialNumber();
89 89
      //Select fields from xml_replication table
90 90
      pstmt = dbConn.prepareStatement("select server, last_checked, replicate,"+
91
                    " datareplicate, systemmetadatareplicate, hub from xml_replication");
91
                    " datareplicate, hub from xml_replication");
92 92
      //Execute prepare statement
93 93
      pstmt.execute();
94 94
      //Get result set
......
109 109
        //Data replication value
110 110
        int dataReplication = rs.getInt(4);
111 111
        logMetacat.info("DataReplication value: "+dataReplication);
112
        //Data replication value
113
        int systemMetadataReplication = rs.getInt(5);
114
        logMetacat.info("DataReplication value: "+dataReplication);
115 112
        //Hub value
116
        int hubValue = rs.getInt(6);
113
        int hubValue = rs.getInt(5);
117 114
        logMetacat.info("Hub value: "+hubValue);
118 115
        //Get rid of local host
119 116
        if(!serverName.equals("localhost"))
......
155 152
          }//else
156 153
          //Set data replciation value
157 154
          server.setDataReplication(hasDataReplication);
158
          if (systemMetadataReplication == 1) {
159
        	  hasSystemMetadataReplication = true;
160
          }
161
          server.setSystemMetadataReplication(hasSystemMetadataReplication);
162 155
          
163 156
          //Detemine isHub by hubValue
164 157
          if (hubValue ==1)
......
392 385
    return server.getDataReplication();
393 386
  }//getDataReplicationValue
394 387
  
395
  /**
396
   * To a given server name, try to get its data file replciation option
397
   * If couldn't find the server in the server list, false will return
398
   * @param givenServerName, the server's name which we want to get data file 
399
   * replication value
400
   */
401
  public synchronized boolean getSystemMetadataReplicationValue(String givenServerName)
402
  {
403
    int index = -1;//Variable to store the index
404
    ReplicationServer server = null;//Variable for replication server
405
    
406
    //Get the server's index in server list
407
    index = findIndexInServerList(givenServerName);
408
    //If index = -1, couldn't find this server, null will return
409
    if (index == -1)
410
    {
411
      return false;
412
    }//if
413
    //Get Replication server object
414
    server = (ReplicationServer) serverList.elementAt(index);
415
    //Return this object's data replication value
416
    return server.isSystemMetadataReplication();
417
  }
418 388

  
419 389
  /**
420 390
   * To a given server name, try to get its hub option

Also available in: Unified diff