Project

General

Profile

1 522 berkley
/**
2
 *  '$RCSfile$'
3
 *    Purpose: A class to asyncronously do delta-T replication checking
4
 *  Copyright: 2000 Regents of the University of California and the
5
 *             National Center for Ecological Analysis and Synthesis
6
 *    Authors: Chad Berkley
7
 *
8
 *   '$Author$'
9
 *     '$Date$'
10
 * '$Revision$'
11 669 jones
 *
12
 * This program is free software; you can redistribute it and/or modify
13
 * it under the terms of the GNU General Public License as published by
14
 * the Free Software Foundation; either version 2 of the License, or
15
 * (at your option) any later version.
16
 *
17
 * This program is distributed in the hope that it will be useful,
18
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20
 * GNU General Public License for more details.
21
 *
22
 * You should have received a copy of the GNU General Public License
23
 * along with this program; if not, write to the Free Software
24
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
25 522 berkley
 */
26 2286 tao
27 5014 daigle
package edu.ucsb.nceas.metacat.replication;
28 522 berkley
29 5014 daigle
import edu.ucsb.nceas.metacat.CatalogMessageHandler;
30
import edu.ucsb.nceas.metacat.DBUtil;
31
import edu.ucsb.nceas.metacat.DocInfoHandler;
32
import edu.ucsb.nceas.metacat.DocumentImpl;
33
import edu.ucsb.nceas.metacat.DocumentImplWrapper;
34
import edu.ucsb.nceas.metacat.EventLog;
35 5089 daigle
import edu.ucsb.nceas.metacat.accesscontrol.AccessControlForSingleFile;
36 5098 daigle
import edu.ucsb.nceas.metacat.accesscontrol.XMLAccessDAO;
37 5014 daigle
import edu.ucsb.nceas.metacat.database.DBConnection;
38
import edu.ucsb.nceas.metacat.database.DBConnectionPool;
39
import edu.ucsb.nceas.metacat.database.DatabaseService;
40 5030 daigle
import edu.ucsb.nceas.metacat.properties.PropertyService;
41 5014 daigle
import edu.ucsb.nceas.metacat.shared.HandlerException;
42 4698 daigle
import edu.ucsb.nceas.metacat.util.MetacatUtil;
43 5324 berkley
import edu.ucsb.nceas.metacat.IdentifierManager;
44 4080 daigle
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
45
46 522 berkley
import java.sql.*;
47
import java.util.*;
48 2555 tao
import java.util.Date;
49 522 berkley
import java.io.*;
50
import java.net.*;
51 543 berkley
import java.text.*;
52 2663 sgarg
53
import org.apache.log4j.Logger;
54 522 berkley
import org.xml.sax.ContentHandler;
55
import org.xml.sax.ErrorHandler;
56
import org.xml.sax.InputSource;
57 5014 daigle
import org.xml.sax.SAXException;
58 522 berkley
import org.xml.sax.XMLReader;
59
import org.xml.sax.helpers.XMLReaderFactory;
60 561 berkley
import org.xml.sax.helpers.DefaultHandler;
61 522 berkley
62 561 berkley
63
64 522 berkley
/**
65
 * This class handles deltaT replication checking.  Whenever this TimerTask
66
 * is fired it checks each server in xml_replication for updates and updates
67
 * the local db as needed.
68
 */
69
public class ReplicationHandler extends TimerTask
70
{
71 573 berkley
  int serverCheckCode = 1;
72 2286 tao
  ReplicationServerList serverList = null;
73 2573 tao
  //PrintWriter out;
74 4698 daigle
//  private static final AbstractDatabase dbAdapter = MetacatUtil.dbAdapter;
75 5014 daigle
  private static Logger logReplication = Logger.getLogger("ReplicationLogging");
76 2663 sgarg
  private static Logger logMetacat = Logger.getLogger(ReplicationHandler.class);
77 5392 berkley
  private static Logger logD1 = Logger.getLogger("DataOneLogger");
78
79 2725 tao
  private static int DOCINSERTNUMBER = 1;
80
  private static int DOCERRORNUMBER  = 1;
81
  private static int REVINSERTNUMBER = 1;
82
  private static int REVERRORNUMBER  = 1;
83 5175 daigle
84
  private static int _xmlDocQueryCount = 0;
85
  private static int _xmlRevQueryCount = 0;
86
  private static long _xmlDocQueryTime = 0;
87
  private static long _xmlRevQueryTime = 0;
88
89
90 2573 tao
  public ReplicationHandler()
91 522 berkley
  {
92 2573 tao
    //this.out = o;
93 1292 tao
    serverList = new ReplicationServerList();
94 522 berkley
  }
95 2286 tao
96 2573 tao
  public ReplicationHandler(int serverCheckCode)
97 573 berkley
  {
98 2573 tao
    //this.out = o;
99 573 berkley
    this.serverCheckCode = serverCheckCode;
100 1292 tao
    serverList = new ReplicationServerList();
101 573 berkley
  }
102 2286 tao
103 522 berkley
  /**
104 2286 tao
   * Method that implements TimerTask.run().  It runs whenever the timer is
105 522 berkley
   * fired.
106
   */
107
  public void run()
108
  {
109
    //find out the last_checked time of each server in the server list and
110 2286 tao
    //send a query to each server to see if there are any documents in
111 522 berkley
    //xml_documents with an update_date > last_checked
112 5014 daigle
113 1032 tao
      //if serverList is null, metacat don't need to replication
114
      if (serverList==null||serverList.isEmpty())
115
      {
116
        return;
117
      }
118 1292 tao
      updateCatalog();
119
      update();
120 1217 tao
      //conn.close();
121 522 berkley
  }
122 2286 tao
123 522 berkley
  /**
124 5175 daigle
   * Method that uses revision tagging for replication instead of update_date.
125 577 berkley
   */
126 1292 tao
  private void update()
127 577 berkley
  {
128 5175 daigle
129
	  _xmlDocQueryCount = 0;
130
	  _xmlRevQueryCount = 0;
131
	  _xmlDocQueryTime = 0;
132
	  _xmlRevQueryTime = 0;
133 577 berkley
    /*
134
     Pseudo-algorithm
135
     - request a doc list from each server in xml_replication
136 2286 tao
     - check the rev number of each of those documents agains the
137 577 berkley
       documents in the local database
138
     - pull any documents that have a lesser rev number on the local server
139
       from the remote server
140
     - delete any documents that still exist in the local xml_documents but
141
       are in the deletedDocuments tag of the remote host response.
142
     - update last_checked to keep track of the last time it was checked.
143 2286 tao
       (this info is theoretically not needed using this system but probably
144 577 berkley
       should be kept anyway)
145
    */
146 2286 tao
147
    ReplicationServer replServer = null; // Variable to store the
148
                                        // ReplicationServer got from
149 1292 tao
                                        // Server list
150 2298 tao
    String server = null; // Variable to store server name
151 5014 daigle
//    String update;
152
    Vector<String> responses = new Vector<String>();
153 577 berkley
    URL u;
154 5175 daigle
    long replicationStartTime = System.currentTimeMillis();
155
    long timeToGetServerList = 0;
156 3898 tao
157 1585 tao
    //Check for every server in server list to get updated list and put
158
    // them in to response
159 5175 daigle
    long startTimeToGetServers = System.currentTimeMillis();
160 1585 tao
    for (int i=0; i<serverList.size(); i++)
161
    {
162 1292 tao
        // Get ReplicationServer object from server list
163
        replServer = serverList.serverAt(i);
164
        // Get server name from ReplicationServer object
165 2578 tao
        server = replServer.getServerName().trim();
166 1585 tao
        String result = null;
167 5014 daigle
        logReplication.info("ReplicationHandler.update - full update started to: " + server);
168 1292 tao
        // Send command to that server to get updated docid information
169
        try
170
        {
171 1585 tao
          u = new URL("https://" + server + "?server="
172 4698 daigle
          +MetacatUtil.getLocalReplicationServerName()+"&action=update");
173 5014 daigle
          logReplication.info("ReplicationHandler.update - Sending infomation " +u.toString());
174
          result = ReplicationService.getURLContent(u);
175 1585 tao
        }
176 1292 tao
        catch (Exception e)
177
        {
178 5014 daigle
          logMetacat.error("ReplicationHandler.update - " + ReplicationService.METACAT_REPL_ERROR_MSG);
179
          logReplication.error( "ReplicationHandler.update - Failed to get updated doc list "+
180 2663 sgarg
                       "for server " + server + " because "+e.getMessage());
181 1292 tao
          continue;
182 1585 tao
        }
183 2286 tao
184 5175 daigle
        //logReplication.info("ReplicationHandler.update - docid: "+server+" "+result);
185 1292 tao
        //check if result have error or not, if has skip it.
186 1609 tao
        if (result.indexOf("<error>")!=-1 && result.indexOf("</error>")!=-1)
187 1102 tao
        {
188 5014 daigle
          logMetacat.error("ReplicationHandler.update - " + ReplicationService.METACAT_REPL_ERROR_MSG);
189
          logReplication.error( "ReplicationHandler.update - Failed to get updated doc list "+
190 2663 sgarg
                       "for server " + server + " because "+result);
191 1102 tao
          continue;
192
        }
193 1585 tao
        //Add result to vector
194 577 berkley
        responses.add(result);
195 1585 tao
    }
196 5175 daigle
    timeToGetServerList = System.currentTimeMillis() - startTimeToGetServers;
197 2286 tao
198 1585 tao
    //make sure that there is updated file list
199
    //If response is null, metacat don't need do anything
200
    if (responses==null || responses.isEmpty())
201
    {
202 5014 daigle
    	logMetacat.error("ReplicationHandler.update - " + ReplicationService.METACAT_REPL_ERROR_MSG);
203
        logReplication.info( "ReplicationHandler.update - No updated doc list for "+
204 1585 tao
                           "every server and failed to replicate");
205 1032 tao
        return;
206 1585 tao
    }
207 2286 tao
208
209 5175 daigle
    //logReplication.info("ReplicationHandler.update - Responses from remote metacat about updated "+
210
    //               "document information: "+ responses.toString());
211
212
    long totalServerListParseTime = 0;
213 1585 tao
    // go through response vector(it contains updated vector and delete vector
214
    for(int i=0; i<responses.size(); i++)
215 2286 tao
    {
216 5175 daigle
    	long startServerListParseTime = System.currentTimeMillis();
217 3898 tao
    	XMLReader parser;
218
    	ReplMessageHandler message = new ReplMessageHandler();
219
    	try
220
        {
221
          parser = initParser(message);
222
        }
223
        catch (Exception e)
224
        {
225 5014 daigle
          logMetacat.error("ReplicationHandler.update - " + ReplicationService.METACAT_REPL_ERROR_MSG);
226
          logReplication.error("ReplicationHandler.update - Failed to replicate becaue couldn't " +
227 3898 tao
                                " initParser for message and " +e.getMessage());
228
           // stop replication
229
           return;
230
        }
231
232 1585 tao
        try
233
        {
234
          parser.parse(new InputSource(
235 577 berkley
                     new StringReader(
236
                     (String)(responses.elementAt(i)))));
237 1585 tao
        }
238
        catch(Exception e)
239
        {
240 5014 daigle
          logMetacat.error("ReplicationHandler.update - " + ReplicationService.METACAT_REPL_ERROR_MSG);
241
          logReplication.error("ReplicationHandler.update - Couldn't parse one responses "+
242 2663 sgarg
                                   "because "+ e.getMessage());
243 1585 tao
          continue;
244
        }
245 1037 tao
        //v is the list of updated documents
246 5014 daigle
        Vector<Vector<String>> updateList = new Vector<Vector<String>>(message.getUpdatesVect());
247
        logReplication.info("ReplicationHandler.update - The document list size is "+updateList.size()+ " from "+message.getServerName());
248 577 berkley
        //System.out.println("v: " + v.toString());
249 1037 tao
        //d is the list of deleted documents
250 5014 daigle
        Vector<Vector<String>> deleteList = new Vector<Vector<String>>(message.getDeletesVect());
251 1037 tao
        //System.out.println("d: " + d.toString());
252 5014 daigle
        logReplication.info("ReplicationHandler.update - Update vector size: "+ updateList.size()+" from "+message.getServerName());
253
        logReplication.info("ReplicationHandler.update - Delete vector size: "+ deleteList.size()+" from "+message.getServerName());
254
        logReplication.info("ReplicationHandler.update - The delete document list size is "+deleteList.size()+" from "+message.getServerName());
255 1585 tao
        // go though every element in updated document vector
256 2608 tao
        handleDocList(updateList, DocumentImpl.DOCUMENTTABLE);
257 1037 tao
        //handle deleted docs
258 1585 tao
        for(int k=0; k<deleteList.size(); k++)
259 577 berkley
        { //delete the deleted documents;
260 5014 daigle
          Vector<String> w = new Vector<String>(deleteList.elementAt(k));
261 1585 tao
          String docId = (String)w.elementAt(0);
262
          try
263 579 berkley
          {
264 2298 tao
            handleDeleteSingleDocument(docId, server);
265 579 berkley
          }
266 1585 tao
          catch (Exception ee)
267
          {
268
            continue;
269
          }
270 1037 tao
        }//for delete docs
271 2608 tao
272
        // handle replicate doc in xml_revision
273 5014 daigle
        Vector<Vector<String>> revisionList = new Vector<Vector<String>>(message.getRevisionsVect());
274
        logReplication.info("ReplicationHandler.update - The revision document list size is "+revisionList.size()+ " from "+message.getServerName());
275 2608 tao
        handleDocList(revisionList, DocumentImpl.REVISIONTABLE);
276 2725 tao
        DOCINSERTNUMBER = 1;
277
        DOCERRORNUMBER  = 1;
278
        REVINSERTNUMBER = 1;
279
        REVERRORNUMBER  = 1;
280 5175 daigle
281
        totalServerListParseTime += (System.currentTimeMillis() - startServerListParseTime);
282 1585 tao
    }//for response
283 2286 tao
284 1585 tao
    //updated last_checked
285
    for (int i=0;i<serverList.size(); i++)
286
    {
287
       // Get ReplicationServer object from server list
288
       replServer = serverList.serverAt(i);
289
       try
290
       {
291
         updateLastCheckTimeForSingleServer(replServer);
292
       }
293
       catch(Exception e)
294
       {
295
         continue;
296
       }
297
    }//for
298 5175 daigle
299
    long replicationEndTime = System.currentTimeMillis();
300
    logMetacat.debug("ReplicationHandler.update - Total replication time: " +
301
    		(replicationEndTime - replicationStartTime));
302
    logMetacat.debug("ReplicationHandler.update - time to get server list: " +
303
    		timeToGetServerList);
304
    logMetacat.debug("ReplicationHandler.update - server list parse time: " +
305
    		totalServerListParseTime);
306
    logMetacat.debug("ReplicationHandler.update - 'in xml_documents' total query count: " +
307
    		_xmlDocQueryCount);
308
    logMetacat.debug("ReplicationHandler.update - 'in xml_documents' total query time: " +
309
    		_xmlDocQueryTime + " ms");
310
    logMetacat.debug("ReplicationHandler.update - 'in xml_revisions' total query count: " +
311
    		_xmlRevQueryCount);
312
    logMetacat.debug("ReplicationHandler.update - 'in xml_revisions' total query time: " +
313
    		_xmlRevQueryTime + " ms");;
314 2286 tao
315 1585 tao
  }//update
316 2286 tao
317 1585 tao
  /* Handle replicate single xml document*/
318 2286 tao
  private void handleSingleXMLDocument(String remoteserver, String actions,
319 2641 tao
                                       String accNumber, String tableName)
320 5014 daigle
               throws HandlerException
321 1585 tao
  {
322
    DBConnection dbConn = null;
323
    int serialNumber = -1;
324
    try
325
    {
326
      // Get DBConnection from pool
327
      dbConn=DBConnectionPool.
328
                  getDBConnection("ReplicationHandler.handleSingleXMLDocument");
329
      serialNumber=dbConn.getCheckOutSerialNumber();
330
      //if the document needs to be updated or inserted, this is executed
331 1600 tao
      String readDocURLString = "https://" + remoteserver + "?server="+
332 4698 daigle
              MetacatUtil.getLocalReplicationServerName()+"&action=read&docid="+accNumber;
333
      readDocURLString = MetacatUtil.replaceWhiteSpaceForURL(readDocURLString);
334 1600 tao
      URL u = new URL(readDocURLString);
335 2286 tao
336 1585 tao
      // Get docid content
337 5014 daigle
      String newxmldoc = ReplicationService.getURLContent(u);
338 1585 tao
      // If couldn't get skip it
339 1609 tao
      if ( newxmldoc.indexOf("<error>")!= -1 && newxmldoc.indexOf("</error>")!=-1)
340 1292 tao
      {
341 5014 daigle
         throw new HandlerException("ReplicationHandler.handleSingleXMLDocument - " + newxmldoc);
342 1585 tao
      }
343 5014 daigle
      //logReplication.info("xml documnet:");
344
      //logReplication.info(newxmldoc);
345 2286 tao
346 1585 tao
      // Try get the docid info from remote server
347
      DocInfoHandler dih = new DocInfoHandler();
348
      XMLReader docinfoParser = initParser(dih);
349 2286 tao
      String docInfoURLStr = "https://" + remoteserver +
350 4698 daigle
                       "?server="+MetacatUtil.getLocalReplicationServerName()+
351 2641 tao
                       "&action=getdocumentinfo&docid="+accNumber;
352 4698 daigle
      docInfoURLStr = MetacatUtil.replaceWhiteSpaceForURL(docInfoURLStr);
353 1600 tao
      URL docinfoUrl = new URL(docInfoURLStr);
354 5014 daigle
      logReplication.info("ReplicationHandler.handleSingleXMLDocument - Sending message: " +
355 2663 sgarg
                                                  docinfoUrl.toString());
356 5014 daigle
      String docInfoStr = ReplicationService.getURLContent(docinfoUrl);
357 1585 tao
      docinfoParser.parse(new InputSource(new StringReader(docInfoStr)));
358 5014 daigle
      Hashtable<String, String> docinfoHash = dih.getDocInfo();
359 1585 tao
      // Get home server of the docid
360 5014 daigle
      String docHomeServer = docinfoHash.get("home_server");
361
      logReplication.info("ReplicationHandler.handleSingleXMLDocument - doc home server in repl: "+docHomeServer);
362
      String createdDate = docinfoHash.get("date_created");
363
      String updatedDate = docinfoHash.get("date_updated");
364 1585 tao
      //docid should include rev number too
365 4212 daigle
      /*String accnum=docId+util.getProperty("document.accNumSeparator")+
366 2641 tao
                                              (String)docinfoHash.get("rev");*/
367 5014 daigle
      logReplication.info("ReplicationHandler.handleSingleXMLDocument - docid in repl: "+accNumber);
368
      String docType = docinfoHash.get("doctype");
369
      logReplication.info("ReplicationHandler.handleSingleXMLDocument - doctype in repl: "+docType);
370 2286 tao
371 1585 tao
      String parserBase = null;
372
      // this for eml2 and we need user eml2 parser
373 2169 sgarg
      if (docType != null && (docType.trim()).equals(DocumentImpl.EML2_0_0NAMESPACE))
374 1585 tao
      {
375 2163 tao
         parserBase = DocumentImpl.EML200;
376 1585 tao
      }
377 2286 tao
      else if (docType != null && (docType.trim()).equals(DocumentImpl.EML2_0_1NAMESPACE))
378
      {
379
        parserBase = DocumentImpl.EML200;
380
      }
381
      else if (docType != null && (docType.trim()).equals(DocumentImpl.EML2_1_0NAMESPACE))
382
      {
383
        parserBase = DocumentImpl.EML210;
384
      }
385 5709 leinfelder
      else if (docType != null && (docType.trim()).equals(DocumentImpl.EML2_1_1NAMESPACE))
386
      {
387
        parserBase = DocumentImpl.EML210;
388
      }
389 1585 tao
      // Write the document into local host
390
      DocumentImplWrapper wrapper = new DocumentImplWrapper(parserBase, false);
391 2286 tao
      String newDocid = wrapper.writeReplication(dbConn,
392 5195 daigle
                              newxmldoc,
393 5014 daigle
                              docinfoHash.get("public_access"),
394 1585 tao
                              null,  /* the dtd text */
395 2286 tao
                              actions,
396 2641 tao
                              accNumber,
397 5014 daigle
                              docinfoHash.get("user_owner"),
398 1585 tao
                              null, /* null for groups[] */
399 2286 tao
                              docHomeServer,
400 2624 tao
                              remoteserver, tableName, true,// true is for time replication
401
                              createdDate,
402 3230 tao
                              updatedDate);
403 4419 leinfelder
404 5098 daigle
      //process extra access rules
405
      Vector<XMLAccessDAO> xmlAccessDAOList = dih.getAccessControlList();
406
      if (xmlAccessDAOList != null) {
407
      	AccessControlForSingleFile acfsf = new AccessControlForSingleFile(accNumber);
408
      	for (XMLAccessDAO xmlAccessDAO : xmlAccessDAOList) {
409
      		if (!acfsf.accessControlExists(xmlAccessDAO)) {
410
      			acfsf.insertPermissions(xmlAccessDAO);
411
      		}
412 4419 leinfelder
          }
413
      }
414
415 5324 berkley
      //process guid
416
      logReplication.debug("Processing guid information from docinfoHash: " + docinfoHash.toString());
417
      String guid = docinfoHash.get("guid");
418 5451 berkley
      String docName = docinfoHash.get("docName");
419 5440 berkley
      System.out.println("%%%%%%%%%%%%%%%%%%%%%%%%%%guid passed from docinfo hash: " + guid);
420 5324 berkley
      IdentifierManager idman = IdentifierManager.getInstance();
421
      if(guid != null && !idman.identifierExists(guid))
422
      { //if the guid was passed in, put it in the identifiers table
423
        logReplication.debug("Creating guid/docid mapping for docid " +
424
          docinfoHash.get("docid") + " and guid: " + guid);
425 5451 berkley
        System.out.println("docname: " + docName);
426
        if(docName.trim().equals("systemMetadata"))
427
        {
428
            System.out.println("creating mapping for systemMetadata: guid: " + guid + " localId: " + docinfoHash.get("docid"));
429
            idman.createSystemMetadataMapping(guid, docinfoHash.get("docid"));
430
        }
431
        else
432
        {
433
            System.out.println("creating mapping: guid: " + guid + " localId: " + docinfoHash.get("docid"));
434
            idman.createMapping(guid, docinfoHash.get("docid"));
435
        }
436 5324 berkley
      }
437
      else
438
      {
439
        logReplication.debug("No guid information was included with the replicated document");
440
      }
441
442 5457 berkley
      if(guid != null)
443
      {
444 5459 berkley
          if(!docName.trim().equals("systemMetadata"))
445
          {
446
              logReplication.info("replicate D1GUID:" + guid + ":D1SCIMETADATA:" +
447
                      accNumber + ":");
448
          }
449
          else
450
          {
451
              logReplication.info("replicate D1GUID:" + guid + ":D1SYSMETADATA:" +
452
                      accNumber + ":");
453
          }
454 5457 berkley
      }
455
456 5014 daigle
      logReplication.info("ReplicationHandler.handleSingleXMLDocument - Successfully replicated doc " + accNumber);
457 2725 tao
      if (tableName.equals(DocumentImpl.DOCUMENTTABLE))
458
      {
459 5014 daigle
        logReplication.info("ReplicationHandler.handleSingleXMLDocument - " + DOCINSERTNUMBER + " Wrote xml doc " + accNumber +
460 2725 tao
                                     " into "+tableName + " from " +
461 1585 tao
                                         remoteserver);
462 2725 tao
        DOCINSERTNUMBER++;
463
      }
464
      else
465
      {
466 5014 daigle
          logReplication.info("ReplicationHandler.handleSingleXMLDocument - " +REVINSERTNUMBER + " Wrote xml doc " + accNumber +
467 2725 tao
                  " into "+tableName + " from " +
468
                      remoteserver);
469
          REVINSERTNUMBER++;
470
      }
471 3234 tao
      String ip = getIpFromURL(u);
472 5014 daigle
      EventLog.getInstance().log(ip, ReplicationService.REPLICATIONUSER, accNumber, actions);
473 2725 tao
474 2286 tao
475
    }//try
476 577 berkley
    catch(Exception e)
477
    {
478 2725 tao
479
        if (tableName.equals(DocumentImpl.DOCUMENTTABLE))
480
        {
481 5014 daigle
        	logMetacat.error("ReplicationHandler.handleSingleXMLDocument - " + ReplicationService.METACAT_REPL_ERROR_MSG);
482
        	logReplication.error("ReplicationHandler.handleSingleXMLDocument - " +DOCERRORNUMBER + " Failed to write xml doc " + accNumber +
483 2725 tao
                                       " into "+tableName + " from " +
484
                                           remoteserver + " because "+e.getMessage());
485
          DOCERRORNUMBER++;
486
        }
487
        else
488
        {
489 5014 daigle
        	logMetacat.error("ReplicationHandler.handleSingleXMLDocument - " + ReplicationService.METACAT_REPL_ERROR_MSG);
490
        	logReplication.error("ReplicationHandler.handleSingleXMLDocument - " +REVERRORNUMBER + " Failed to write xml doc " + accNumber +
491 2725 tao
                    " into "+tableName + " from " +
492
                        remoteserver +" because "+e.getMessage());
493
            REVERRORNUMBER++;
494
        }
495 5014 daigle
        logMetacat.error("ReplicationHandler.handleSingleXMLDocument - " + ReplicationService.METACAT_REPL_ERROR_MSG);
496
        logReplication.error("ReplicationHandler.handleSingleXMLDocument - Failed to write doc " + accNumber +
497 2663 sgarg
                                      " into db because " +e.getMessage());
498 5014 daigle
      throw new HandlerException("ReplicationHandler.handleSingleXMLDocument - generic exception "
499
    		  + "writing Replication: " +e.getMessage());
500 577 berkley
    }
501 667 berkley
    finally
502
    {
503 1585 tao
       //return DBConnection
504
       DBConnectionPool.returnDBConnection(dbConn, serialNumber);
505
    }//finally
506 5392 berkley
    logD1.info("replication.create localId:" + accNumber);
507 1585 tao
  }
508 2286 tao
509
510
511 1585 tao
  /* Handle replicate single xml document*/
512 2286 tao
  private void handleSingleDataFile(String remoteserver, String actions,
513 2641 tao
                                    String accNumber, String tableName)
514 5014 daigle
               throws HandlerException
515 1585 tao
  {
516 5014 daigle
    logReplication.info("ReplicationHandler.handleSingleDataFile - Try to replicate data file: " + accNumber);
517 1585 tao
    DBConnection dbConn = null;
518
    int serialNumber = -1;
519
    try
520
    {
521
      // Get DBConnection from pool
522
      dbConn=DBConnectionPool.
523
                  getDBConnection("ReplicationHandler.handleSinlgeDataFile");
524
      serialNumber=dbConn.getCheckOutSerialNumber();
525
      // Try get docid info from remote server
526
      DocInfoHandler dih = new DocInfoHandler();
527
      XMLReader docinfoParser = initParser(dih);
528 2286 tao
      String docInfoURLString = "https://" + remoteserver +
529 4698 daigle
                  "?server="+MetacatUtil.getLocalReplicationServerName()+
530 2641 tao
                  "&action=getdocumentinfo&docid="+accNumber;
531 4698 daigle
      docInfoURLString = MetacatUtil.replaceWhiteSpaceForURL(docInfoURLString);
532 1600 tao
      URL docinfoUrl = new URL(docInfoURLString);
533 2286 tao
534 5014 daigle
      String docInfoStr = ReplicationService.getURLContent(docinfoUrl);
535 1585 tao
      docinfoParser.parse(new InputSource(new StringReader(docInfoStr)));
536 5014 daigle
      Hashtable<String, String> docinfoHash = dih.getDocInfo();
537 5175 daigle
      // Get docid owner
538 5014 daigle
      String user = docinfoHash.get("user_owner");
539 1585 tao
      // Get docid name (such as acl or dataset)
540 5014 daigle
      String docName = docinfoHash.get("docname");
541 2286 tao
      // Get doc type (eml public id)
542 5014 daigle
      String docType = docinfoHash.get("doctype");
543 1585 tao
      // Get docid home sever. it might be different to remoteserver
544 5175 daigle
      // because of hub feature
545 5014 daigle
      String docHomeServer = docinfoHash.get("home_server");
546
      String createdDate = docinfoHash.get("date_created");
547
      String updatedDate = docinfoHash.get("date_updated");
548 1585 tao
      //docid should include rev number too
549 4212 daigle
      /*String accnum=docId+util.getProperty("document.accNumSeparator")+
550 2641 tao
                                              (String)docinfoHash.get("rev");*/
551 2286 tao
552
553 4080 daigle
      String datafilePath = PropertyService.getProperty("application.datafilepath");
554 1585 tao
      // Get data file content
555 1600 tao
      String readDataURLString = "https://" + remoteserver + "?server="+
556 4698 daigle
                                        MetacatUtil.getLocalReplicationServerName()+
557 2641 tao
                                            "&action=readdata&docid="+accNumber;
558 4698 daigle
      readDataURLString = MetacatUtil.replaceWhiteSpaceForURL(readDataURLString);
559 1600 tao
      URL u = new URL(readDataURLString);
560 2286 tao
      InputStream input = u.openStream();
561 1585 tao
      //register data file into xml_documents table and wite data file
562
      //into file system
563 1600 tao
      if ( input != null)
564 667 berkley
      {
565 2286 tao
        DocumentImpl.writeDataFileInReplication(input,
566 2608 tao
                                                datafilePath,
567
                                                docName,docType,
568 2641 tao
                                                accNumber, user,
569 2608 tao
                                                docHomeServer,
570
                                                remoteserver,
571
                                                tableName,
572 2624 tao
                                                true, //true means timed replication
573
                                                createdDate,
574 3230 tao
                                                updatedDate);
575 2624 tao
576 4419 leinfelder
        //process extra access rules
577 5098 daigle
        Vector<XMLAccessDAO> xmlAccessDAOList = dih.getAccessControlList();
578
        if (xmlAccessDAOList != null) {
579
        	AccessControlForSingleFile acfsf = new AccessControlForSingleFile(accNumber);
580
        	for (XMLAccessDAO xmlAccessDAO : xmlAccessDAOList) {
581
        		if (!acfsf.accessControlExists(xmlAccessDAO)) {
582
        			acfsf.insertPermissions(xmlAccessDAO);
583
        		}
584 4419 leinfelder
            }
585
        }
586
587 5014 daigle
        logReplication.info("ReplicationHandler.handleSingleDataFile - Successfully to write datafile " + accNumber);
588 2725 tao
        /*MetacatReplication.replLog("wrote datafile " + accNumber + " from " +
589 5175 daigle
                                    remote server);*/
590 2725 tao
        if (tableName.equals(DocumentImpl.DOCUMENTTABLE))
591
        {
592 5014 daigle
          logReplication.info("ReplicationHandler.handleSingleDataFile - " + DOCINSERTNUMBER + " Wrote data file" + accNumber +
593 2725 tao
                                       " into "+tableName + " from " +
594
                                           remoteserver);
595
          DOCINSERTNUMBER++;
596
        }
597
        else
598
        {
599 5014 daigle
            logReplication.info("ReplicationHandler.handleSingleDataFile - " + REVINSERTNUMBER + " Wrote data file" + accNumber +
600 2725 tao
                    " into "+tableName + " from " +
601
                        remoteserver);
602
            REVINSERTNUMBER++;
603
        }
604 3234 tao
        String ip = getIpFromURL(u);
605 5014 daigle
        EventLog.getInstance().log(ip, ReplicationService.REPLICATIONUSER, accNumber, actions);
606 2725 tao
607 1585 tao
      }//if
608
      else
609 1217 tao
      {
610 5014 daigle
         logReplication.info("ReplicationHandler.handleSingleDataFile - Couldn't open the data file: " + accNumber);
611
         throw new HandlerException("ReplicationHandler.handleSingleDataFile - Couldn't open the data file: " + accNumber);
612 1585 tao
      }//else
613 2286 tao
614
    }//try
615 1585 tao
    catch(Exception e)
616
    {
617 5175 daigle
      /*MetacatReplication.replErrorLog("Failed to try wrote data file " + accNumber +
618 2725 tao
                                      " because " +e.getMessage());*/
619
      if (tableName.equals(DocumentImpl.DOCUMENTTABLE))
620
      {
621 5014 daigle
    	logMetacat.error("ReplicationHandler.handleSingleDataFile - " + ReplicationService.METACAT_REPL_ERROR_MSG);
622
    	logReplication.error("ReplicationHandler.handleSingleDataFile - " + DOCERRORNUMBER + " Failed to write data file " + accNumber +
623
                                     " into " + tableName + " from " +
624
                                         remoteserver + " because " + e.getMessage());
625 2725 tao
        DOCERRORNUMBER++;
626
      }
627
      else
628
      {
629 5014 daigle
    	  logMetacat.error("ReplicationHandler.handleSingleDataFile - " + ReplicationService.METACAT_REPL_ERROR_MSG);
630
    	  logReplication.error("ReplicationHandler.handleSingleDataFile - " + REVERRORNUMBER + " Failed to write data file" + accNumber +
631
                  " into " + tableName + " from " +
632
                      remoteserver +" because "+ e.getMessage());
633 2725 tao
          REVERRORNUMBER++;
634
      }
635 5014 daigle
      logMetacat.error("ReplicationHandler.handleSingleDataFile - " + ReplicationService.METACAT_REPL_ERROR_MSG);
636
      logReplication.error("ReplicationHandler.handleSingleDataFile - Failed to try wrote datafile " + accNumber +
637
                                      " because " + e.getMessage());
638
      throw new HandlerException("ReplicationHandler.handleSingleDataFile - generic exception "
639
    		  + "writing Replication: " + e.getMessage());
640 1585 tao
    }
641
    finally
642
    {
643
       //return DBConnection
644
       DBConnectionPool.returnDBConnection(dbConn, serialNumber);
645
    }//finally
646 5392 berkley
    logD1.info("replication.create localId:" + accNumber);
647 1585 tao
  }
648 2286 tao
649
650
651 1585 tao
  /* Handle delete single document*/
652 2298 tao
  private void handleDeleteSingleDocument(String docId, String notifyServer)
653 5014 daigle
               throws HandlerException
654 1585 tao
  {
655 5014 daigle
    logReplication.info("ReplicationHandler.handleDeleteSingleDocument - Try delete doc: "+docId);
656 1585 tao
    DBConnection dbConn = null;
657
    int serialNumber = -1;
658
    try
659
    {
660
      // Get DBConnection from pool
661
      dbConn=DBConnectionPool.
662
                  getDBConnection("ReplicationHandler.handleDeleteSingleDoc");
663
      serialNumber=dbConn.getCheckOutSerialNumber();
664
      if(!alreadyDeleted(docId))
665 1217 tao
      {
666 2286 tao
667 1585 tao
         //because delete method docid should have rev number
668
         //so we just add one for it. This rev number is no sence.
669 4212 daigle
         String accnum=docId+PropertyService.getProperty("document.accNumSeparator")+"1";
670 1585 tao
         //System.out.println("accnum: "+accnum);
671 3230 tao
         DocumentImpl.delete(accnum, null, null, notifyServer);
672 5014 daigle
         logReplication.info("ReplicationHandler.handleDeleteSingleDocument - Successfully deleted doc " + docId);
673
         logReplication.info("ReplicationHandler.handleDeleteSingleDocument - Doc " + docId + " deleted");
674 3234 tao
         URL u = new URL("https://"+notifyServer);
675
         String ip = getIpFromURL(u);
676 5014 daigle
         EventLog.getInstance().log(ip, ReplicationService.REPLICATIONUSER, docId, "delete");
677 1585 tao
      }
678 2286 tao
679
    }//try
680 1585 tao
    catch(Exception e)
681
    {
682 5014 daigle
      logMetacat.error("ReplicationHandler.handleDeleteSingleDocument - " + ReplicationService.METACAT_REPL_ERROR_MSG);
683
      logReplication.error("ReplicationHandler.handleDeleteSingleDocument - Failed to delete doc " + docId +
684 2663 sgarg
                                 " in db because because " + e.getMessage());
685 5014 daigle
      throw new HandlerException("ReplicationHandler.handleDeleteSingleDocument - generic exception "
686
    		  + "when handling document: " + e.getMessage());
687 1585 tao
    }
688
    finally
689
    {
690
       //return DBConnection
691
       DBConnectionPool.returnDBConnection(dbConn, serialNumber);
692 1037 tao
    }//finally
693 5392 berkley
    logD1.info("replication.handleDeleteSingleDocument localId:" + docId);
694 1585 tao
  }
695 2286 tao
696 1585 tao
  /* Handle updateLastCheckTimForSingleServer*/
697 2286 tao
  private void updateLastCheckTimeForSingleServer(ReplicationServer repServer)
698 5014 daigle
                                                  throws HandlerException
699 590 berkley
  {
700 1585 tao
    String server = repServer.getServerName();
701 1217 tao
    DBConnection dbConn = null;
702
    int serialNumber = -1;
703
    PreparedStatement pstmt = null;
704 590 berkley
    try
705
    {
706 1585 tao
      // Get DBConnection from pool
707 1217 tao
      dbConn=DBConnectionPool.
708 1585 tao
             getDBConnection("ReplicationHandler.updateLastCheckTimeForServer");
709 1217 tao
      serialNumber=dbConn.getCheckOutSerialNumber();
710 2286 tao
711 5014 daigle
      logReplication.info("ReplicationHandler.updateLastCheckTimeForSingleServer - Try to update last_check for server: "+server);
712 1585 tao
      // Get time from remote server
713
      URL dateurl = new URL("https://" + server + "?server="+
714 4698 daigle
      MetacatUtil.getLocalReplicationServerName()+"&action=gettime");
715 5014 daigle
      String datexml = ReplicationService.getURLContent(dateurl);
716
      logReplication.info("ReplicationHandler.updateLastCheckTimeForSingleServer - datexml: "+datexml);
717 1585 tao
      if (datexml!=null && !datexml.equals(""))
718
      {
719
         String datestr = datexml.substring(11, datexml.indexOf('<', 11));
720
         StringBuffer sql = new StringBuffer();
721 1751 tao
         /*sql.append("update xml_replication set last_checked = to_date('");
722 1585 tao
         sql.append(datestr).append("', 'YY-MM-DD HH24:MI:SS') where ");
723 1751 tao
         sql.append("server like '").append(server).append("'");*/
724
         sql.append("update xml_replication set last_checked = ");
725 5319 jones
         sql.append(DatabaseService.getInstance().getDBAdapter().toDate(datestr, "MM/DD/YY HH24:MI:SS"));
726 1751 tao
         sql.append(" where server like '").append(server).append("'");
727 1585 tao
         pstmt = dbConn.prepareStatement(sql.toString());
728 2286 tao
729 1585 tao
         pstmt.executeUpdate();
730
         dbConn.commit();
731
         pstmt.close();
732 5014 daigle
         logReplication.info("ReplicationHandler.updateLastCheckTimeForSingleServer - last_checked updated to "+datestr+" on "
733 2663 sgarg
                                      + server);
734 1585 tao
      }//if
735
      else
736
      {
737 2286 tao
738 5014 daigle
         logReplication.info("ReplicationHandler.updateLastCheckTimeForSingleServer - Failed to update last_checked for server "  +
739 2286 tao
                                  server + " in db because couldn't get time "
740 2663 sgarg
                                  );
741 1585 tao
         throw new Exception("Couldn't get time for server "+ server);
742
      }
743 2286 tao
744
    }//try
745 1585 tao
    catch(Exception e)
746
    {
747 5014 daigle
      logMetacat.error("ReplicationHandler.updateLastCheckTimeForSingleServer - " + ReplicationService.METACAT_REPL_ERROR_MSG);
748
      logReplication.error("ReplicationHandler.updateLastCheckTimeForSingleServer - Failed to update last_checked for server " +
749
                                server + " in db because because " + e.getMessage());
750
      throw new HandlerException("ReplicationHandler.updateLastCheckTimeForSingleServer - "
751
    		  + "Error updating last checked time: " + e.getMessage());
752 1585 tao
    }
753
    finally
754
    {
755
       //return DBConnection
756
       DBConnectionPool.returnDBConnection(dbConn, serialNumber);
757
    }//finally
758
  }
759 2286 tao
760
761
762 1585 tao
  /**
763
   * updates xml_catalog with entries from other servers.
764
   */
765
  private void updateCatalog()
766
  {
767 5014 daigle
    logReplication.info("ReplicationHandler.updateCatalog - Start of updateCatalog");
768 1585 tao
    // ReplicationServer object in server list
769
    ReplicationServer replServer = null;
770
    PreparedStatement pstmt = null;
771
    String server = null;
772 2286 tao
773
774 1585 tao
    // Go through each ReplicationServer object in sererlist
775
    for (int j=0; j<serverList.size(); j++)
776 2286 tao
    {
777 5014 daigle
      Vector<Vector<String>> remoteCatalog = new Vector<Vector<String>>();
778
      Vector<String> publicId = new Vector<String>();
779 1585 tao
      try
780
      {
781 1292 tao
        // Get ReplicationServer object from server list
782
        replServer = serverList.serverAt(j);
783
        // Get server name from the ReplicationServer object
784
        server = replServer.getServerName();
785
        // Try to get catalog
786 1011 tao
        URL u = new URL("https://" + server + "?server="+
787 4698 daigle
        MetacatUtil.getLocalReplicationServerName()+"&action=getcatalog");
788 5014 daigle
        logReplication.info("ReplicationHandler.updateCatalog - sending message " + u.toString());
789
        String catxml = ReplicationService.getURLContent(u);
790 2286 tao
791 1292 tao
        // Make sure there are not error, no empty string
792
        if (catxml.indexOf("error")!=-1 || catxml==null||catxml.equals(""))
793
        {
794 1585 tao
          throw new Exception("Couldn't get catalog list form server " +server);
795 1292 tao
        }
796 5175 daigle
        logReplication.debug("ReplicationHandler.updateCatalog - catxml: " + catxml);
797 590 berkley
        CatalogMessageHandler cmh = new CatalogMessageHandler();
798
        XMLReader catparser = initParser(cmh);
799
        catparser.parse(new InputSource(new StringReader(catxml)));
800
        //parse the returned catalog xml and put it into a vector
801 1585 tao
        remoteCatalog = cmh.getCatalogVect();
802 2286 tao
803 5175 daigle
        // Make sure remoteCatalog is not empty
804 1292 tao
        if (remoteCatalog.isEmpty())
805
        {
806 1585 tao
          throw new Exception("Couldn't get catalog list form server " +server);
807 1292 tao
        }
808 2286 tao
809 5014 daigle
        String localcatxml = ReplicationService.getCatalogXML();
810 2286 tao
811 1292 tao
        // Make sure local catalog is no empty
812
        if (localcatxml==null||localcatxml.equals(""))
813
        {
814 1585 tao
          throw new Exception("Couldn't get catalog list form server " +server);
815 1292 tao
        }
816 2286 tao
817 590 berkley
        cmh = new CatalogMessageHandler();
818
        catparser = initParser(cmh);
819
        catparser.parse(new InputSource(new StringReader(localcatxml)));
820 5014 daigle
        Vector<Vector<String>> localCatalog = cmh.getCatalogVect();
821 2286 tao
822 590 berkley
        //now we have the catalog from the remote server and this local server
823
        //we now need to compare the two and merge the differences.
824
        //the comparison is base on the public_id fields which is the 4th
825
        //entry in each row vector.
826 5014 daigle
        publicId = new Vector<String>();
827 590 berkley
        for(int i=0; i<localCatalog.size(); i++)
828
        {
829 5014 daigle
          Vector<String> v = new Vector<String>(localCatalog.elementAt(i));
830
          logReplication.info("ReplicationHandler.updateCatalog - v1: " + v.toString());
831 590 berkley
          publicId.add(new String((String)v.elementAt(3)));
832 595 berkley
          //System.out.println("adding " + (String)v.elementAt(3));
833 590 berkley
        }
834 1585 tao
      }//try
835
      catch (Exception e)
836
      {
837 5014 daigle
        logMetacat.error("ReplicationHandler.updateCatalog - " + ReplicationService.METACAT_REPL_ERROR_MSG);
838
        logReplication.error("ReplicationHandler.updateCatalog - Failed to update catalog for server "+
839 1585 tao
                                    server + " because " +e.getMessage());
840
      }//catch
841 2286 tao
842 1585 tao
      for(int i=0; i<remoteCatalog.size(); i++)
843
      {
844
         // DConnection
845
        DBConnection dbConn = null;
846
        // DBConnection checkout serial number
847
        int serialNumber = -1;
848
        try
849 590 berkley
        {
850 1585 tao
            dbConn=DBConnectionPool.
851
                  getDBConnection("ReplicationHandler.updateCatalog");
852
            serialNumber=dbConn.getCheckOutSerialNumber();
853 5014 daigle
            Vector<String> v = remoteCatalog.elementAt(i);
854 1585 tao
            //System.out.println("v2: " + v.toString());
855
            //System.out.println("i: " + i);
856
            //System.out.println("remoteCatalog.size(): " + remoteCatalog.size());
857
            //System.out.println("publicID: " + publicId.toString());
858 5014 daigle
            logReplication.info
859
                              ("ReplicationHandler.updateCatalog - v.elementAt(3): " + (String)v.elementAt(3));
860 1585 tao
           if(!publicId.contains(v.elementAt(3)))
861
           { //so we don't have this public id in our local table so we need to
862
             //add it.
863
             //System.out.println("in if");
864
             StringBuffer sql = new StringBuffer();
865
             sql.append("insert into xml_catalog (entry_type, source_doctype, ");
866
             sql.append("target_doctype, public_id, system_id) values (?,?,?,");
867
             sql.append("?,?)");
868
             //System.out.println("sql: " + sql.toString());
869
             pstmt = dbConn.prepareStatement(sql.toString());
870
             pstmt.setString(1, (String)v.elementAt(0));
871
             pstmt.setString(2, (String)v.elementAt(1));
872
             pstmt.setString(3, (String)v.elementAt(2));
873
             pstmt.setString(4, (String)v.elementAt(3));
874
             pstmt.setString(5, (String)v.elementAt(4));
875
             pstmt.execute();
876
             pstmt.close();
877 5014 daigle
             logReplication.info("ReplicationHandler.updateCatalog - Success fully to insert new publicid "+
878 1585 tao
                               (String)v.elementAt(3) + " from server"+server);
879
           }
880 590 berkley
        }
881 1585 tao
        catch(Exception e)
882
        {
883 5014 daigle
           logMetacat.error("ReplicationHandler.updateCatalog - " + ReplicationService.METACAT_REPL_ERROR_MSG);
884
           logReplication.error("ReplicationHandler.updateCatalog - Failed to update catalog for server "+
885 1585 tao
                                    server + " because " +e.getMessage());
886
        }//catch
887
        finally
888
        {
889
           DBConnectionPool.returnDBConnection(dbConn, serialNumber);
890 5175 daigle
        }//finally
891 1585 tao
      }//for remote catalog
892
    }//for server list
893 5014 daigle
    logReplication.info("End of updateCatalog");
894 590 berkley
  }
895 2286 tao
896 590 berkley
  /**
897 579 berkley
   * Method that returns true if docid has already been "deleted" from metacat.
898 582 berkley
   * This method really implements a truth table for deleted documents
899 590 berkley
   * The table is (a docid in one of the tables is represented by the X):
900 582 berkley
   * xml_docs      xml_revs      deleted?
901
   * ------------------------------------
902
   *   X             X             FALSE
903
   *   X             _             FALSE
904
   *   _             X             TRUE
905
   *   _             _             TRUE
906 579 berkley
   */
907 5014 daigle
  private static boolean alreadyDeleted(String docid) throws HandlerException
908 579 berkley
  {
909 1217 tao
    DBConnection dbConn = null;
910
    int serialNumber = -1;
911
    PreparedStatement pstmt = null;
912 579 berkley
    try
913
    {
914 1217 tao
      dbConn=DBConnectionPool.
915
                  getDBConnection("ReplicationHandler.alreadyDeleted");
916
      serialNumber=dbConn.getCheckOutSerialNumber();
917 582 berkley
      boolean xml_docs = false;
918
      boolean xml_revs = false;
919 2286 tao
920 579 berkley
      StringBuffer sb = new StringBuffer();
921 582 berkley
      sb.append("select docid from xml_revisions where docid like '");
922 579 berkley
      sb.append(docid).append("'");
923 1217 tao
      pstmt = dbConn.prepareStatement(sb.toString());
924 579 berkley
      pstmt.execute();
925
      ResultSet rs = pstmt.getResultSet();
926
      boolean tablehasrows = rs.next();
927
      if(tablehasrows)
928
      {
929 582 berkley
        xml_revs = true;
930
      }
931 2286 tao
932 582 berkley
      sb = new StringBuffer();
933
      sb.append("select docid from xml_documents where docid like '");
934
      sb.append(docid).append("'");
935 667 berkley
      pstmt.close();
936 1217 tao
      pstmt = dbConn.prepareStatement(sb.toString());
937
      //increase usage count
938
      dbConn.increaseUsageCount(1);
939 582 berkley
      pstmt.execute();
940
      rs = pstmt.getResultSet();
941
      tablehasrows = rs.next();
942 667 berkley
      pstmt.close();
943 582 berkley
      if(tablehasrows)
944
      {
945
        xml_docs = true;
946
      }
947 2286 tao
948 582 berkley
      if(xml_docs && xml_revs)
949
      {
950
        return false;
951
      }
952
      else if(xml_docs && !xml_revs)
953
      {
954
        return false;
955
      }
956
      else if(!xml_docs && xml_revs)
957
      {
958 579 berkley
        return true;
959
      }
960 582 berkley
      else if(!xml_docs && !xml_revs)
961
      {
962
        return true;
963
      }
964 579 berkley
    }
965
    catch(Exception e)
966
    {
967 5014 daigle
      logMetacat.error("ReplicationHandler.alreadyDeleted - " + ReplicationService.METACAT_REPL_ERROR_MSG);
968
      logReplication.error("ReplicationHandler.alreadyDeleted - general error in alreadyDeleted: " +
969 2663 sgarg
                          e.getMessage());
970 5014 daigle
      throw new HandlerException("ReplicationHandler.alreadyDeleted - general error: "
971
    		  + e.getMessage());
972 579 berkley
    }
973 667 berkley
    finally
974
    {
975 1217 tao
      try
976
      {
977
        pstmt.close();
978
      }//try
979
      catch (SQLException ee)
980
      {
981 5014 daigle
    	logMetacat.error("ReplicationHandler.alreadyDeleted - " + ReplicationService.METACAT_REPL_ERROR_MSG);
982
        logReplication.error("ReplicationHandler.alreadyDeleted - Error in replicationHandler.alreadyDeleted "+
983 2663 sgarg
                          "to close pstmt: "+ee.getMessage());
984 5014 daigle
        throw new HandlerException("ReplicationHandler.alreadyDeleted - SQL error when closing prepared statement: "
985
      		  + ee.getMessage());
986 1217 tao
      }//catch
987
      finally
988
      {
989
        DBConnectionPool.returnDBConnection(dbConn, serialNumber);
990
      }//finally
991
    }//finally
992 579 berkley
    return false;
993
  }
994 533 berkley
995 2286 tao
996 533 berkley
  /**
997 574 berkley
   * Method to initialize the message parser
998
   */
999
  public static XMLReader initParser(DefaultHandler dh)
1000 5014 daigle
          throws HandlerException
1001 574 berkley
  {
1002
    XMLReader parser = null;
1003
1004
    try {
1005
      ContentHandler chandler = dh;
1006
1007
      // Get an instance of the parser
1008 4213 daigle
      String parserName = PropertyService.getProperty("xml.saxparser");
1009 574 berkley
      parser = XMLReaderFactory.createXMLReader(parserName);
1010
1011
      // Turn off validation
1012
      parser.setFeature("http://xml.org/sax/features/validation", false);
1013 2286 tao
1014 574 berkley
      parser.setContentHandler((ContentHandler)chandler);
1015
      parser.setErrorHandler((ErrorHandler)chandler);
1016
1017 5014 daigle
    } catch (SAXException se) {
1018
      throw new HandlerException("ReplicationHandler.initParser - Sax error when "
1019
    		  + " initializing parser: " + se.getMessage());
1020
    } catch (PropertyNotFoundException pnfe) {
1021
        throw new HandlerException("ReplicationHandler.initParser - Property error when "
1022
      		  + " getting parser name: " + pnfe.getMessage());
1023
    }
1024 574 berkley
1025
    return parser;
1026
  }
1027 2286 tao
1028 2555 tao
  /**
1029 5175 daigle
	 * This method will combine given time string(in short format) to current
1030 5014 daigle
	 * date. If the given time (e.g 10:00 AM) passed the current time (e.g 2:00
1031
	 * PM Aug 21, 2005), then the time will set to second day, 10:00 AM Aug 22,
1032
	 * 2005. If the given time (e.g 10:00 AM) haven't passed the current time
1033
	 * (e.g 8:00 AM Aug 21, 2005) The time will set to be 10:00 AM Aug 21, 2005.
1034
	 *
1035
	 * @param givenTime
1036
	 *            the format should be "10:00 AM " or "2:00 PM"
1037
	 * @return
1038
	 * @throws Exception
1039
	 */
1040
	public static Date combinateCurrentDateAndGivenTime(String givenTime) throws HandlerException
1041 2555 tao
  {
1042 5014 daigle
	  try {
1043 2555 tao
     Date givenDate = parseTime(givenTime);
1044
     Date newDate = null;
1045
     Date now = new Date();
1046
     String currentTimeString = getTimeString(now);
1047
     Date currentTime = parseTime(currentTimeString);
1048
     if ( currentTime.getTime() >= givenDate.getTime())
1049
     {
1050 5014 daigle
        logReplication.info("ReplicationHandler.combinateCurrentDateAndGivenTime - Today already pass the given time, we should set it as tomorrow");
1051 2555 tao
        String dateAndTime = getDateString(now) + " " + givenTime;
1052
        Date combinationDate = parseDateTime(dateAndTime);
1053
        // new date should plus 24 hours to make is the second day
1054
        newDate = new Date(combinationDate.getTime()+24*3600*1000);
1055
     }
1056
     else
1057
     {
1058 5014 daigle
         logReplication.info("ReplicationHandler.combinateCurrentDateAndGivenTime - Today haven't pass the given time, we should it as today");
1059 2555 tao
         String dateAndTime = getDateString(now) + " " + givenTime;
1060
         newDate = parseDateTime(dateAndTime);
1061
     }
1062 5014 daigle
     logReplication.warn("ReplicationHandler.combinateCurrentDateAndGivenTime - final setting time is "+ newDate.toString());
1063 2555 tao
     return newDate;
1064 5014 daigle
	  } catch (ParseException pe) {
1065
		  throw new HandlerException("ReplicationHandler.combinateCurrentDateAndGivenTime - "
1066
				  + "parsing error: "  + pe.getMessage());
1067
	  }
1068 2555 tao
  }
1069 2286 tao
1070 2555 tao
  /*
1071 5014 daigle
	 * parse a given string to Time in short format. For example, given time is
1072
	 * 10:00 AM, the date will be return as Jan 1 1970, 10:00 AM
1073
	 */
1074
  private static Date parseTime(String timeString) throws ParseException
1075 2555 tao
  {
1076
    DateFormat format = DateFormat.getTimeInstance(DateFormat.SHORT);
1077
    Date time = format.parse(timeString);
1078 5014 daigle
    logReplication.info("ReplicationHandler.parseTime - Date string is after parse a time string "
1079 2663 sgarg
                              +time.toString());
1080 2555 tao
    return time;
1081
1082
  }
1083
1084
  /*
1085 5175 daigle
   * Parse a given string to date and time. Date format is long and time
1086 2555 tao
   * format is short.
1087
   */
1088 5014 daigle
  private static Date parseDateTime(String timeString) throws ParseException
1089 2555 tao
  {
1090
    DateFormat format = DateFormat.getDateTimeInstance(DateFormat.LONG, DateFormat.SHORT);
1091
    Date time = format.parse(timeString);
1092 5014 daigle
    logReplication.info("ReplicationHandler.parseDateTime - Date string is after parse a time string "+
1093 2663 sgarg
                             time.toString());
1094 2555 tao
    return time;
1095
  }
1096
1097
  /*
1098
   * Get a date string from a Date object. The date format will be long
1099
   */
1100
  private static String getDateString(Date now)
1101
  {
1102
     DateFormat df = DateFormat.getDateInstance(DateFormat.LONG);
1103
     String s = df.format(now);
1104 5014 daigle
     logReplication.info("ReplicationHandler.getDateString - Today is " + s);
1105 2555 tao
     return s;
1106
  }
1107
1108
  /*
1109
   * Get a time string from a Date object, the time format will be short
1110
   */
1111
  private static String getTimeString(Date now)
1112
  {
1113
     DateFormat df = DateFormat.getTimeInstance(DateFormat.SHORT);
1114
     String s = df.format(now);
1115 5014 daigle
     logReplication.info("ReplicationHandler.getTimeString - Time is " + s);
1116 2555 tao
     return s;
1117
  }
1118 2608 tao
1119
1120
  /*
1121 4080 daigle
	 * This method will go through the docid list both in xml_Documents table
1122
	 * and in xml_revisions table @author tao
1123
	 */
1124 5014 daigle
	private void handleDocList(Vector<Vector<String>> docList, String tableName) {
1125 4080 daigle
		boolean dataFile = false;
1126
		for (int j = 0; j < docList.size(); j++) {
1127
			// initial dataFile is false
1128
			dataFile = false;
1129
			// w is information for one document, information contain
1130
			// docid, rev, server or datafile.
1131 5014 daigle
			Vector<String> w = new Vector<String>(docList.elementAt(j));
1132 4080 daigle
			// Check if the vector w contain "datafile"
1133
			// If it has, this document is data file
1134
			try {
1135 4173 daigle
				if (w.contains((String) PropertyService.getProperty("replication.datafileflag"))) {
1136 4080 daigle
					dataFile = true;
1137
				}
1138
			} catch (PropertyNotFoundException pnfe) {
1139 5014 daigle
				logMetacat.error("ReplicationHandler.handleDocList - " + ReplicationService.METACAT_REPL_ERROR_MSG);
1140
				logReplication.error("ReplicationHandler.handleDocList - Could not retrieve data file flag property.  "
1141 4080 daigle
						+ "Leaving as false: " + pnfe.getMessage());
1142
			}
1143
			// System.out.println("w: " + w.toString());
1144
			// Get docid
1145
			String docid = (String) w.elementAt(0);
1146 5014 daigle
			logReplication.info("docid: " + docid);
1147 4080 daigle
			// Get revision number
1148
			int rev = Integer.parseInt((String) w.elementAt(1));
1149 5014 daigle
			logReplication.info("rev: " + rev);
1150 5175 daigle
			// Get remote server name (it is may not be doc home server because
1151 4080 daigle
			// the new hub feature
1152
			String remoteServer = (String) w.elementAt(2);
1153
			remoteServer = remoteServer.trim();
1154 2608 tao
1155 4080 daigle
			try {
1156
				if (tableName.equals(DocumentImpl.DOCUMENTTABLE)) {
1157
					handleDocInXMLDocuments(docid, rev, remoteServer, dataFile);
1158
				} else if (tableName.equals(DocumentImpl.REVISIONTABLE)) {
1159
					handleDocInXMLRevisions(docid, rev, remoteServer, dataFile);
1160
				} else {
1161
					continue;
1162
				}
1163 2608 tao
1164 4861 daigle
			} catch (Exception e) {
1165 5014 daigle
				logMetacat.error("ReplicationHandler.handleDocList - " + ReplicationService.METACAT_REPL_ERROR_MSG);
1166
				logReplication.error("ReplicationHandler.handleDocList - error to handle update doc in " + tableName
1167 4861 daigle
						+ " in time replication" + e.getMessage());
1168 4080 daigle
				continue;
1169
			}
1170 5175 daigle
1171
	        if (_xmlDocQueryCount > 0 && (_xmlDocQueryCount % 100) == 0) {
1172
	        	logMetacat.debug("ReplicationHandler.update - xml_doc query count: " + _xmlDocQueryCount +
1173
	        			", xml_doc avg query time: " + (_xmlDocQueryTime / _xmlDocQueryCount));
1174
	        }
1175
1176
	        if (_xmlRevQueryCount > 0 && (_xmlRevQueryCount % 100) == 0) {
1177
	        	logMetacat.debug("ReplicationHandler.update - xml_rev query count: " + _xmlRevQueryCount +
1178
	        			", xml_rev avg query time: " + (_xmlRevQueryTime / _xmlRevQueryCount));
1179
	        }
1180 4080 daigle
1181
		}// for update docs
1182
1183
	}
1184 2608 tao
1185
   /*
1186 4080 daigle
	 * This method will handle doc in xml_documents table.
1187
	 */
1188 2608 tao
   private void handleDocInXMLDocuments(String docid, int rev, String remoteServer, boolean dataFile)
1189 5014 daigle
                                        throws HandlerException
1190 2608 tao
   {
1191
       // compare the update rev and local rev to see what need happen
1192
       int localrev = -1;
1193
       String action = null;
1194
       boolean flag = false;
1195
       try
1196
       {
1197 5175 daigle
    	 long docQueryStartTime = System.currentTimeMillis();
1198 2641 tao
         localrev = DBUtil.getLatestRevisionInDocumentTable(docid);
1199 5175 daigle
         long docQueryEndTime = System.currentTimeMillis();
1200
         _xmlDocQueryTime += (docQueryEndTime - docQueryStartTime);
1201
         _xmlDocQueryCount++;
1202 2608 tao
       }
1203
       catch (SQLException e)
1204
       {
1205 5014 daigle
    	 logMetacat.error("ReplicationHandler.handleDocInXMLDocuments - " + ReplicationService.METACAT_REPL_ERROR_MSG);
1206
         logReplication.error("ReplicationHandler.handleDocInXMLDocuments - Local rev for docid "+ docid + " could not "+
1207 2663 sgarg
                                " be found because " + e.getMessage());
1208 5014 daigle
         logReplication.error("ReplicationHandler.handleDocInXMLDocuments - " + DOCERRORNUMBER+"Docid "+ docid + " could not be "+
1209 2608 tao
                 "written because error happend to find it's local revision");
1210 2739 tao
         DOCERRORNUMBER++;
1211 5014 daigle
         throw new HandlerException ("ReplicationHandler.handleDocInXMLDocuments - Local rev for docid "+ docid + " could not "+
1212
                 " be found: " + e.getMessage());
1213 2608 tao
       }
1214 5014 daigle
       logReplication.info("ReplicationHandler.handleDocInXMLDocuments - Local rev for docid "+ docid + " is "+
1215 2663 sgarg
                               localrev);
1216 2608 tao
1217
       //check the revs for an update because this document is in the
1218
       //local DB, it might be out of date.
1219
       if (localrev == -1)
1220
       {
1221 2713 tao
          // check if the revision is in the revision table
1222 5014 daigle
    	   Vector<Integer> localRevVector = null;
1223
    	 try {
1224 5175 daigle
        	 long revQueryStartTime = System.currentTimeMillis();
1225 5014 daigle
    		 localRevVector = DBUtil.getRevListFromRevisionTable(docid);
1226 5175 daigle
             long revQueryEndTime = System.currentTimeMillis();
1227
             _xmlRevQueryTime += (revQueryEndTime - revQueryStartTime);
1228
             _xmlRevQueryCount++;
1229 5014 daigle
    	 } catch (SQLException sqle) {
1230
    		 throw new HandlerException("ReplicationHandler.handleDocInXMLDocuments - SQL error "
1231
    				 + " when getting rev list for docid: " + docid + " : " + sqle.getMessage());
1232
    	 }
1233 2713 tao
         if (localRevVector != null && localRevVector.contains(new Integer(rev)))
1234
         {
1235
             // this version was deleted, so don't need replicate
1236
             flag = false;
1237
         }
1238
         else
1239
         {
1240
           //insert this document as new because it is not in the local DB
1241
           action = "INSERT";
1242
           flag = true;
1243
         }
1244 2608 tao
       }
1245
       else
1246
       {
1247
         if(localrev == rev)
1248
         {
1249
           // Local meatacat has the same rev to remote host, don't need
1250
           // update and flag set false
1251
           flag = false;
1252
         }
1253
         else if(localrev < rev)
1254
         {
1255
           //this document needs to be updated so send an read request
1256
           action = "UPDATE";
1257
           flag = true;
1258
         }
1259
       }
1260 2641 tao
1261 5014 daigle
       String accNumber = null;
1262
       try {
1263
    	   accNumber = docid + PropertyService.getProperty("document.accNumSeparator") + rev;
1264
       } catch (PropertyNotFoundException pnfe) {
1265
    	   throw new HandlerException("ReplicationHandler.handleDocInXMLDocuments - error getting "
1266
    			   + "account number separator : " + pnfe.getMessage());
1267
       }
1268 2608 tao
       // this is non-data file
1269
       if(flag && !dataFile)
1270
       {
1271
         try
1272
         {
1273 2641 tao
           handleSingleXMLDocument(remoteServer, action, accNumber, DocumentImpl.DOCUMENTTABLE);
1274 2608 tao
         }
1275 5014 daigle
         catch(HandlerException he)
1276 2608 tao
         {
1277
           // skip this document
1278 5014 daigle
           throw he;
1279 2608 tao
         }
1280
       }//if for non-data file
1281
1282
        // this is for data file
1283
       if(flag && dataFile)
1284
       {
1285
         try
1286
         {
1287 2641 tao
           handleSingleDataFile(remoteServer, action, accNumber, DocumentImpl.DOCUMENTTABLE);
1288 2608 tao
         }
1289 5014 daigle
         catch(HandlerException he)
1290 2608 tao
         {
1291 5175 daigle
           // skip this data file
1292 5014 daigle
           throw he;
1293 2608 tao
         }
1294
1295 5175 daigle
       }//for data file
1296 2608 tao
   }
1297
1298
   /*
1299
    * This method will handle doc in xml_documents table.
1300
    */
1301
   private void handleDocInXMLRevisions(String docid, int rev, String remoteServer, boolean dataFile)
1302 5014 daigle
                                        throws HandlerException
1303 2608 tao
   {
1304
       // compare the update rev and local rev to see what need happen
1305 5014 daigle
       logReplication.info("ReplicationHandler.handleDocInXMLRevisions - In handle repliation revsion table");
1306
       logReplication.info("ReplicationHandler.handleDocInXMLRevisions - the docid is "+ docid);
1307
       logReplication.info("ReplicationHandler.handleDocInXMLRevisions - The rev is "+rev);
1308
       Vector<Integer> localrev = null;
1309 2608 tao
       String action = "INSERT";
1310
       boolean flag = false;
1311
       try
1312
       {
1313 5175 daigle
      	 long revQueryStartTime = System.currentTimeMillis();
1314 2608 tao
         localrev = DBUtil.getRevListFromRevisionTable(docid);
1315 5175 daigle
         long revQueryEndTime = System.currentTimeMillis();
1316
         _xmlRevQueryTime += (revQueryEndTime - revQueryStartTime);
1317
         _xmlRevQueryCount++;
1318 2608 tao
       }
1319 5014 daigle
       catch (SQLException sqle)
1320 2608 tao
       {
1321 5014 daigle
    	 logMetacat.error("ReplicationHandler.handleDocInXMLDocuments - " + ReplicationService.METACAT_REPL_ERROR_MSG);
1322
         logReplication.error("ReplicationHandler.handleDocInXMLRevisions - Local rev for docid "+ docid + " could not "+
1323
                                " be found because " + sqle.getMessage());
1324 2739 tao
         REVERRORNUMBER++;
1325 5014 daigle
         throw new HandlerException ("ReplicationHandler.handleDocInXMLRevisions - SQL exception getting rev list: "
1326
        		 + sqle.getMessage());
1327 2608 tao
       }
1328 5014 daigle
       logReplication.info("ReplicationHandler.handleDocInXMLRevisions - rev list in xml_revision table for docid "+ docid + " is "+
1329 2663 sgarg
                               localrev.toString());
1330 2608 tao
1331
       // if the rev is not in the xml_revision, we need insert it
1332
       if (!localrev.contains(new Integer(rev)))
1333
       {
1334
           flag = true;
1335
       }
1336 2641 tao
1337 5014 daigle
       String accNumber = null;
1338
       try {
1339
    	   accNumber = docid + PropertyService.getProperty("document.accNumSeparator") + rev;
1340
       } catch (PropertyNotFoundException pnfe) {
1341
    	   throw new HandlerException("ReplicationHandler.handleDocInXMLRevisions - error getting "
1342
    			   + "account number separator : " + pnfe.getMessage());
1343
       }
1344 2608 tao
       // this is non-data file
1345
       if(flag && !dataFile)
1346
       {
1347
         try
1348
         {
1349 2641 tao
1350
           handleSingleXMLDocument(remoteServer, action, accNumber, DocumentImpl.REVISIONTABLE);
1351 2608 tao
         }
1352 5014 daigle
         catch(HandlerException he)
1353 2608 tao
         {
1354
           // skip this document
1355 5014 daigle
           throw he;
1356 2608 tao
         }
1357
       }//if for non-data file
1358
1359
        // this is for data file
1360
       if(flag && dataFile)
1361
       {
1362
         try
1363
         {
1364 2641 tao
           handleSingleDataFile(remoteServer, action, accNumber, DocumentImpl.REVISIONTABLE);
1365 2608 tao
         }
1366 5014 daigle
         catch(HandlerException he)
1367 2608 tao
         {
1368 5175 daigle
           // skip this data file
1369 5014 daigle
           throw he;
1370 2608 tao
         }
1371
1372 5175 daigle
       }//for data file
1373 2608 tao
   }
1374 3234 tao
1375
   /*
1376
    * Return a ip address for given url
1377
    */
1378
   private String getIpFromURL(URL url)
1379
   {
1380
	   String ip = null;
1381
	   try
1382
	   {
1383
	      InetAddress address = InetAddress.getByName(url.getHost());
1384
	      ip = address.getHostAddress();
1385
	   }
1386
	   catch(UnknownHostException e)
1387
	   {
1388 5014 daigle
		   logMetacat.error("ReplicationHandler.getIpFromURL - " + ReplicationService.METACAT_REPL_ERROR_MSG);
1389
		   logReplication.error("ReplicationHandler.getIpFromURL - Error in get ip address for host: "
1390 3234 tao
                   +e.getMessage());
1391
	   }
1392
1393
	   return ip;
1394
   }
1395 2608 tao
1396 522 berkley
}