Project

General

Profile

« Previous | Next » 

Revision 590

Added by berkley over 23 years ago

added replication for the xml_catalog table. Right now it is only replicated when the deltaT handler is called. I will add more replication times later.

View differences:

src/edu/ucsb/nceas/metacat/MetacatReplication.java
144 144
      {
145 145
        handleGetTimeRequest(out, params, response);
146 146
      }
147
      else if(((String[])params.get("action"))[0].equals("getcatalog"))
148
      {
149
        handleGetCatalogRequest(out, params, response, true);
150
      }
147 151
    }
148 152
  }
149 153
  
......
172 176
        serverCode = MetacatReplication.getServerCode(server);
173 177
        override = true; //we are now overriding the default action
174 178
      }
175
      //System.out.println("action in forcereplicate is: " + dbaction);
176
      //System.out.println("serverCode in forcereplicate is: " + serverCode);
179
      MetaCatUtil.debugMessage("action in forcereplicate is: " + dbaction);
180
      MetaCatUtil.debugMessage("serverCode in forcereplicate is: " + serverCode);
177 181
      MetacatReplication.replLog("force replication request from " + server); 
178 182
      
179 183
      int serverCheckCode = MetacatReplication.getServerCode(server);
180 184
      URL u = new URL("http://" + server + "?action=read&docid=" + docid);
181
      //System.out.println("sending message: " + u.toString());
185
      MetaCatUtil.debugMessage("sending message: " + u.toString());
182 186
      String xmldoc = MetacatReplication.getURLContent(u);
187
      MetaCatUtil.debugMessage("document: " + xmldoc);
183 188
      //get the document to write
184 189
      URL docinfourl = new URL("http://" + server + 
185 190
                               "?action=getdocumentinfo&docid=" +
186 191
                               docid);
187 192
      //we need to get the document's info so we can set the correct user
188 193
      //and group once we get the document and write it to our DB
189
      //System.out.println("sending message: " + docinfourl.toString());
194
      MetaCatUtil.debugMessage("sending message: " + docinfourl.toString());
190 195
      String docInfoStr = MetacatReplication.getURLContent(docinfourl);
196
      MetaCatUtil.debugMessage("docInfo: " + docInfoStr);
191 197
      DocInfoHandler dih = new DocInfoHandler();
192 198
      //dih is the parser for the docinfo xml format
193 199
      XMLReader docinfoParser = ReplicationHandler.initParser(dih);
......
310 316
    try
311 317
    {
312 318
      String docid = ((String[])(params.get("docid")))[0];
313
      //System.out.println("incoming get request for document: " + docid);
319
      MetaCatUtil.debugMessage("incoming get request for document: " + docid);
314 320
      Connection conn = util.openDBConnection();
315 321
      DocumentImpl di = new DocumentImpl(conn, docid);
316 322
      response.setContentType("text/xml");
......
344 350
  {
345 351
    try
346 352
    {
347
      //System.out.println("received update request");
353
      MetaCatUtil.debugMessage("received update request");
348 354
      StringBuffer docsql = new StringBuffer();
349 355
      StringBuffer doclist = new StringBuffer();
350 356
      
......
391 397
      }
392 398
      
393 399
      doclist.append("</updates></replication>");
394
      //System.out.println("doclist: " + doclist.toString());
400
      MetaCatUtil.debugMessage("doclist: " + doclist.toString());
395 401
      conn.close();
396 402
      response.setContentType("text/xml");
397 403
      out.println(doclist.toString());
......
406 412
  }
407 413
  
408 414
  /**
415
   * Returns the xml_catalog table encoded in xml
416
   */
417
  public static String getCatalogXML()
418
  {
419
    return handleGetCatalogRequest(null, null, null, false);
420
  }
421
  
422
  /**
423
   * Sends the contents of the xml_catalog table encoded in xml
424
   * The xml format is:
425
   * <!ELEMENT xml_catalog (row*)>
426
   * <!ELEMENT row (entry_type, source_doctype, target_doctype, public_id,
427
   *                system_id)>
428
   * All of the sub elements of row are #PCDATA
429
   
430
   * If printFlag == false then do not print to out.
431
   */
432
  private static String handleGetCatalogRequest(PrintWriter out, 
433
                                                Hashtable params,
434
                                                HttpServletResponse response,
435
                                                boolean printFlag)
436
  {
437
    try
438
    {
439
      Connection conn = util.openDBConnection();
440
      PreparedStatement pstmt = conn.prepareStatement("select entry_type, " +
441
                              "source_doctype, target_doctype, public_id, " +
442
                              "system_id from xml_catalog");
443
      pstmt.execute();
444
      ResultSet rs = pstmt.getResultSet();
445
      boolean tablehasrows = rs.next();
446
      StringBuffer sb = new StringBuffer();
447
      sb.append("<?xml version=\"1.0\"?><xml_catalog>");
448
      while(tablehasrows)
449
      {
450
        sb.append("<row><entry_type>").append(rs.getString(1));
451
        sb.append("</entry_type><source_doctype>").append(rs.getString(2));
452
        sb.append("</source_doctype><target_doctype>").append(rs.getString(3));
453
        sb.append("</target_doctype><public_id>").append(rs.getString(4));
454
        sb.append("</public_id><system_id>").append(rs.getString(5));
455
        sb.append("</system_id></row>");
456
      
457
        tablehasrows = rs.next();
458
      }
459
      sb.append("</xml_catalog>");
460
      conn.close();
461
      if(printFlag)
462
      {
463
        response.setContentType("text/xml");
464
        out.println(sb.toString());
465
      }
466
      return sb.toString();
467
    }
468
    catch(Exception e)
469
    {
470
      System.out.println("error in handleGetCatalogRequest: " + e.getMessage());
471
      e.printStackTrace(System.out);
472
    }
473
    return null;
474
  }
475
  
476
  /**
409 477
   * Sends the current system date to the remote server.  Using this action
410 478
   * for replication gets rid of any problems with syncronizing clocks 
411 479
   * because a time specific to a document is always kept on its home server.
......
430 498
  {
431 499
    try
432 500
    {
433
      //System.out.println("thread started for docid: " + 
434
      //                   (String)fileLocks.elementAt(0));
501
      MetaCatUtil.debugMessage("thread started for docid: " + 
502
                               (String)fileLocks.elementAt(0));
435 503
      Thread.sleep(30000); //the lock will expire in 30 seconds
436
      //System.out.println("thread for docid: " + 
437
      //                   (String)fileLocks.elementAt(fileLocks.size() - 1) + 
438
      //                   " exiting.");
504
      MetaCatUtil.debugMessage("thread for docid: " + 
505
                            (String)fileLocks.elementAt(fileLocks.size() - 1) + 
506
                            " exiting.");
439 507
      fileLocks.remove(fileLocks.size() - 1);
440 508
      //fileLocks is treated as a FIFO queue.  If there are more than one lock
441 509
      //in the vector, the first one inserted will be removed.
src/edu/ucsb/nceas/metacat/DocumentImpl.java
767 767
    AccessionNumber ac = new AccessionNumber(conn);
768 768
    String newdocid = ac.generate(docid, action);
769 769
    
770
    //System.out.println("action: " + action + " servercode: " + 
771
    //                    serverCode + " override: " + override);
770
    MetaCatUtil.debugMessage("action: " + action + " servercode: " + 
771
                             serverCode + " override: " + override);
772 772
                        
773 773
    if((serverCode != 1 && action.equals("UPDATE")) && !override)
774 774
    { //if this document being written is not a resident of this server then
......
822 822
        //after inserting the document locally, tell the document's home server
823 823
        //to come get a copy from here.
824 824
        ForceReplicationHandler frh = new ForceReplicationHandler(docid);
825
        /*
826
        URL comeAndGetIt = new URL("http://" + server + 
827
                                   "?action=forcereplicate&server=" + 
828
                                   util.getOption("server") + 
829
                                   util.getOption("replicationpath") +
830
                                   "&docid=" + docid);
831
        System.out.println("sending message: " + comeAndGetIt.toString());
832
        String message = MetacatReplication.getURLContent(comeAndGetIt);
833
        */
825
        
834 826
        if ( (docid != null) && !(newdocid.equals(docid)) ) 
835 827
        {
836 828
          return new String("New document ID generated:" + newdocid);
src/edu/ucsb/nceas/metacat/ReplicationHandler.java
71 71
      conn = util.openDBConnection();
72 72
      serverList = buildServerList(conn);
73 73
      update(serverList, conn);
74
      updateCatalog(serverList, conn);
74 75
      conn.close();
75 76
    }
76 77
    catch (Exception e)
......
114 115
    
115 116
    try
116 117
    {
117
      //System.out.println("init parser");
118
      MetaCatUtil.debugMessage("init parser");
118 119
      parser = initParser(message);
119 120
      keys = serverList.keys();
120 121
      while(keys.hasMoreElements())
121 122
      {
122
        //System.out.println("get responses");
123
        MetaCatUtil.debugMessage("get responses");
123 124
        server = (String)(keys.nextElement());
124 125
        MetacatReplication.replLog("full update started to: " + server);
125 126
        u = new URL("http://" + server + "?action=update");
126
        //System.out.println(u.toString());
127
        MetaCatUtil.debugMessage(u.toString());
127 128
        String result = MetacatReplication.getURLContent(u);
128
        //System.out.println(result);
129
        MetaCatUtil.debugMessage(result);
129 130
        responses.add(result);
130 131
      }
131 132
      
132
      //System.out.println("responses: " + responses.toString());
133
      MetaCatUtil.debugMessage("responses: " + responses.toString());
133 134
      
134 135
      for(int i=0; i<responses.size(); i++)
135 136
      { //check each server for updated files
......
196 197
            docinfoParser.parse(new InputSource(new StringReader(docInfoStr)));
197 198
            Hashtable docinfoHash = dih.getDocInfo();
198 199
            int serverCode = MetacatReplication.getServerCode(docServer);
199
            //System.out.println("updating doc: " + docid + " action: "+ action);
200
            MetaCatUtil.debugMessage("updating doc: " + docid + " action: "+ action);
200 201
            String newDocid = DocumentImpl.write(conn, 
201 202
                              new StringReader(newxmldoc),
202 203
                              null,
......
218 219
          if(!alreadyDeleted(docid, conn))
219 220
          {
220 221
            DocumentImpl.delete(conn, docid, null, null);
221
            //System.out.println("Document " + docid + " deleted.");
222
            MetaCatUtil.debugMessage("Document " + docid + " deleted.");
222 223
            MetacatReplication.replLog("doc " + docid + " deleted");
223 224
          }
224 225
        }
......
238 239
          pstmt = conn.prepareStatement(sql.toString());
239 240
          pstmt.executeUpdate();
240 241
          //conn.commit();
241
          //System.out.println("last_checked updated to " + datestr + " on " +
242
          //                  server);
242
          MetaCatUtil.debugMessage("last_checked updated to " + datestr + " on " +
243
                            server);
243 244
        }
244 245
      }
245 246
    }
......
251 252
  }
252 253
  
253 254
  /**
255
   * updates xml_catalog with entries from other servers.
256
   */
257
  private void updateCatalog(Hashtable serverList, Connection conn)
258
  {
259
    System.out.println("in updateCatalog");
260
    try
261
    {
262
      String server;
263
      Enumeration keys = serverList.keys();
264
      while(keys.hasMoreElements())
265
      { //go through each server
266
        server = (String)(keys.nextElement());
267
        System.out.println("server: " + server);
268
        URL u = new URL("http://" + server + "?action=getcatalog");
269
        String catxml = MetacatReplication.getURLContent(u);
270
        System.out.println("catxml: " + catxml);
271
        CatalogMessageHandler cmh = new CatalogMessageHandler();
272
        XMLReader catparser = initParser(cmh);
273
        catparser.parse(new InputSource(new StringReader(catxml)));
274
        //parse the returned catalog xml and put it into a vector
275
        Vector remoteCatalog = cmh.getCatalogVect();
276
        
277
        String localcatxml = MetacatReplication.getCatalogXML();
278
        System.out.println("localcatxml: " + localcatxml);
279
        cmh = new CatalogMessageHandler();
280
        catparser = initParser(cmh);
281
        catparser.parse(new InputSource(new StringReader(localcatxml)));
282
        Vector localCatalog = cmh.getCatalogVect();
283
        
284
        //now we have the catalog from the remote server and this local server
285
        //we now need to compare the two and merge the differences.
286
        //the comparison is base on the public_id fields which is the 4th
287
        //entry in each row vector.
288
        Vector publicId = new Vector();
289
        for(int i=0; i<localCatalog.size(); i++)
290
        {
291
          Vector v = new Vector((Vector)localCatalog.elementAt(i));
292
          System.out.println("v1: " + v.toString());
293
          publicId.add(new String((String)v.elementAt(3)));
294
          System.out.println("adding " + (String)v.elementAt(3));
295
        }
296
        
297
        for(int i=0; i<remoteCatalog.size(); i++)
298
        {
299
          Vector v = (Vector)remoteCatalog.elementAt(i);
300
          System.out.println("v2: " + v.toString());
301
          //System.out.println("i: " + i);
302
          //System.out.println("remoteCatalog.size(): " + remoteCatalog.size());
303
          System.out.println("publicID: " + publicId.toString());
304
          System.out.println("v.elementAt(3): " + (String)v.elementAt(3));
305
          if(!publicId.contains(v.elementAt(3)))
306
          { //so we don't have this public id in our local table so we need to
307
            //add it.
308
            System.out.println("in if");
309
            StringBuffer sql = new StringBuffer();
310
            sql.append("insert into xml_catalog (entry_type, source_doctype, ");
311
            sql.append("target_doctype, public_id, system_id) values (?,?,?,");
312
            sql.append("?,?)");
313
            System.out.println("sql: " + sql.toString());
314
            PreparedStatement pstmt = conn.prepareStatement(sql.toString());
315
            pstmt.setString(1, (String)v.elementAt(0));
316
            pstmt.setString(2, (String)v.elementAt(1));
317
            pstmt.setString(3, (String)v.elementAt(2));
318
            pstmt.setString(4, (String)v.elementAt(3));
319
            pstmt.setString(5, (String)v.elementAt(4));
320
            pstmt.execute();
321
          }
322
        }
323
      }
324
    }
325
    catch(Exception e)
326
    {
327
      System.out.println("error in updateCatalog: " + e.getMessage());
328
      e.printStackTrace(System.out);
329
    }
330
  }
331
  
332
  /**
254 333
   * Method that returns true if docid has already been "deleted" from metacat.
255 334
   * This method really implements a truth table for deleted documents
256
   * The table is (a document in one of the tables is represented by the X):
335
   * The table is (a docid in one of the tables is represented by the X):
257 336
   * xml_docs      xml_revs      deleted?
258 337
   * ------------------------------------
259 338
   *   X             X             FALSE
src/edu/ucsb/nceas/metacat/ForceReplicationHandler.java
95 95
                                 util.getOption("replicationpath") +
96 96
                                 "&docid=" + docid);
97 97
        }
98
        //System.out.println("sending message: " + comeAndGetIt.toString());
98
        MetaCatUtil.debugMessage("sending message: " + comeAndGetIt.toString());
99 99
        String message = MetacatReplication.getURLContent(comeAndGetIt);
100 100
        //send out the url.  message is a dummy variable as the target of
101 101
        //the URL never directly replies to the request.  this simply
......
108 108
      System.out.println("error in ForceReplicationHandler: " + e.getMessage());
109 109
      e.printStackTrace(System.out);
110 110
    }
111
    //System.out.println("exiting ForceReplicationHandler Thread");
111
    MetaCatUtil.debugMessage("exiting ForceReplicationHandler Thread");
112 112
  }
113 113
}

Also available in: Unified diff