Metacat

/**

 *  '$RCSfile$'

 *    Purpose: A Class that implements replication for metacat

 *  Copyright: 2000 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *    Authors: Chad Berkley

 *    Release: @release@

 *

 *   '$Author: berkley $'

 *     '$Date: 2000-12-12 10:55:33 -0800 (Tue, 12 Dec 2000) $'

 * '$Revision: 595 $'

 */

package edu.ucsb.nceas.metacat;

import java.util.*;

import java.io.*;

import java.sql.*;

import java.net.*;

import java.lang.*;

import java.text.*;

import javax.servlet.*;

import javax.servlet.http.*;

import org.xml.sax.*;

public class MetacatReplication extends HttpServlet implements Runnable

{  

  private String deltaT;

  Timer replicationDaemon;

  private static MetaCatUtil util = new MetaCatUtil();

  private Vector fileLocks = new Vector();

  private Thread lockThread = null;

  /**

   * Initialize the servlet by creating appropriate database connections

   */

  public void init(ServletConfig config) throws ServletException 

  {

    //initialize db connections to handle any update requests

    MetaCatUtil util = new MetaCatUtil();

    deltaT = util.getOption("deltaT");

    //the default deltaT can be set from metacat.properties

    //create a thread to do the delta-T check but don't execute it yet

    replicationDaemon = new Timer(true);

  }

  public void destroy() 

  {

    replicationDaemon.cancel();

    System.out.println("Replication daemon cancelled.");

  }

  public void doGet (HttpServletRequest request, HttpServletResponse response)

                     throws ServletException, IOException 

  {

    // Process the data and send back the response

    handleGetOrPost(request, response);

  }

  public void doPost(HttpServletRequest request, HttpServletResponse response)

                     throws ServletException, IOException 

  {

    // Process the data and send back the response

    handleGetOrPost(request, response);

  }

  private void handleGetOrPost(HttpServletRequest request, 

                               HttpServletResponse response) 

                               throws ServletException, IOException 

  {

    PrintWriter out = response.getWriter();

    Hashtable params = new Hashtable();

    Enumeration paramlist = request.getParameterNames();

    while (paramlist.hasMoreElements()) 

    {

      String name = (String)paramlist.nextElement();

      String[] value = request.getParameterValues(name);

      params.put(name, value);  

    }

    if(params.containsKey("action"))

    {

      if(((String[])params.get("action"))[0].equals("stop"))

      { //stop the replication server

        replicationDaemon.cancel();

        out.println("Replication Handler Stopped");

        System.out.println("Replication Handler Stopped");

        MetacatReplication.replLog("deltaT handler stopped");

      }

      else if(((String[])params.get("action"))[0].equals("start"))

      { //start the replication server

        int rate;

        if(params.containsKey("rate"))

        {

          rate = new Integer(

                 new String(((String[])params.get("rate"))[0])).intValue();

          if(rate < 30)

          {

            out.println("Replication deltaT rate cannot be less than 30!");

            //deltaT<30 is a timing mess!

            rate = 1000;

          }

        }

        else

        {

          rate = 1000;

        }

        out.println("New rate is: " + rate + " seconds.");

        replicationDaemon.cancel();

        replicationDaemon = new Timer(true);

        replicationDaemon.scheduleAtFixedRate(new ReplicationHandler(out), 0, 

                                              rate * 1000);

        MetacatReplication.replLog("deltaT handler started with rate=" + 

                                    rate + " seconds");

        out.println("Replication Handler Started");

        System.out.println("Replication Handler Started");

      }

      else if(((String[])params.get("action"))[0].equals("getall"))

      { //updates this server exactly once

        replicationDaemon.schedule(new ReplicationHandler(out), 0);

        out.println("<html><body>\"Get All\" Done</body></html>");

      }

      else if(((String[])params.get("action"))[0].equals("forcereplicate"))

      {

        handleForceReplicateRequest(out, params, response);

      }

      else if(((String[])params.get("action"))[0].equals("update"))

      { //request an update list from the server

        handleUpdateRequest(out, params, response);

      }

      else if(((String[])params.get("action"))[0].equals("read"))

      { //request a specific document from the server

        //note that this could be replaced by a call to metacatServlet

        //handleGetDocumentAction().

        handleGetDocumentRequest(out, params, response);

      }

      else if(((String[])params.get("action"))[0].equals("getlock"))

      {

        handleGetLockRequest(out, params, response);

      }

      else if(((String[])params.get("action"))[0].equals("getdocumentinfo"))

      {

        handleGetDocumentInfoRequest(out, params, response);

      }

      else if(((String[])params.get("action"))[0].equals("gettime"))

      {

        handleGetTimeRequest(out, params, response);

      }

      else if(((String[])params.get("action"))[0].equals("getcatalog"))

      {

        handleGetCatalogRequest(out, params, response, true);

      }

      else if(((String[])params.get("action"))[0].equals("servercontrol"))

      {

        handleServerControlRequest(out, params, response);

      }

    }

  }

  /** 

   * This method can add, delete and list the servers currently included in

   * xml_replication.

   * action           subaction            other needed params

   * ---------------------------------------------------------

   * servercontrol    add                  server

   * servercontrol    delete               server

   * servercontrol    list                 

   */

  private void handleServerControlRequest(PrintWriter out, Hashtable params,

                                          HttpServletResponse response)

  {

    String subaction = ((String[])params.get("subaction"))[0];

    try

    {

      Connection conn = util.openDBConnection();

      PreparedStatement pstmt;

      if(subaction.equals("add"))

      {

        String server = ((String[])params.get("server"))[0];

        pstmt = conn.prepareStatement("insert into xml_replication (server, " +

                "last_checked) values ('" + server + "', to_date(" +

                "'01/01/00', 'MM/DD/YY'))");

        pstmt.execute();

        out.println("server " + server + " added");

      }

      else if(subaction.equals("delete"))

      {

        String server = ((String[])params.get("server"))[0];

        pstmt = conn.prepareStatement("delete from xml_replication where " +

                "server like '" + server + "'");

        pstmt.execute();

        out.println("server " + server + " deleted");

      }

      else if(subaction.equals("list"))

      {

        response.setContentType("text/html");

        out.println("<html><body><table border=\"1\">");

        out.println("<tr><td><b>server</b></td><td><b>last_checked</b></td></tr>");

        pstmt = conn.prepareStatement("select * from xml_replication");

        pstmt.execute();

        ResultSet rs = pstmt.getResultSet();

        boolean tablehasrows = rs.next();

        while(tablehasrows)

        {

          out.println("<tr><td>" + rs.getString(2) + "</td><td>");

          out.println(rs.getString(3) + "</td></tr>");

          tablehasrows = rs.next();

        }

        out.println("</table></body></html>");

      }

      conn.close();

    }

    catch(Exception e)

    {

      System.out.println("error in handleServerControlRequest " + 

                         e.getMessage());

      e.printStackTrace(System.out);

    }

  }

  /**

   * when a forcereplication request comes in, this method sends a read request

   * to the requesting server for the specified docid.

   */

  private void handleForceReplicateRequest(PrintWriter out, Hashtable params,

                                           HttpServletResponse response)

  {

    //System.out.println("in handleforcereplicaterequest");

    String server = ((String[])params.get("server"))[0];

    //the server that the request came from

    String docid = ((String[])params.get("docid"))[0];

    //the docid of the document to get

    String dbaction = "UPDATE";

    //default action is update

    boolean override = false;

    int serverCode = 1;

    try

    {

      if(params.containsKey("dbaction"))

      { //if the url contains a dbaction then the default action is overridden

        dbaction = ((String[])params.get("dbaction"))[0];

        serverCode = MetacatReplication.getServerCode(server);

        override = true; //we are now overriding the default action

      }

      MetaCatUtil.debugMessage("action in forcereplicate is: " + dbaction);

      MetaCatUtil.debugMessage("serverCode in forcereplicate is: " + serverCode);

      MetacatReplication.replLog("force replication request from " + server); 

      int serverCheckCode = MetacatReplication.getServerCode(server);

      URL u = new URL("http://" + server + "?action=read&docid=" + docid);

      MetaCatUtil.debugMessage("sending message: " + u.toString());

      String xmldoc = MetacatReplication.getURLContent(u);

      MetaCatUtil.debugMessage("document: " + xmldoc);

      //get the document to write

      URL docinfourl = new URL("http://" + server + 

                               "?action=getdocumentinfo&docid=" +

                               docid);

      //we need to get the document's info so we can set the correct user

      //and group once we get the document and write it to our DB

      MetaCatUtil.debugMessage("sending message: " + docinfourl.toString());

      String docInfoStr = MetacatReplication.getURLContent(docinfourl);

      MetaCatUtil.debugMessage("docInfo: " + docInfoStr);

      DocInfoHandler dih = new DocInfoHandler();

      //dih is the parser for the docinfo xml format

      XMLReader docinfoParser = ReplicationHandler.initParser(dih);

      docinfoParser.parse(new InputSource(new StringReader(docInfoStr)));

      Hashtable docinfoHash = dih.getDocInfo();

      String user = (String)docinfoHash.get("user_owner");

      String group = new String(user);

      //right now the user and group are the same.

      Connection conn = util.openDBConnection();

      DocumentImpl.write(conn, new StringReader(xmldoc), null, dbaction, docid, 

                         user, group, serverCode, override);

      MetacatReplication.replLog("document " + docid + " added to DB with " +

                                 "action " + dbaction);

      conn.close();

    }

    catch(Exception e)

    {

      System.out.println("error in metacatReplication.handleForceReplicate" +

                         "Request: " + e.getMessage());

    }

  }

  /**

   * Grants or denies a lock to a requesting host.

   * The servlet parameters of interrest are:

   * docid: the docid of the file the lock is being requested for

   * currentdate: the timestamp of the document on the remote server

   * 

   */

  private void handleGetLockRequest(PrintWriter out, Hashtable params,

                                    HttpServletResponse response)

  {

    try

    {

      Connection conn = util.openDBConnection();

      String docid = ((String[])params.get("docid"))[0];

      String remoteRev = ((String[])params.get("updaterev"))[0];

      DocumentImpl requestDoc = new DocumentImpl(conn, docid);

      MetacatReplication.replLog("lock request for " + docid);

      int localRevInt = requestDoc.getRev();

      int remoteRevInt = Integer.parseInt(remoteRev);

      if(remoteRevInt >= localRevInt)

      {

        if(!fileLocks.contains(docid))

        { //grant the lock if it is not already locked

          fileLocks.add(0, docid); //insert at the beginning of the queue Vector

          //send a message back to the the remote host authorizing the insert

          out.println("<lockgranted><docid>" +docid+ "</docid></lockgranted>");

          lockThread = new Thread(this);

          lockThread.setPriority(Thread.MIN_PRIORITY);

          lockThread.start();

          System.out.println("lock granted for " + docid);

          MetacatReplication.replLog("lock granted for " + docid);

        }

        else

        { //deny the lock

          out.println("<filelocked><docid>" + docid + "</docid></filelocked>");

          MetacatReplication.replLog("lock denied for " + docid + 

                                     "reason: file already locked");

        }

      }

      else

      {//deny the lock.

        out.println("<outdatedfile><docid>" + docid + "</docid></filelocked>");

        MetacatReplication.replLog("lock denied for " + docid + 

                                   "reason: client has outdated file");

      }

      conn.close();

    }

    catch(Exception e)

    {

      System.out.println("error requesting file lock: " + e.getMessage());

      e.printStackTrace(System.out);

    }

  }

  /**

   * Sends all of the xml_documents information encoded in xml to a requestor

   * the format is:

   * <!ELEMENT documentinfo (docid, docname, doctype, doctitle, user_owner,

   *                         user_updated, public_access, rev)

   * all of the subelements of document info are #PCDATA

   */

  private void handleGetDocumentInfoRequest(PrintWriter out, Hashtable params, 

                                        HttpServletResponse response)

  {

    String docid = ((String[])(params.get("docid")))[0];

    StringBuffer sb = new StringBuffer();

    try

    {

      Connection conn = util.openDBConnection();

      DocumentImpl doc = new DocumentImpl(conn, docid);

      sb.append("<documentinfo><docid>").append(docid);

      sb.append("</docid><docname>").append(doc.getDocname());

      sb.append("</docname><doctype>").append(doc.getDoctype());

      sb.append("</doctype><doctitle>").append(doc.getDocTitle());

      sb.append("</doctitle><user_owner>").append(doc.getUserowner());

      sb.append("</user_owner><user_updated>").append(doc.getUserupdated());

      sb.append("</user_updated><public_access>").append(doc.getPublicaccess());

      sb.append("</public_access><rev>").append(doc.getRev());

      sb.append("</rev></documentinfo>");

      response.setContentType("text/xml");

      out.println(sb.toString());

      conn.close();

    }

    catch (Exception e)

    {

      System.out.println("error in metacatReplication.handlegetdocumentinforequest: " + 

      e.getMessage());

    }

  }

  /**

   * Sends a document to a remote host

   */

  private void handleGetDocumentRequest(PrintWriter out, Hashtable params, 

                                        HttpServletResponse response)

  {

    try

    {

      String docid = ((String[])(params.get("docid")))[0];

      MetaCatUtil.debugMessage("incoming get request for document: " + docid);

      Connection conn = util.openDBConnection();

      DocumentImpl di = new DocumentImpl(conn, docid);

      response.setContentType("text/xml");

      out.print(di.toString());

      conn.close();

      MetacatReplication.replLog("document " + docid + " sent");

      System.out.println("document " + docid + " sent");

    }

    catch(Exception e)

    {

      System.out.println("error getting document: " + e.getMessage());

    }

  }

  /**

   * Sends a list of all of the documents on this sever along with their

   * revision numbers.  

   * The format is:

   * <!ELEMENT replication (server, updates)>

   * <!ELEMENT server (#PCDATA)>

   * <!ELEMENT updates ((updatedDocument | deleteDocument)*)>

   * <!ELEMENT updatedDocument (docid, rev)>

   * <!ELEMENT deletedDocument (docid, rev)>

   * <!ELEMENT docid (#PCDATA)>

   * <!ELEMENT rev (#PCDATA)>

   * note that the rev in deletedDocument is always empty.  I just left

   * it in there to make the parser implementation easier.

   */

  private void handleUpdateRequest(PrintWriter out, Hashtable params, 

                                    HttpServletResponse response)

  {

    try

    {

      System.out.println("received update request");

      StringBuffer docsql = new StringBuffer();

      StringBuffer doclist = new StringBuffer();

      //get all docs that reside on this server

      doclist.append("<?xml version=\"1.0\"?><replication>");

      doclist.append("<server>").append(util.getOption("server"));

      doclist.append(util.getOption("replicationpath"));

      doclist.append("</server><updates>");

      docsql.append("select docid, rev from xml_documents where "); 

      docsql.append("server_location = 1");

      //get any deleted documents

      StringBuffer delsql = new StringBuffer();

      delsql.append("select distinct docid from ");

      delsql.append("xml_revisions where docid not in (select docid from ");

      delsql.append("xml_documents) and server_location = 1");

      Connection conn = util.openDBConnection();

      PreparedStatement pstmt = conn.prepareStatement(docsql.toString());

      pstmt.execute();

      ResultSet rs = pstmt.getResultSet();

      boolean tablehasrows = rs.next();

      while(tablehasrows)

      {

        doclist.append("<updatedDocument>");

        doclist.append("<docid>").append(rs.getString(1));

        doclist.append("</docid><rev>").append(rs.getInt(2));

        doclist.append("</rev>");

        doclist.append("</updatedDocument>");

        tablehasrows = rs.next();

      }

      pstmt = conn.prepareStatement(delsql.toString());

      pstmt.execute();

      rs = pstmt.getResultSet();

      tablehasrows = rs.next();

      while(tablehasrows)

      { //handle the deleted documents

        doclist.append("<deletedDocument><docid>").append(rs.getString(1));

        doclist.append("</docid><rev></rev></deletedDocument>");

        //note that rev is always empty for deleted docs

        tablehasrows = rs.next();

      }

      doclist.append("</updates></replication>");

      MetaCatUtil.debugMessage("doclist: " + doclist.toString());

      conn.close();

      response.setContentType("text/xml");

      out.println(doclist.toString());

      System.out.println("update request handled");

    }

    catch(Exception e)

    {

      System.out.println("error in handleupdaterequest2: " + e.getMessage());

      e.printStackTrace(System.out);

    }

  }

  /**

   * Returns the xml_catalog table encoded in xml

   */

  public static String getCatalogXML()

  {

    return handleGetCatalogRequest(null, null, null, false);

  }

  /**

   * Sends the contents of the xml_catalog table encoded in xml

   * The xml format is:

   * <!ELEMENT xml_catalog (row*)>

   * <!ELEMENT row (entry_type, source_doctype, target_doctype, public_id,

   *                system_id)>

   * All of the sub elements of row are #PCDATA

   * If printFlag == false then do not print to out.

   */

  private static String handleGetCatalogRequest(PrintWriter out, 

                                                Hashtable params,

                                                HttpServletResponse response,

                                                boolean printFlag)

  {

    try

    {

      Connection conn = util.openDBConnection();

      PreparedStatement pstmt = conn.prepareStatement("select entry_type, " +

                              "source_doctype, target_doctype, public_id, " +

                              "system_id from xml_catalog");

      pstmt.execute();

      ResultSet rs = pstmt.getResultSet();

      boolean tablehasrows = rs.next();

      StringBuffer sb = new StringBuffer();

      sb.append("<?xml version=\"1.0\"?><xml_catalog>");

      while(tablehasrows)

      {

        sb.append("<row><entry_type>").append(rs.getString(1));

        sb.append("</entry_type><source_doctype>").append(rs.getString(2));

        sb.append("</source_doctype><target_doctype>").append(rs.getString(3));

        sb.append("</target_doctype><public_id>").append(rs.getString(4));

        sb.append("</public_id><system_id>").append(rs.getString(5));

        sb.append("</system_id></row>");

        tablehasrows = rs.next();

      }

      sb.append("</xml_catalog>");

      conn.close();

      if(printFlag)

      {

        response.setContentType("text/xml");

        out.println(sb.toString());

      }

      return sb.toString();

    }

    catch(Exception e)

    {

      System.out.println("error in handleGetCatalogRequest: " + e.getMessage());

      e.printStackTrace(System.out);

    }

    return null;

  }

  /**

   * Sends the current system date to the remote server.  Using this action

   * for replication gets rid of any problems with syncronizing clocks 

   * because a time specific to a document is always kept on its home server.

   */

  private void handleGetTimeRequest(PrintWriter out, Hashtable params, 

                                    HttpServletResponse response)

  {

    SimpleDateFormat formatter = new SimpleDateFormat ("yy-MM-dd HH:mm:ss");

    java.util.Date localtime = new java.util.Date();

    String dateString = formatter.format(localtime);

    response.setContentType("text/xml");

    out.println("<timestamp>" + dateString + "</timestamp>");

  }

  /**

   * this method handles the timeout for a file lock.  when a lock is 

   * granted it is granted for 30 seconds.  When this thread runs out

   * it deletes the docid from the queue, thus eliminating the lock.

   */

  public void run()

  {

    try

    {

      MetaCatUtil.debugMessage("thread started for docid: " + 

                               (String)fileLocks.elementAt(0));

      System.out.println("thread started for docid: " + 

                               (String)fileLocks.elementAt(0));

      Thread.sleep(30000); //the lock will expire in 30 seconds

      MetaCatUtil.debugMessage("thread for docid: " + 

                             (String)fileLocks.elementAt(fileLocks.size() - 1) + 

                              " exiting.");

      System.out.println("thread for docid: " + 

                         (String)fileLocks.elementAt(fileLocks.size() - 1) + 

                         " exiting.");

      fileLocks.remove(fileLocks.size() - 1);

      //fileLocks is treated as a FIFO queue.  If there are more than one lock

      //in the vector, the first one inserted will be removed.

    }

    catch(Exception e)

    {

      System.out.println("error in file lock thread: " + e.getMessage());

    }

  }

  /**

   * Returns the name of a server given a serverCode

   * @param serverCode the serverid of the server

   * @return the servername or null if the specified serverCode does not

   *         exist.

   */

  public static String getServer(int serverCode)

  {

    //System.out.println("serverid: " + serverCode);

    try

    {

      Connection conn = util.openDBConnection();

      String sql = new String("select server from " +

                              "xml_replication where serverid = " + 

                              serverCode);

      PreparedStatement pstmt = conn.prepareStatement(sql);

      //System.out.println("getserver sql: " + sql);

      pstmt.execute();

      ResultSet rs = pstmt.getResultSet();

      boolean tablehasrows = rs.next();

      if(tablehasrows)

      {

        //System.out.println("server: " + rs.getString(1));

        return rs.getString(1);

      }

      conn.close();

    }

    catch(Exception e)

    {

      System.out.println("Error in MetacatReplication.getServer: " + 

                          e.getMessage());

    }

    return null;

      //return null if the server does not exist

  }

  /**

   * Returns a server code given a server name

   * @param server the name of the server

   * @return integer > 0 representing the code of the server, 0 if the server

   *  does not exist.

   */

  public static int getServerCode(String server) throws Exception

  {

    try

    {

      Connection conn = util.openDBConnection();

      PreparedStatement pstmt = conn.prepareStatement("select serverid from " +

                                         "xml_replication where server " +

                                         "like '" + server + "'");

      pstmt.execute();

      ResultSet rs = pstmt.getResultSet();

      boolean tablehasrows = rs.next();

      int serverCode = 0;

      if(tablehasrows)

      {

         return rs.getInt(1);

      }

      else

      {

        return 0;

      }

    }

    catch(Exception e)

    {

      throw e;

    }

  }

  /**

   * This method returns the content of a url

   * @param u the url to return the content from

   * @return a string representing the content of the url

   * @throws java.io.IOException

   */

  public static String getURLContent(URL u) throws java.io.IOException

  {

    //System.out.println("url: " + u.toString());

    char istreamChar;

    int istreamInt;

    InputStreamReader istream = new InputStreamReader(u.openStream());

    StringBuffer serverResponse = new StringBuffer();

    while((istreamInt = istream.read()) != -1)

    {

      istreamChar = (char)istreamInt;

      serverResponse.append(istreamChar);

    }

    return serverResponse.toString();

  }

  /**

   * Method for writing replication messages to a log file specified in 

   * metacat.properties

   */

  public static void replLog(String message)

  {

    try

    {

      FileOutputStream fos = new FileOutputStream(

                                 util.getOption("replicationlog"), true);

      PrintWriter pw = new PrintWriter(fos);

      SimpleDateFormat formatter = new SimpleDateFormat ("yy-MM-dd HH:mm:ss");

      java.util.Date localtime = new java.util.Date();

      String dateString = formatter.format(localtime);

      dateString += " :: " + message;

      //time stamp each entry

      pw.println(dateString);

      pw.flush();

    }

    catch(Exception e)

    {

      System.out.println("error writing to replication log");

      //e.printStackTrace(System.out);

    }

  }

}