Index: src/edu/ucsb/nceas/metacat/MetaCatUtil.java
===================================================================
--- src/edu/ucsb/nceas/metacat/MetaCatUtil.java	(revision 730)
+++ src/edu/ucsb/nceas/metacat/MetaCatUtil.java	(revision 731)
@@ -258,45 +258,48 @@
     boolean poundflag = false;
     int arrcount = 0;
 
-    for (int i=0; i < query.length(); i++) { 
+    if ( query != null ) {
+      for (int i=0; i < query.length(); i++) { 
 
-      // go throught the remainder of the query one character at a time.
-      if (query.charAt(i) == '=') { 
-        // if the current char is a # then the preceding should be a name
-        if (!poundflag && ampflag) {
-          params[arrcount][0] = temp.trim();
-          temp = "";
-        } else { 
-          //if there are two #s or &s in a row throw an exception.
-          throw new MalformedURLException("metacatURL: Two parameter names " +                                            "not allowed in sequence");
-        }
-        poundflag = true;
-        ampflag = false;
-      } else if (query.charAt(i) == '&' || i == query.length()-1) { 
-        //the text preceding the & should be the param value.
-        if (i == query.length() - 1) { 
-          //if at the end of the string grab the last value and append it.
-          if (query.charAt(i) != '=') { 
+        // go throught the remainder of the query one character at a time.
+        if (query.charAt(i) == '=') { 
+          // if the current char is a # then the preceding should be a name
+          if (!poundflag && ampflag) {
+            params[arrcount][0] = temp.trim();
+            temp = "";
+          } else { 
+            //if there are two #s or &s in a row throw an exception.
+            throw new MalformedURLException("metacatURL: Two parameter names "+
+                                            "not allowed in sequence");
+          }
+          poundflag = true;
+          ampflag = false;
+        } else if (query.charAt(i) == '&' || i == query.length()-1) { 
+          //the text preceding the & should be the param value.
+          if (i == query.length() - 1) { 
+            //if at the end of the string grab the last value and append it.
+            if (query.charAt(i) != '=') { 
             //ignore an extra & on the end of the string
-            temp += query.charAt(i);
+              temp += query.charAt(i);
+            }
           }
-        }
-
-        if (!ampflag && poundflag) {
-          params[arrcount][1] = temp.trim();
-          parameters.put(params[arrcount][0], params[arrcount][1]);
-          temp = "";
-          arrcount++; //increment the array to the next row.
+  
+          if (!ampflag && poundflag) {
+            params[arrcount][1] = temp.trim();
+            parameters.put(params[arrcount][0], params[arrcount][1]);
+            temp = "";
+            arrcount++; //increment the array to the next row.
+          } else { 
+            //if there are two =s or &s in a row through an exception
+            throw new MalformedURLException("metacatURL: Two parameter values "+
+                                            "not allowed in sequence");
+          }
+          poundflag = false;
+          ampflag = true;
         } else { 
-          //if there are two =s or &s in a row through an exception
-          throw new MalformedURLException("metacatURL: Two parameter values " +
-                                          "not allowed in sequence");
+          //get the next character in the string
+          temp += query.charAt(i);
         }
-        poundflag = false;
-        ampflag = true;
-      } else { 
-        //get the next character in the string
-        temp += query.charAt(i);
       }
     }
     return parameters;
Index: src/edu/ucsb/nceas/metacat/MetaCatServlet.java
===================================================================
--- src/edu/ucsb/nceas/metacat/MetaCatServlet.java	(revision 730)
+++ src/edu/ucsb/nceas/metacat/MetaCatServlet.java	(revision 731)
@@ -29,14 +29,9 @@
 
 import java.io.PrintWriter;
 import java.io.IOException;
-import java.io.Reader;
 import java.io.StringReader;
-import java.io.BufferedReader;
-import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStreamReader;
-import java.io.DataInputStream;
+import java.io.BufferedInputStream;
 import java.util.Enumeration;
 import java.util.Hashtable;
 import java.util.ResourceBundle; 
@@ -77,14 +72,15 @@
  * action=query -- query the values of all elements and attributes
  *                     and return a result set of nodes<br>
  * action=squery -- structured query (see pathquery.dtd)<br>
+ * action=read -- read any metadata/data file from Metacat and from Internet<br>
  * action=insert -- insert an XML document into the database store<br>
  * action=update -- update an XML document that is in the database store<br>
  * action=delete --  delete an XML document from the database store<br>
  * action=validate -- vallidate the xml contained in valtext<br>
- * action=read -- display an XML document in XML or HTML<br>
  * doctype -- document type list returned by the query (publicID)<br>
  * qformat=xml -- display resultset from query in XML<br>
  * qformat=html -- display resultset from query in HTML<br>
+ * qformat=zip -- zip resultset from query<br>
  * docid=34 -- display the document with the document ID number 34<br>
  * doctext -- XML text of the document to load into the database<br>
  * acltext -- XML access text for a document to load into the database<br>
@@ -91,7 +87,7 @@
  * dtdtext -- XML DTD text for a new DTD to load into Metacat XML Catalog<br>
  * query -- actual query text (to go with 'action=query' or 'action=squery')<br>
  * valtext -- XML text to be validated<br>
- * action=getdatadoc -- retrieve a stored datadocument<br>
+ * abstractpath -- XPath in metadata document to read from<br>
  * action=getaccesscontrol -- retrieve acl info for Metacat document<br>
  * action=getdoctypes -- retrieve all doctypes (publicID)<br>
  * action=getdtdschema -- retrieve a DTD or Schema file<br>
@@ -144,6 +140,7 @@
       servletpath = util.getOption("servletpath");
       htmlpath = util.getOption("htmlpath");
 
+// MOVED IT TO doGet() & doPost()
 //      try {
 //        // Open a pool of db connections
 //        connectionPool = util.getConnectionPool();
@@ -219,7 +216,6 @@
       // Decode the docid and mouse click information
       if (name.endsWith(".y")) {
         docid[0] = name.substring(0,name.length()-2);
-        //out.println("docid => " + docid[0]);
         params.put("docid", docid);
         name = "ypos";
       }
@@ -227,7 +223,6 @@
         name = "xpos";
       } 
 
-      //pwout.println(name + " => " + value[0]);
       params.put(name,value); 
     }  
     
@@ -236,13 +231,6 @@
     String action = ((String[])params.get("action"))[0];  
     util.debugMessage("Line 213: Action is: " + action);
 
-    //MBJELIMINATE String action = decodeMouseAction(params);
-    //if(action.equals("error"))
-    //{
-      //util.debugMessage("Line 218: Action is: " + action);
-      //action = ((String[])params.get("action"))[0];  
-    //}
-    
     // This block handles session management for the servlet
     // by looking up the current session information for all actions
     // other than "login" and "logout"
@@ -273,12 +261,9 @@
         username = (String)sess.getAttribute("username");
         password = (String)sess.getAttribute("password");
         groupname = (String)sess.getAttribute("groupname");
-        try
-        {
+        try {
           sess_id = (String)sess.getId();
-        }
-        catch(IllegalStateException ise)
-        {
+        } catch(IllegalStateException ise) {
           System.out.println("error in handleGetOrPost: this shouldn't " +
                              "happen: the session should be valid: " + 
                              ise.getMessage());
@@ -288,48 +273,19 @@
 
     // Now that we know the session is valid, we can delegate the request
     // to a particular action handler
-    if(action.equals("query"))
-    {
+    if(action.equals("query")) {
       handleQuery(response.getWriter(), params, response, username, groupname); 
-    } 
-    else if(action.equals("squery"))
-    {
-      if(params.containsKey("query"))
-      {
+    } else if(action.equals("squery")) {
+      if(params.containsKey("query")) {
         handleSQuery(response.getWriter(), params, response, username, groupname); 
-      }
-      else
-      {
+      } else {
         PrintWriter out = response.getWriter();
         out.println("Illegal action squery without \"query\" parameter");
       }
-    }
-    else if (action.equals("read")) {
-      //PrintWriter out = response.getWriter();
-      try {
-        handleReadAction(/*out,*/ params, response, username);
-      } catch (ClassNotFoundException e) {
-        System.out.println("Error in MetacatServlet.handlGetOrPost: " + 
-                            e.getMessage());
-      } catch (SQLException se) {
-        System.out.println("Error in MetaCatServlet.handleGetOrPost: " + 
-                            se.getMessage());
-      }
-    } 
-/*
-    else if (action.equals("getrelateddocument")) {
+    } else if (action.equals("read")) {
+      handleReadAction(params, response, username, groupname);
+    } else if (action.equals("insert") || action.equals("update")) {
       PrintWriter out = response.getWriter();
-      try {
-        handleGetRelatedDocumentAction(out, params, response);
-      } catch (ClassNotFoundException e) {
-        out.println(e.getMessage());
-      } catch (SQLException se) {
-        out.println(se.getMessage());
-      }
-    }
-*/
-    else if (action.equals("insert") || action.equals("update")) {
-      PrintWriter out = response.getWriter();
       if ( (username != null) &&  !username.equals("public") ) {
         handleInsertOrUpdateAction(out, params, response, username, groupname);
       } else {  
@@ -345,26 +301,6 @@
     } else if (action.equals("validate")) {
       PrintWriter out = response.getWriter();
       handleValidateAction(out, params, response); 
-    } else if (action.equals("getabstract")) {
-      PrintWriter out = response.getWriter();
-      try{
-        handleViewAbstractAction(out, params, response);
-      }
-      catch(Exception e)
-      {
-        out.println("error viewing abstract from " + 
-                    "MetacatServlet.handleGetorPost: " + e.getMessage());
-      }
-    } else if (action.equals("getdatadoc")) {
-      response.setContentType("application/zip");
-      ServletOutputStream out = response.getOutputStream();
-      handleGetDataDocumentAction(out, params, response);  
-    } else if (action.equals("getaccesscontrol")) {
-      PrintWriter out = response.getWriter();
-      handleGetAccessControlAction(out, params, response, username, groupname);
-    } else if (action.equals("getdoctypes")) {
-      PrintWriter out = response.getWriter();
-      handleGetDoctypesAction(out, params, response);  
     } else if (action.equals("getdataport")) {
       PrintWriter out = response.getWriter();
       if ( (username != null) &&  !username.equals("public") ) {
@@ -374,6 +310,15 @@
         out.println("You must be authenticated to perform the getdataport " +
                     "action!");
       }
+    } else if (action.equals("getaccesscontrol")) {
+      PrintWriter out = response.getWriter();
+      handleGetAccessControlAction(out, params, response, username, groupname);
+    } else if (action.equals("getprincipals")) {
+      PrintWriter out = response.getWriter();
+      handleGetPrincipalsAction(out, username, password);  
+    } else if (action.equals("getdoctypes")) {
+      PrintWriter out = response.getWriter();
+      handleGetDoctypesAction(out, params, response);  
     } else if (action.equals("getdtdschema")) {
       PrintWriter out = response.getWriter();
       handleGetDTDSchemaAction(out, params, response);  
@@ -380,9 +325,6 @@
     } else if (action.equals("getdataguide")) {
       PrintWriter out = response.getWriter();
       handleGetDataGuideAction(out, params, response);  
-    } else if (action.equals("getprincipals")) {
-      PrintWriter out = response.getWriter();
-      handleGetPrincipalsAction(out, username, password);  
     } else if (action.equals("login") || action.equals("logout")) {
     } else if (action.equals("protocoltest")) {
       String testURL = "metacat://dev.nceas.ucsb.edu/NCEAS.897766.9";
@@ -429,89 +371,13 @@
       PrintWriter out = response.getWriter();
       out.println("Error: action not registered.  Please report this error.");
     }
+    
     util.closeConnections();
     // Close the stream to the client
-    //out.close();
+    // out.close();
   }
   
-  /**
-   * decodes the mouse click information coming from the client.
-   * This function may be overwritten to provide specific functionality
-   * for different applications.
-   * @param params the parameters from the CGI
-   * @return action the action to be performed or "error" if an error was
-   * generated
-   */
-  protected String decodeMouseAction(Hashtable params)
-  {
-    // Determine what type of request the user made
-    // if the action parameter is set, use it as a default
-    // but if the ypos param is set, calculate the action needed
-    String action=null;
-    long ypos = 0;
-    try {
-      ypos = (new Long(((String[])params.get("ypos"))[0]).longValue());
-      //out.println("<P>YPOS IS " + ypos);
-      if (ypos <= 13) {
-        action = "read";
-      } else if (ypos > 13 && ypos <= 27) {
-        action = "validate";
-      } else if (ypos > 27) {
-        action = "transform";
-      }
-      return action;
-    } catch (Exception npe) {
-      //
-      // MBJ -- NOTE that this should be handled more gracefully with
-      //        the new exception infrastructure -- this "error" return
-      //        value is inappropriate
-      //out.println("<P>Caught exception looking for Y value.");
-      return "error";
-    }  
-  }
-  
-  /**
-   * sends the port number that the data socket is running on.  This is a 
-   * parameter set in the metacat.properties file.
-   */
-  private void handleGetDataPortAction(PrintWriter out, Hashtable params, 
-                                       HttpServletResponse response, 
-                                       String username, String groupname,
-                                       String sess_id)
-  {
-    int port;
-    String filedir = null;
-    try
-    {
-      filedir = util.getOption("datafilepath");
-      
-      Random r = new Random();
-      port = r.nextInt(65000);  //pick a random port between 0-65000
-      //System.out.println("random port is: " + port);
-      while(!DataFileServer.portIsAvailable(port))
-      {
-        port = r.nextInt(65000);
-        //System.out.println("next port used: " + port);
-      }
-      DataFileServer dfs = new DataFileServer(port, username, sess_id);
-      dfs.start();
-      
-      //System.out.println("filedir: " + filedir);
-      //System.out.println("port: " + port);
-      response.setContentType("text/xml");
-      out.println("<?xml version=\"1.0\"?>");
-      out.println("<port>");
-      out.print(port);
-      out.print("</port>");
-      
-    }
-	  catch (Exception e) 
-    {
-      System.out.println("error in MetacatServlet.handleGetDataPortAction: " + 
-                          e.getMessage());
-    }
-  }
-
+  // LOGIN & LOGOUT SECTION
   /** 
    * Handle the login request. Create a new session object.
    * Do user authentication through the session.
@@ -555,25 +421,6 @@
       response.setContentType("text/xml");
       out.println(sess.getMessage()); 
     }
-        
-/* WITHOUT XSLT transformation
-    // redirects response to html page
-    if (qformat.equals("html")) {
-      // user authentication successful
-      if (isValid) {
-        response.sendRedirect(
-                 response.encodeRedirectUrl(htmlpath + "/metacat.html"));
-      // unsuccessful user authentication 
-      } else {
-        response.sendRedirect(htmlpath + "/login.html");
-      }
-    // any output is returned  
-    } else {
-      response.setContentType("text/xml");
-      out.println(sess.getMessage()); 
-    }
-*/        
-
   }    
 
   /** 
@@ -613,23 +460,10 @@
       response.setContentType("text/xml");
       out.println(output.toString()); 
     }
-
-/* WITHOUT XSLT transformation
-    // redirects response to html page
-    if (qformat.equals("html")) {
-        response.sendRedirect(htmlpath + "/index.html"); 
-      } catch(Exception e) {
-        util.returnConnection(conn); 
-      } 
-    // any output is returned  
-    } else {
-      response.setContentType("text/xml");
-      out.println(output.toString()); 
-    }
-*/
   }
-
+  // END OF LOGIN & LOGOUT SECTION
   
+  // SQUERY & QUERY SECTION
   /**      
    * Retreive the squery xml, execute it and display it
    *
@@ -829,328 +663,339 @@
     //System.out.println(resultset.toString());
     return resultset.toString();
   }
+  // END OF SQUERY & QUERY SECTION
   
-  /**
-   * Handle the request to view the abstract of a document.
-   * The abstractpath CGI parameter gives the xml path to the abstract
-   * node.  
+  // READ SECTION
+  /** 
+   * Handle the "read" request of metadata/data files from Metacat
+   * or any files from Internet;
+   * transformed metadata XML document into HTML presentation if requested;
+   * zip files when more than one were requested.
+   *
+   * @param params the Hashtable of HTTP request parameters
+   * @param response the HTTP response object linked to the client
+   * @param user the username sent the request
+   * @param group the user's groupname
    */
-  private void handleViewAbstractAction(PrintWriter out, Hashtable params,
-               HttpServletResponse response) throws IOException, SQLException
+  private void handleReadAction(Hashtable params, HttpServletResponse response,
+                                String user, String group) 
   {
-    String abstractpath = null;
-    String docid = null;
-    Connection conn = null;
-    response.setContentType("text/html");
-    try
-    {
-      docid = ((String[])params.get("docid"))[0];
-      if(params.containsKey("abstractpath"))
-      {
-        //the CGI parameter abstractpath holds the path to the abstract
-        //that should be displayed.
-        abstractpath = ((String[])params.get("abstractpath"))[0];
+    ServletOutputStream out = null;
+    ZipOutputStream zout = null;
+    
+    try {
+      String[] docs = new String[0];
+      String docid = "";
+      String qformat = "";
+      String abstrpath = null;
+      boolean zip = false;
+      // read the params
+      if (params.containsKey("docid")) {
+        docs = (String[])params.get("docid");
       }
-      else
-      {
-        out.println("error: no abstractpath parameter"); 
+      if (params.containsKey("qformat")) {
+        qformat = ((String[])params.get("qformat"))[0];
       }
-      conn = util.getConnection();
-    
-      Object[] abstracts = DBQuery.getNodeContent(abstractpath, docid, conn);
-    
-      out.println("<html><head><title>Abstract</title></head>");
-      out.println("<body bgcolor=\"white\"><h1>Abstract</h1>");
-      for(int i=0; i<abstracts.length; i++)
-      {
-        out.println("<p>" + (String)abstracts[i] + "</p>");
+      if (params.containsKey("abstractpath")) {
+        abstrpath = ((String[])params.get("abstractpath"))[0];
+        viewAbstract(response, abstrpath, docs[0]);
       }
-      out.println("</body></html>");
-    }
-    catch (IOException ioe)
-    {
-       util.debugMessage("error in MetacatServlet.handlegetabstract: " + 
-                          ioe.getMessage());
-       System.out.println("IO error in MetacatServlet.handlegetabstract: " + 
-                          ioe.getMessage());
-    }
-    catch(SQLException sqle)
-    {
-      util.debugMessage("sql error in MetacatServLet.handlegetabstract: " + 
-                         sqle.getMessage());
-      System.out.println("sql error in MetacatServLet.handlegetabstract: " + 
-                         sqle.getMessage());
-    }
-    catch(Exception e)
-    {
-      util.debugMessage("error in handlegetabstract: " + e.getMessage());
-      System.out.println("error in MetacatServlEt.handlegetabstract: " + 
+      if ( (docs.length > 1) || qformat.equals("zip") ) {
+        zip = true;
+        out = response.getOutputStream();
+        response.setContentType("application/zip"); //MIME type
+        zout = new ZipOutputStream(out);
+      }
+      // go through the list of docs to read
+      for (int i=0; i < docs.length; i++ ) {
+        try {
+
+          URL murl = new URL(docs[i]);
+          Hashtable murlQueryStr = util.parseQuery(murl.getQuery());
+          // case docid="http://.../?docid=aaa" 
+          // or docid="metacat://.../?docid=bbb"
+          if (murlQueryStr.containsKey("docid")) {
+            // get only docid, eliminate the rest
+            docid = (String)murlQueryStr.get("docid");
+            if ( zip ) {
+              addDocToZip(docid, zout);
+            } else {
+              readFromMetacat(response, docid, qformat, abstrpath,
+                              user, group, zip, zout);
+            }
+
+          // case docid="http://.../filename"
+          } else {
+            docid = docs[i];
+            if ( zip ) {
+              addDocToZip(docid, zout);
+            } else {
+              readFromURLConnection(response, docid);
+            }
+          }
+
+        // case docid="ccc"
+        } catch (MalformedURLException mue) {
+          docid = docs[i];
+          if ( zip ) {
+            addDocToZip(docid, zout);
+          } else {
+            readFromMetacat(response, docid, qformat, abstrpath,
+                            user, group, zip, zout);
+          }
+        }
+        
+      } /* end for */
+      
+      if ( zip ) {
+        zout.finish(); //terminate the zip file
+        zout.close();  //close the zip stream
+      }
+      
+    } catch (Exception e) {
+      try {
+        if ( out != null ) { out.close(); }
+        if ( zout != null ) { zout.close(); }
+        response.setContentType("text/xml"); //MIME type
+        PrintWriter pw = response.getWriter();
+        pw.println(e.getMessage());
+      } catch (IOException ioe) {
+        System.out.println("Problem with the servlet output " +
+                           "in MetacatServlet.handleReadAction: " +
+                           ioe.getMessage());
+        ioe.printStackTrace(System.out);
+        
+      }
+
+      System.out.println("Error in MetacatServlet.handleReadAction: " +
                          e.getMessage());
+      e.printStackTrace(System.out);
     }
     
-    util.returnConnection(conn);
   }
-
-  /** 
-   * Handle the database getrelateddocument request and return a XML document, 
-   * possibly transformed from XML into HTML
-   */
-  private void handleGetRelatedDocumentAction(PrintWriter out, Hashtable params,
-               HttpServletResponse response, URL murl) 
-               throws ClassNotFoundException, IOException, SQLException 
+  
+  // read metadata or data from Metacat
+  private void readFromMetacat(HttpServletResponse response, String docid,
+                               String qformat, String abstrpath, String user,
+                               String group, boolean zip, ZipOutputStream zout)
+               throws ClassNotFoundException, IOException, SQLException, 
+                      McdbException, Exception
   {
-    String docid = null;
     Connection conn = null;
+    try {
+      conn = util.getConnection();
+      DocumentImpl doc = new DocumentImpl(conn, docid);
       
-    //if(params.containsKey("url"))
-    //{//the identifier for the related document is contained in the URL param
-      try
-      {
-        DocumentImpl xmldoc=null;
-        //MetacatURL murl = new MetacatURL(((String[])params.get("url"))[0]);
-        if(murl.getProtocol().equals("metacat"))
-        {//get the document from the database if it is the right type of url
-          //Hashtable murlParams = murl.getHashParams();
-          Hashtable murlParams = util.parseQuery(murl.getQuery());
-          if(murlParams.containsKey("docid"))
-          {//the docid should be first
-            docid = (String)murlParams.get("docid"); //get the docid value
-            conn = util.getConnection();
-            xmldoc = new DocumentImpl(conn, docid);
-            String qformat = ((String[])params.get("qformat"))[0];
-            if (qformat.equals("xml")) 
-            { 
-              // set content type and other response header fields first
-              response.setContentType("text/xml");
-              xmldoc.toXml(out);
-              //out.println(xmldoc);
-            } 
-            else if (qformat.equals("html")) 
-            {
-              response.setContentType("text/html");
-              // Look up the document type
-              String sourcetype = xmldoc.getDoctype();
-              // Transform the document to the new doctype
-              DBTransform dbt = new DBTransform(conn);
-              dbt.transformXMLDocument(xmldoc.toString(), sourcetype, 
-                                 "-//W3C//HTML//EN", out);
-            }
-
-            util.returnConnection(conn);
+      if ( doc.getRootNodeID() == 0 ) {
+        // this is data file
+        // ??? look at handleGetData
+        ServletOutputStream out = response.getOutputStream(); 
+        String filepath = util.getOption("datafilepath");
+        if(!filepath.endsWith("/")) {
+          filepath += "/";
+        }
+        String filename = filepath + doc.getDocname();      //MIME type
+        String contentType = getServletContext().getMimeType(filename);
+        if (contentType == null) {
+          if (filename.endsWith(".xml")) {
+            contentType="text/xml";
+          } else if (filename.endsWith(".css")) {
+            contentType="text/css";
+          } else if (filename.endsWith(".dtd")) {
+            contentType="text/plain";
+          } else if (filename.endsWith(".xsd")) {
+            contentType="text/plain";
+          } else if (filename.endsWith("/")) {
+            contentType="text/directory";
+          } else {
+            contentType="text/plain";
           }
-          else
-          {
-            //throw new Exception("handleGetDocument: bad URL");
-            System.err.println("handleGetDocument: bad URL");
-          }
         }
-        else if(murl.getProtocol().equals("http"))
-        {//get the document from the internet
-          //Hashtable murlParams = murl.getHashParams();
-          Hashtable murlParams = util.parseQuery(murl.getQuery());
-          if(murlParams.containsKey("httpurl"))
-          {//httpurl is the param name for an http url.
-            URL urlconn = new URL((String)murlParams.get("httpurl"));  
-            //create a new url obj.
-            //DataInputStream htmldoc = new DataInputStream(urlconn.openStream());
-            BufferedReader htmldoc = new BufferedReader(
-                                   new InputStreamReader(urlconn.openStream()));
-            //bind a data stream.
-            try
-            { //display the document
-              String line=null;
-              while((line = htmldoc.readLine()) != null)
-              {
-                out.println(line); 
-              }
-            }
-            catch(Exception e)
-            {
-              System.out.println("error viewing html document from " +
-                            "MetacatServlet.handleGetRelatedDocumentAction: " +
-                            e.getMessage()); 
-            }
+        response.setContentType(contentType);
+        FileInputStream fin = null;
+        try {
+          fin = new FileInputStream(filename);
+          byte[] buf = new byte[4 * 1024]; // 4K buffer
+          int b = fin.read(buf);
+          while (b != -1) {
+            out.write(buf, 0, b);
+            b = fin.read(buf);
           }
+        } finally {
+          if (fin != null) fin.close();
         }
+
+      } else {
+        // this is metadata doc
+        if ( qformat.equals("html") ) { 
+          response.setContentType("text/html");  //MIME type
+          PrintWriter out = response.getWriter();
+    
+          // Look up the document type
+          String doctype = doc.getDoctype();
+          // Transform the document to the new doctype
+          DBTransform dbt = new DBTransform(conn);
+          dbt.transformXMLDocument(doc.toString(),
+                                   doctype,"-//W3C//HTML//EN",out);
+        } else {
+          // set content type first
+          response.setContentType("text/xml");   //MIME type
+          PrintWriter out = response.getWriter();
+          doc.toXml(out);
+        }
+      
       }
-      catch (McdbException e) {
-        response.setContentType("text/xml");
-        e.toXml(out);
-      } catch   (Throwable t) {
-        response.setContentType("text/html");
-        out.println(t.getMessage());
-      } finally {
-        util.returnConnection(conn);
-      }
-    //} // end if
-  }   
+    } finally {
+      util.returnConnection(conn);
+    }
+    
+  }
   
-  /** 
-   * Handle the database read request and return an XML document, 
-   * possibly transformed from XML into HTML
-   */
-  private void handleReadAction(/*PrintWriter out,*/ Hashtable params, 
-               HttpServletResponse response, String username) 
-               throws ClassNotFoundException, IOException, SQLException 
+  // read data from URLConnection
+  private void readFromURLConnection(HttpServletResponse response, String docid)
+               throws IOException, MalformedURLException
   {
-     PrintWriter out;
+    ServletOutputStream out = response.getOutputStream(); 
+    String contentType = getServletContext().getMimeType(docid); //MIME type
+    if (contentType == null) {
+      if (docid.endsWith(".xml")) {
+        contentType="text/xml";
+      } else if (docid.endsWith(".css")) {
+        contentType="text/css";
+      } else if (docid.endsWith(".dtd")) {
+        contentType="text/plain";
+      } else if (docid.endsWith(".xsd")) {
+        contentType="text/plain";
+      } else if (docid.endsWith("/")) {
+        contentType="text/directory";
+      } else {
+        contentType="text/plain";
+      }
+    }
+    response.setContentType(contentType);
+
+    // this is http url
+    URL url = new URL(docid);
+    BufferedInputStream bis = null;
     try {
-      //MetacatURL murl = new MetacatURL(((String[])params.get("docid"))[0]);
-      if(params.containsKey(new String("qformat")) && 
-         ((String[])params.get("qformat"))[0].equals("bin"))
-      {
-        handleGetData(params, response, username);
+      bis = new BufferedInputStream(url.openStream());
+      byte[] buf = new byte[4 * 1024]; // 4K buffer
+      int b = bis.read(buf);
+      while (b != -1) {
+        out.write(buf, 0, b);
+        b = bis.read(buf);
       }
-      else
-      {
-        out = response.getWriter();
-        URL murl = new URL(((String[])params.get("docid"))[0]);
-        handleGetRelatedDocumentAction(out, params, response, murl);
-      }
-    } catch (MalformedURLException mue) {
-      out = response.getWriter();
-      handleGetDocumentAction(out, params, response);
+    } finally {
+      if (bis != null) bis.close();
     }
+    
   }
   
-  /**
-   * Handle the read of a data file.
-   */
-  private void handleGetData(Hashtable params, 
-                             HttpServletResponse response, String username)
+  // read file/doc and write to ZipOutputStream
+  private void addDocToZip(String docid, ZipOutputStream zout)
+               throws ClassNotFoundException, IOException, SQLException, 
+                      McdbException, Exception
   {
-    String docid = null;
-    try
-    {
-      URL murl = new URL(((String[])params.get("docid"))[0]);
-      Hashtable murlParams = util.parseQuery(murl.getQuery());
-      if(murlParams.containsKey("docid"))
-      {
-        docid = (String)murlParams.get("docid");
+    byte[] bytestring = null;
+    ZipEntry zentry = null;
+
+    try {
+      URL url = new URL(docid);
+
+      // this http url; read from URLConnection; add to zip
+      zentry = new ZipEntry(docid);
+      zout.putNextEntry(zentry);
+      BufferedInputStream bis = null;
+      try {
+        bis = new BufferedInputStream(url.openStream());
+        byte[] buf = new byte[4 * 1024]; // 4K buffer
+        int b = bis.read(buf);
+        while(b != -1) {
+          zout.write(buf, 0, b);
+          b = bis.read(buf);
+        }
+      } finally {
+        if (bis != null) bis.close();
       }
-    }
-    catch(MalformedURLException mue)
-    {
-      docid = ((String[])params.get("docid"))[0];
-    }
-    
-    File f = null;
-    FileInputStream fin = null;
-    Connection conn;
-    
-    try
-    {
-      StringBuffer sql = new StringBuffer();
-      sql.append("select docname from xml_documents where docid like '");
-      sql.append(docid).append("'");
-      conn = util.openDBConnection();
+      zout.closeEntry();
+
+    } catch (MalformedURLException mue) {
       
-      AccessControlList aclobj = new AccessControlList(conn);
-      boolean hasPermission = aclobj.hasPermission("READ",username,docid);
+      // this is metacat doc (data file or metadata doc)
+      Connection conn = null;
+      try {
+        conn = util.getConnection();
+        DocumentImpl doc = new DocumentImpl(conn, docid);
       
-      if(!hasPermission)
-      {
-        response.setContentType("text/html");
-        PrintWriter out = response.getWriter();
-        out.println("Error: you do not have permission to view this document");
-        return;
-      }
-      response.setContentType("application/octet-stream");
-      ServletOutputStream sosout = response.getOutputStream(); 
-      PreparedStatement pstmt = conn.prepareStatement(sql.toString());
-      pstmt.execute();
-      ResultSet rs = pstmt.getResultSet();
-      boolean tablehasrows = rs.next();
-      while(tablehasrows)
-      {
-        //get the file stream from the file then send it to the output stream
-        String filepath = util.getOption("datafilepath");
-        if(!filepath.endsWith("/"))
-        {
-          filepath += "/";
+        if ( doc.getRootNodeID() == 0 ) {
+          // this is data file; add file to zip
+          String filepath = util.getOption("datafilepath");
+          if(!filepath.endsWith("/")) {
+            filepath += "/";
+          }
+          String filename = filepath + doc.getDocname();
+          zentry = new ZipEntry(filename);
+          zout.putNextEntry(zentry);
+          FileInputStream fin = null;
+          try {
+            fin = new FileInputStream(filename);
+            byte[] buf = new byte[4 * 1024]; // 4K buffer
+            int b = fin.read(buf);
+            while (b != -1) {
+              zout.write(buf, 0, b);
+              b = fin.read(buf);
+            }
+          } finally {
+            if (fin != null) fin.close();
+          }
+          zout.closeEntry();
+
+        } else {
+          // this is metadata doc; add doc to zip
+          bytestring = doc.toString().getBytes();
+          zentry = new ZipEntry(docid + ".xml");
+          zentry.setSize(bytestring.length);
+          zout.putNextEntry(zentry);
+          zout.write(bytestring, 0, bytestring.length);
+          zout.closeEntry();
         }
-        f = new File(filepath + rs.getString(1)); 
-        fin = new FileInputStream(f);
-        
-        int b = fin.read();
-        while(b != -1)
-        {
-          sosout.write(b);
-          b = fin.read();
-        }
-        tablehasrows = rs.next();
+      } finally {
+        util.returnConnection(conn);
       }
       
-      fin.close();
-      pstmt.close();
-      conn.close();
     }
-    catch(Exception e)
-    {
-      System.out.println("error in MetacatServlet.handleGetData: " + 
-                          e.getMessage());
-      e.printStackTrace(System.out);
-    }
+      
   }
-
-  /** 
-   * Handle the database read request and return an XML document, 
-   * possibly transformed from XML into HTML
-   */
-  private void handleGetDocumentAction(PrintWriter out, Hashtable params, 
-               HttpServletResponse response) 
-               throws ClassNotFoundException, IOException, SQLException {
-    String docidstr = null;
-    String docid = null;
-    String doc = null;
+  
+  // view abstract within document
+  private void viewAbstract(HttpServletResponse response,
+                            String abstractpath, String docid)
+               throws ClassNotFoundException, IOException, SQLException,
+                      McdbException, Exception
+  {
     Connection conn = null;
-    
     try {
-      // Find the document id number
-      docidstr = ((String[])params.get("docid"))[0]; 
-      docid = docidstr;
-      
       conn = util.getConnection();
-      DocumentImpl xmldoc = new DocumentImpl(conn, docid);
-      // Get the document indicated from the db
-      //doc = docreader.readXMLDocument(docid);
+      DocumentImpl doc = new DocumentImpl(conn, docid);
+    
+      Object[] abstracts = DBQuery.getNodeContent(abstractpath, docid, conn);
+    
+      response.setContentType("text/html");  //MIME type
+      PrintWriter out = response.getWriter();
+      out.println("<html><head><title>Abstract</title></head>");
+      out.println("<body bgcolor=\"white\"><h1>Abstract</h1>");
+      for (int i=0; i<abstracts.length; i++) {
+        out.println("<p>" + (String)abstracts[i] + "</p>");
+      }
+      out.println("</body></html>");
 
-      // Return the document in XML or HTML format
-      String qformat=null;
-      if(params.containsKey("qformat"))
-      {
-        qformat = ((String[])params.get("qformat"))[0];
-      }
-      else
-      {
-        qformat = "html";        
-      }
-      if (qformat.equals("xml")) { 
-        // set content type and other response header fields first
-        response.setContentType("text/xml");
-        xmldoc.toXml(out);
-        //out.println(xmldoc);
-      } else if (qformat.equals("html")) {
-        response.setContentType("text/html");
-        // Look up the document type
-        String sourcetype = xmldoc.getDoctype();
-        // Transform the document to the new doctype
-        DBTransform dbt = new DBTransform(conn);
-        dbt.transformXMLDocument(xmldoc.toString(), sourcetype, 
-                                 "-//W3C//HTML//EN", out);
-      }
-    } catch (McdbException e) {
-      response.setContentType("text/xml");
-      e.toXml(out);
-    } catch (Throwable t) {
-      response.setContentType("text/html");
-      out.println(t.getMessage());
     } finally {
       util.returnConnection(conn);
-    }    
-
+    }
   }
-
+  // END OF READ SECTION
+    
+  // INSERT/UPDATE SECTION
   /** 
    * Handle the database putdocument request and write an XML document 
    * to the database connection
@@ -1306,7 +1151,9 @@
 System.out.println("Validation is " + validate);
     return validate;
   }
+  // END OF INSERT/UPDATE SECTION
 
+  // DELETE SECTION
   /** 
    * Handle the database delete request and delete an XML document 
    * from the database connection
@@ -1349,7 +1196,9 @@
       util.returnConnection(conn);
     }  
   }
+  // END OF DELETE SECTION
   
+  // VALIDATE SECTION
   /** 
    * Handle the validation request and return the results to the requestor
    */
@@ -1411,166 +1260,50 @@
       util.returnConnection(conn);
     }  
   }
-
-  /** 
-   * Handle the document request and return the results to the requestor
-   * If a docid is passed in through the params then that document
-   * will be retrieved form the DB and put in the zip file.
-   * In addition if 1 or more relations parameters are passed, those file
-   * will be zipped as well.  Currently this is only implemented for 
-   * metacat:// and http:// files.  Support should be added for srb:// files
-   * as well.
+  // END OF VALIDATE SECTION
+ 
+  // OTHER ACTION HANDLERS
+  /**
+   * sends the port number that the data socket is running on.
+   * This is a parameter set in the metacat.properties file.
    */
-  private void handleGetDataDocumentAction(ServletOutputStream out, 
-               Hashtable params, 
-               HttpServletResponse response) {
-  //find the related files, get them from their source and zip them into 
-  //a zip file.
-  try
+  private void handleGetDataPortAction(PrintWriter out, Hashtable params, 
+                                       HttpServletResponse response, 
+                                       String username, String groupname,
+                                       String sess_id)
   {
-    Connection conn = util.getConnection();
-    String currentDocid = ((String[])params.get("docid"))[0];
-    ZipOutputStream zout = new ZipOutputStream(out);
-    byte[] bytestring = null;
-    ZipEntry zentry = null;
-    DocumentImpl xmldoc = null;
-    String[] reldocs = null;
-    
-    if(params.containsKey("relation"))
-    { //get the relations from the parameters.
-      reldocs = ((String[])params.get("relation"));
-    }
-    else
-    { //let the for loop know that there are no relations to zip
-      reldocs = new String[0];
-    }
-
-    //write the base file to the zip file.
-    xmldoc = new DocumentImpl(conn, currentDocid);
-    bytestring = (xmldoc.toString()).getBytes();
-    zentry = new ZipEntry(currentDocid + ".xml");
-    //create a new zip entry and write the file to the stream
-    zentry.setSize(bytestring.length);
-    zout.putNextEntry(zentry);
-    zout.write(bytestring, 0, bytestring.length);
-    zout.closeEntry(); //get ready for the next entry. 
-
-    //zip up the related documents
-    for(int i=0; i<reldocs.length; i++)
+    int port;
+    String filedir = null;
+    try
     {
-      //MetacatURL murl = new MetacatURL(((String)reldocs[i]));
-      URL murl = new URL(((String)reldocs[i]));
-      Hashtable qparams = util.parseQuery(murl.getQuery());
-      if(murl.getProtocol().equals("metacat"))
+      filedir = util.getOption("datafilepath");
+      
+      Random r = new Random();
+      port = r.nextInt(65000);  //pick a random port between 0-65000
+      //System.out.println("random port is: " + port);
+      while(!DataFileServer.portIsAvailable(port))
       {
-        //get the document from the database
-        //xmldoc = new DocumentImpl(conn, (String)murl.getHashParam("docid"));
-        xmldoc = new DocumentImpl(conn, (String)qparams.get("docid"));
-        bytestring = (xmldoc.toString()).getBytes();
-        zentry = new ZipEntry(qparams.get("docid") + ".xml");
-        //create a new zip entry and write the file to the stream
-        zentry.setSize(bytestring.length);
-        zout.putNextEntry(zentry);
-        zout.write(bytestring, 0, bytestring.length);
-        zout.closeEntry(); //get ready for the next entry.
+        port = r.nextInt(65000);
+        //System.out.println("next port used: " + port);
       }
-      else if(murl.getProtocol().equals("http"))
-      {
-        //Hashtable murlParams = murl.getHashParams();
-        if(qparams.containsKey("httpurl"))
-        {//httpurl is the param name for an http url.
-          URL urlconn = new URL((String)qparams.get("httpurl"));  
-          //create a new url obj.
-          BufferedReader htmldoc = new BufferedReader(
-                                   new InputStreamReader(urlconn.openStream()));
-          //get the data from the web server
-          try
-          { //zip the document
-            String line=null;
-            zentry = new ZipEntry((String)qparams.get("filename"));
-            //get just the filename from the URL.
-            zout.putNextEntry(zentry);
-            //make a new entry in the zip file stream
-            while((line = htmldoc.readLine()) != null)
-            {
-              bytestring = (line.toString()).getBytes();
-              zout.write(bytestring, 0, bytestring.length);
-              //write out the file line by line
-            }
-            zout.closeEntry(); //close the entry in the file
-          }
-          catch(Exception e)
-          {
-            System.out.println("error downloading html document "+ 
-                               "in metacatServlet.handleGetDataDocumentAction"); 
-          }
-        }
-      }
+      DataFileServer dfs = new DataFileServer(port, username, sess_id);
+      dfs.start();
+      
+      //System.out.println("filedir: " + filedir);
+      //System.out.println("port: " + port);
+      response.setContentType("text/xml");
+      out.println("<?xml version=\"1.0\"?>");
+      out.println("<port>");
+      out.print(port);
+      out.print("</port>");
+      
     }
-    zout.finish();  //terminate the zip file
-    zout.close();   //close the stream.
-    util.returnConnection(conn); //return the connection to the pool
+	  catch (Exception e) 
+    {
+      System.out.println("error in MetacatServlet.handleGetDataPortAction: " + 
+                          e.getMessage());
+    }
   }
-  catch(Exception e)
-  {
-    System.out.println("Error creating zip file from " +
-                       "MetacatServlet.handleGetDataDocumentAction: " + 
-                        e.getMessage()); 
-    e.printStackTrace(System.out);
-  }
-           
-   /*
-   //////////old code using a shell script/////////////////////////////////
-   
-      boolean error_flag = false;
-      String error_message = "";
-      // Get the document indicated
-      String[] datadoc = (String[])params.get("datadoc");
-      // defaultdatapath = "C:\\Temp\\";    // for testing only!!!
-      // executescript = "test.bat";        // for testing only!!!
-      
-      // set content type and other response header fields first
-      response.setContentType("application/octet-stream");
-      if (defaultdatapath!=null) {
-        if(!defaultdatapath.endsWith(System.getProperty("file.separator"))) {
-          defaultdatapath=defaultdatapath+System.getProperty("file.separator");
-        }
-        System.out.println("Path= "+defaultdatapath+datadoc[0]);
-        if (executescript!=null) {
-          String command = null;
-          File scriptfile = new File(executescript);
-          if (scriptfile.exists()) {
-            command=executescript+" "+datadoc[0]; // script includes path
-        } else {     // look in defaultdatapath
-            // on Win98 one MUST include the .bat extender
-            command = defaultdatapath+executescript+" "+datadoc[0];  
-        }
-      System.out.println(command);
-      try {
-      Process proc = Runtime.getRuntime().exec(command);
-      proc.waitFor();
-      }
-      catch (Exception eee) {
-        System.out.println("Error running process!");
-        error_flag = true;
-        error_message = "Error running process!";}
-      } // end executescript not null if
-      File datafile = new File(defaultdatapath+datadoc[0]);
-      try {
-      FileInputStream fw = new FileInputStream(datafile);
-      int x;
-      while ((x = fw.read())!=-1) {
-        out.write(x); }
-        fw.close();
-      } catch (Exception e) {
-        System.out.println("Error in returning file\n"+e.getMessage());
-        error_flag=true;
-        error_message = error_message+"\nError in returning file\n"+
-                        e.getMessage();
-      }
-    } // end defaultdatapath not null if
-    */
-  }
   
   /** 
    * Handle "getaccesscontrol" action.
@@ -1603,6 +1336,33 @@
   }
 
   /** 
+   * Handle the "getprincipals" action.
+   * Read all principals from authentication scheme in XML format
+   */
+  private void handleGetPrincipalsAction(PrintWriter out, String user,
+                                         String password) {
+
+    Connection conn = null;
+
+    try {
+
+        // get connection from the pool
+        AuthSession auth = new AuthSession();
+        String principals = auth.getPrincipals(user, password);
+        out.println(principals);
+
+    } catch (Exception e) {
+      out.println("<?xml version=\"1.0\"?>");
+      out.println("<error>");
+      out.println(e.getMessage());
+      out.println("</error>");
+    } finally {
+      util.returnConnection(conn);
+    }  
+    
+  }
+
+  /** 
    * Handle "getdoctypes" action.
    * Read all doctypes from db connection in XML format
    */
@@ -1702,31 +1462,4 @@
     
   }
 
-  /** 
-   * Handle the "getprincipals" action.
-   * Read all principals from authentication scheme in XML format
-   */
-  private void handleGetPrincipalsAction(PrintWriter out, String user,
-                                         String password) {
-
-    Connection conn = null;
-
-    try {
-
-        // get connection from the pool
-        AuthSession auth = new AuthSession();
-        String principals = auth.getPrincipals(user, password);
-        out.println(principals);
-
-    } catch (Exception e) {
-      out.println("<?xml version=\"1.0\"?>");
-      out.println("<error>");
-      out.println(e.getMessage());
-      out.println("</error>");
-    } finally {
-      util.returnConnection(conn);
-    }  
-    
-  }
-
 }
