Metacat

import java.io.BufferedInputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileReader;

import java.io.OutputStreamWriter;

import java.io.StringReader;

import java.io.Writer;

import java.net.MalformedURLException;

import java.net.URL;

import java.sql.Timestamp;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.HashMap;

import java.util.Iterator;

import java.util.Map;

import java.util.zip.ZipEntry;

import java.util.zip.ZipOutputStream;

import org.ecoinformatics.eml.EMLParser;

import au.com.bytecode.opencsv.CSVWriter;

import com.oreilly.servlet.multipart.FilePart;

import com.oreilly.servlet.multipart.MultipartParser;

import com.oreilly.servlet.multipart.ParamPart;

import com.oreilly.servlet.multipart.Part;

import edu.ucsb.nceas.metacat.accesscontrol.AccessControlException;

import edu.ucsb.nceas.metacat.accesscontrol.AccessControlForSingleFile;

import edu.ucsb.nceas.metacat.accesscontrol.AccessControlInterface;

import edu.ucsb.nceas.metacat.cart.CartManager;

import edu.ucsb.nceas.metacat.dataquery.DataQuery;

import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;

import edu.ucsb.nceas.metacat.shared.ServiceException;

import edu.ucsb.nceas.metacat.spatial.SpatialQuery;

import edu.ucsb.nceas.metacat.util.DocumentUtil;

import edu.ucsb.nceas.metacat.util.MetacatUtil;

import edu.ucsb.nceas.metacat.util.SessionData;

import edu.ucsb.nceas.utilities.LSIDUtil;

import edu.ucsb.nceas.utilities.ParseLSIDException;

    private static boolean _sitemapScheduled;

    private static final String PROLOG = "<?xml version=\"1.0\"?>";

    private static final String SUCCESS = "<success>";

    private static final String SUCCESSCLOSE = "</success>";

    private static final String ERROR = "<error>";

    private static final String ERRORCLOSE = "</error>";

//					&& PropertyService.getProperty("spatial.regenerateCacheOnRestart").equals("true")) {

				// if either the point or polygon shape files do not exist, then regenerate the entire spatial cache

				// this may be expensive with many documents

				if( FileUtil.getFileStatus(SpatialFeatureSchema.pointShpUri) == FileUtil.DOES_NOT_EXIST) {		

					SpatialHarvester sh = new SpatialHarvester();

					sh.regenerate();

					sh.destroy();

				}

			_sitemapScheduled = false;

				handleMultipartForm(request, response);

					handleReadAction(params, request, response, "public", null, null);

				handleLoginAction(out, params, request, response);

				handleLogoutAction(out, params, request, response);

				handleQuery(out, params, response, userName, groupNames, sessionId);

					handleSQuery(out, params, response, userName, groupNames, sessionId);

				handleSpatialQuery(out, params, response, userName, groupNames, sessionId);

				handleDataquery(params, response, sessionId);

				handleEditCart(params, response, sessionId);

				handleExportAction(params, response, userName, groupNames, password);

					handleReadAction(params, request, response, userName, password,

				handleReadInlineDataAction(params, request, response, userName, password,

					handleInsertOrUpdateAction(request, response, out, params, userName,

					handleDeleteAction(out, params, request, response, userName,

				handleValidateAction(out, params);

				handleSetAccessAction(out, params, userName, request, response);

				handleGetAccessControlAction(out, params, response, userName, groupNames);

				handleGetPrincipalsAction(out, userName, password);

				handleGetDoctypesAction(out, params, response);

				handleGetDTDSchemaAction(out, params, response);

				handleGetMaxDocidAction(out, params, response);

				handleGetAllDocidsAction(out, params, response);

				handleIdIsRegisteredAction(out, params, response);

				handleGetRevisionAndDocTypeAction(out, params);

				handleGetLogAction(params, request, response, userName, groupNames);

				handleBuildIndexAction(params, request, response, userName, groupNames);

			scheduleSitemapGeneration(request);

	private void handleDataquery(

			Hashtable<String, String[]> params,

            HttpServletResponse response,

            String sessionId) throws PropertyNotFoundException, IOException {

		DataQuery dq = null;

		if (sessionId != null) {

			dq = new DataQuery(sessionId);

		}

		else {

			dq = new DataQuery();

		}

    	String dataqueryXML = (params.get("dataquery"))[0];

    	ResultSet rs = null;

		try {

			rs = dq.executeQuery(dataqueryXML);

		} catch (Exception e) {

			//probably need to do something here

			e.printStackTrace();

			return;

		}

		//process the result set

		String qformat = "csv";

		String[] temp = params.get("qformat");

		if (temp != null && temp.length > 0) {

			qformat = temp[0];

		}

		String fileName = "query-results." + qformat;

		//get the results as csv file

		if (qformat != null && qformat.equalsIgnoreCase("csv")) {

			response.setContentType("text/csv");

			//response.setContentType("application/csv");

	        response.setHeader("Content-Disposition", "attachment; filename=" + fileName);

			Writer writer = new OutputStreamWriter(response.getOutputStream());

			CSVWriter csv = new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);

			try {

				csv.writeAll(rs, true);

				csv.flush();

				response.flushBuffer();

				rs.close();

			} catch (SQLException e) {

				e.printStackTrace();

			}

			return;

		}

	}

	private void handleEditCart(

			Hashtable<String, String[]> params,

            HttpServletResponse response,

            String sessionId) throws PropertyNotFoundException, IOException {

		CartManager cm = null;

		if (sessionId != null) {

			cm = new CartManager(sessionId);

		}

		else {

			cm = new CartManager();

		}

    	String editOperation = (params.get("operation"))[0];

    	String[] docids = params.get("docid");

    	String[] field = params.get("field");

    	String[] path = params.get("path");

    	Map<String,String> fields = null;

    	if (field != null && path != null) {

    		fields = new HashMap<String,String>();

    		fields.put(field[0], path[0]);

    	}

    	//TODO handle attribute map (metadata fields)

    	cm.editCart(editOperation, docids, fields);

	}

    // ///////////////////////////// METACAT SPATIAL ///////////////////////////

	 * handles all spatial queries -- these queries may include any of the

	 * queries supported by the WFS / WMS standards

	 * 

	 * handleSQuery(out, params, response, username, groupnames, sess_id);

	 */

    private void handleSpatialQuery(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletResponse response,

            String username, String[] groupnames,

            String sess_id) throws PropertyNotFoundException {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        if ( !PropertyService.getProperty("spatial.runSpatialOption").equals("true") ) {

            response.setContentType("text/html");

            out.println("<html> Metacat Spatial Option is turned off </html>");

            out.close();

            return ;

        }

        /*

		 * Perform spatial query against spatial cache

		 */

        float _xmax = Float.valueOf( (params.get("xmax"))[0] ).floatValue();

        float _ymax = Float.valueOf( (params.get("ymax"))[0] ).floatValue();

        float _xmin = Float.valueOf( (params.get("xmin"))[0] ).floatValue();

        float _ymin = Float.valueOf( (params.get("ymin"))[0] ).floatValue();

        SpatialQuery sq = new SpatialQuery();

        Vector<String> docids = sq.filterByBbox( _xmin, _ymin, _xmax, _ymax );

        // logMetacat.info(" --- Spatial Query completed. Passing on the SQuery

		// handler");

        // logMetacat.warn("\n\n ******* after spatial query, we've got " +

		// docids.size() + " docids \n\n");

        /*

		 * Create an array matching docids

		 */

        String [] docidArray = new String[docids.size()];

        docids.toArray(docidArray);

        /*

		 * Create squery string

		 */

        String squery = DocumentIdQuery.createDocidQuery( docidArray );

        // logMetacat.info("-----------\n" + squery + "\n------------------");

        String[] queryArray = new String[1];

        queryArray[0] = squery;

        params.put("query", queryArray);

        /*

		 * Determine qformat

		 */

        String[] qformatArray = new String[1];

        try {

            String _skin = (params.get("skin"))[0];

            qformatArray[0] = _skin;

        } catch (java.lang.NullPointerException e) {

            // should be "default" but keep this for backwards compatibility

			// with knp site

            logMetacat.warn("MetaCatServlet.handleSpatialQuery - No SKIN specified for metacat actions=spatial_query... defaulting to 'knp' skin !\n");

            qformatArray[0] = "knp";

        }

        params.put("qformat", qformatArray);

        // change the action

        String[] actionArray = new String[1];

        actionArray[0] = "squery";

        params.put("action", actionArray);

        /*

		 * Pass the docids to the DBQuery contructor

		 */

        // This is a hack to get the empty result set to show...

        // Otherwise dbquery sees no docidOverrides and does a full % percent

		// query

        if (docids.size() == 0)

            docids.add("");

        DBQuery queryobj = new DBQuery(docids);

        queryobj.findDocuments(response, out, params, username, groupnames, sess_id);

    }

    // LOGIN & LOGOUT SECTION

    /**

	 * Handle the login request. Create a new session object. Do user

	 * authentication through the session.

	 */

    private void handleLoginAction(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletRequest request, HttpServletResponse response) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        AuthSession sess = null;

        if(params.get("username") == null){

            response.setContentType("text/xml");

            out.println("<?xml version=\"1.0\"?>");

            out.println("<error>");

            out.println("Username not specified");

            out.println("</error>");

            return;

        }

        // }

        if(params.get("password") == null){

            response.setContentType("text/xml");

            out.println("<?xml version=\"1.0\"?>");

            out.println("<error>");

            out.println("Password not specified");

            out.println("</error>");

            return;

        }

        String un = (params.get("username"))[0];

        logMetacat.info("MetaCatServlet.handleLoginAction - user " + un + " is trying to login");

        String pw = (params.get("password"))[0];

        String qformat = "xml";

        if (params.get("qformat") != null) {

            qformat = (params.get("qformat"))[0];

        }

        try {

            sess = new AuthSession();

        } catch (Exception e) {

        	String errorMsg = "MetacatServlet.handleLoginAction - Problem in MetacatServlet.handleLoginAction() authenicating session: "

                + e.getMessage();

            logMetacat.error(errorMsg);

            out.println(errorMsg);

            return;

        }

        boolean isValid = sess.authenticate(request, un, pw);

        //if it is authernticate is true, store the session

        if (isValid) {

            HttpSession session = sess.getSessions();

            String id = session.getId();

            logMetacat.debug("MetaCatServlet.handleLoginAction - Store session id " + id

                    + " which has username" + session.getAttribute("username")

                    + " into hash in login method");

            try {

				SessionService.registerSession(id, 

						(String) session.getAttribute("username"), 

						(String[]) session.getAttribute("groupnames"), 

						(String) session.getAttribute("password"), 

						(String) session.getAttribute("name"));

			} catch (ServiceException se) {

				String errorMsg = "MetacatServlet.handleLoginAction - service problem registering session: "

						+ se.getMessage();

				logMetacat.error("MetaCatServlet.handleLoginAction - " + errorMsg);

				out.println(errorMsg);

				return;

			} 			

        }

        // format and transform the output

        if (qformat.equals("xml")) {

            response.setContentType("text/xml");

            out.println(sess.getMessage());

        } else {

            try {

                DBTransform trans = new DBTransform();

                response.setContentType("text/html");

                trans.transformXMLDocument(sess.getMessage(),

                        "-//NCEAS//login//EN", "-//W3C//HTML//EN", qformat,

                        out, null, null);

            } catch (Exception e) {               

                logMetacat.error("MetaCatServlet.handleLoginAction - General error"

                        + e.getMessage());

            }

        }

    }

    /**

     * Handle the logout request. Close the connection.

     */

    private void handleLogoutAction(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletRequest request, HttpServletResponse response) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        String qformat = "xml";

        if(params.get("qformat") != null){

            qformat = params.get("qformat")[0];

        }

        // close the connection

        HttpSession sess = request.getSession(false);

        logMetacat.info("MetaCatServlet.handleLogoutAction - After get session in logout request");

        if (sess != null) {

            logMetacat.info("MetaCatServlet.handleLogoutAction - The session id " + sess.getId()

            + " will be invalidate in logout action");

            logMetacat.info("MetaCatServlet.handleLogoutAction - The session contains user "

                    + sess.getAttribute("username")

                    + " will be invalidate in logout action");

            sess.invalidate();

            SessionService.unRegisterSession(sess.getId());

        }

        // produce output

        StringBuffer output = new StringBuffer();

        output.append("<?xml version=\"1.0\"?>");

        output.append("<logout>");

        output.append("User logged out");

        output.append("</logout>");

        //format and transform the output

        if (qformat.equals("xml")) {

            response.setContentType("text/xml");

            out.println(output.toString());

        } else {

            try {

                DBTransform trans = new DBTransform();

                response.setContentType("text/html");

                trans.transformXMLDocument(output.toString(),

                        "-//NCEAS//login//EN", "-//W3C//HTML//EN", qformat,

                        out, null, null);

            } catch (Exception e) {

                logMetacat.error(

                        "MetaCatServlet.handleLogoutAction - General error: "

                        + e.getMessage());

            }

        }

    }

    // END OF LOGIN & LOGOUT SECTION

    // SQUERY & QUERY SECTION

    /**

     * Retreive the squery xml, execute it and display it

     *

     * @param out the output stream to the client

     * @param params the Hashtable of parameters that should be included in the

     *            squery.

     * @param response the response object linked to the client

     * @param conn the database connection

     */

    private void handleSQuery(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletResponse response, String user, String[] groups,

            String sessionid) throws PropertyNotFoundException {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        long squeryWarnLimit = Long.parseLong(PropertyService.getProperty("database.squeryTimeWarnLimit"));

        long startTime = System.currentTimeMillis();

        DBQuery queryobj = new DBQuery();

        queryobj.findDocuments(response, out, params, user, groups, sessionid);

        long outPutTime = System.currentTimeMillis();

        long runTime = outPutTime - startTime;

        if (runTime > squeryWarnLimit) {

        	logMetacat.warn("MetaCatServlet.handleSQuery - Long running squery.  Total time: " + runTime + 

        			" ms, squery: " + ((String[])params.get("query"))[0]);

        }

        logMetacat.debug("MetaCatServlet.handleSQuery - squery: " + ((String[])params.get("query"))[0] + 

        		" ran in " + runTime + " ms");

    }

    /**

     * Create the xml query, execute it and display the results.

     *

     * @param out the output stream to the client

     * @param params the Hashtable of parameters that should be included in the

     *            squery.

     * @param response the response object linked to the client

     */

    private void handleQuery(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletResponse response, String user, String[] groups,

            String sessionid) throws PropertyNotFoundException {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        long queryWarnLimit = Long.parseLong(PropertyService.getProperty("database.queryTimeWarnLimit"));

        //create the query and run it

        String xmlquery = DBQuery.createSQuery(params);

        String[] queryArray = new String[1];

        queryArray[0] = xmlquery;

        params.put("query", queryArray);

        long startTime = System.currentTimeMillis();

        DBQuery queryobj = new DBQuery();

        queryobj.findDocuments(response, out, params, user, groups, sessionid);

        long outPutTime = System.currentTimeMillis();

        long runTime = outPutTime -startTime;

        if (runTime > queryWarnLimit) {

        	logMetacat.warn("MetaCatServlet.handleQuery - Long running squery.  Total time: " + runTime + 

        			" ms, squery: " + ((String[])params.get("query"))[0]);

        }

        logMetacat.debug("MetaCatServlet.handleQuery - query: " + ((String[])params.get("query"))[0] + 

        		" ran in " + runTime + " ms");

    }

    // END OF SQUERY & QUERY SECTION

    //Exoport section

    /**

     * Handle the "export" request of data package from Metacat in zip format

     *

     * @param params the Hashtable of HTTP request parameters

     * @param response the HTTP response object linked to the client

     * @param user the username sent the request

     * @param groups the user's groupnames

     */

    private void handleExportAction(Hashtable<String, String[]> params,

            HttpServletResponse response,

            String user, String[] groups, String passWord) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        // Output stream

        ServletOutputStream out = null;

        // Zip output stream

        ZipOutputStream zOut = null;

        DBQuery queryObj = null;

        String[] docs = new String[10];

        String docId = "";

        try {

            // read the params

            if (params.containsKey("docid")) {

                docs = params.get("docid");

            }

            // Create a DBuery to handle export

            queryObj = new DBQuery();

            // Get the docid

            docId = docs[0];

            // Make sure the client specify docid

            if (docId == null || docId.equals("")) {

                response.setContentType("text/xml"); //MIME type

                // Get a printwriter

                PrintWriter pw = response.getWriter();

                // Send back message

                pw.println("<?xml version=\"1.0\"?>");

                pw.println("<error>");

                pw.println("You didn't specify requested docid");

                pw.println("</error>");

                // Close printwriter

                pw.close();

                return;

            }

            // Get output stream

            out = response.getOutputStream();

            response.setContentType("application/zip"); //MIME type

            response.setHeader("Content-Disposition",

                    "attachment; filename="

                    + docId + ".zip"); // Set the name of the zip file

            zOut = new ZipOutputStream(out);

            zOut = queryObj

                    .getZippedPackage(docId, out, user, groups, passWord);

            zOut.finish(); //terminate the zip file

            zOut.close(); //close the zip stream

        } catch (Exception e) {

            try {

                response.setContentType("text/xml"); //MIME type

                // Send error message back

                if (out != null) {

                    PrintWriter pw = new PrintWriter(out);

                    pw.println("<?xml version=\"1.0\"?>");

                    pw.println("<error>");

                    pw.println(e.getMessage());

                    pw.println("</error>");

                    // Close printwriter

                    pw.close();

                    // Close output stream

                    out.close();

                }

                // Close zip output stream

                if (zOut != null) {

                    zOut.close();

                }

            } catch (IOException ioe) {

                logMetacat.error("MetaCatServlet.handleExportAction - Problem with the servlet output: "

                        + ioe.getMessage());

            }

            logMetacat.error("MetaCatServlet.handleExportAction - General error: "

                    + e.getMessage());

            e.printStackTrace(System.out);

        }

    }

    /**

     * In eml2 document, the xml can have inline data and data was stripped off

     * and store in file system. This action can be used to read inline data

     * only

     *

     * @param params the Hashtable of HTTP request parameters

     * @param response the HTTP response object linked to the client

     * @param user the username sent the request

     * @param groups the user's groupnames

     */

    private void handleReadInlineDataAction(Hashtable<String, String[]> params,

            HttpServletRequest request, HttpServletResponse response,

            String user, String passWord, String[] groups) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        String[] docs = new String[10];

        String inlineDataId = null;

        String docId = "";

        ServletOutputStream out = null;

        try {

            // read the params

            if (params.containsKey("inlinedataid")) {

                docs = params.get("inlinedataid");

            }

            // Get the docid

            inlineDataId = docs[0];

            // Make sure the client specify docid

            if (inlineDataId == null || inlineDataId.equals("")) {

                throw new Exception("You didn't specify requested inlinedataid"); }

            // check for permission

            docId = 

            	DocumentUtil.getDocIdWithoutRevFromInlineDataID(inlineDataId);

            PermissionController controller = new PermissionController(docId);

            // check top level read permission

            if (!controller.hasPermission(user, groups,

                    AccessControlInterface.READSTRING)) {

                throw new Exception("User " + user

                        + " doesn't have permission " + " to read document "

                        + docId);

            } else {

                //check data access level

                try {

                    Hashtable<String,String> unReadableInlineDataList =

                            PermissionController.getUnReadableInlineDataIdList(docId,

                            user, groups, false);

                    String inlineDataIdWithoutRev = DocumentUtil.getInlineDataIdWithoutRev(inlineDataId);

                    if (unReadableInlineDataList.containsValue(inlineDataIdWithoutRev)) {

                        throw new Exception("User " + user

                                + " doesn't have permission " + " to read inlinedata "

                                + inlineDataId);

                    }//if

                }//try

                catch (Exception e) {

                    throw e;

                }//catch

            }//else

            // Get output stream

            out = response.getOutputStream();

            // read the inline data from the file

            String inlinePath = PropertyService.getProperty("application.inlinedatafilepath");

            File lineData = new File(inlinePath, inlineDataId);

            FileInputStream input = new FileInputStream(lineData);

            byte[] buffer = new byte[4 * 1024];

            int bytes = input.read(buffer);

            while (bytes != -1) {

                out.write(buffer, 0, bytes);

                bytes = input.read(buffer);

            }

            out.close();

            EventLog.getInstance().log(request.getRemoteAddr(), user,

                    inlineDataId, "readinlinedata");

        } catch (Exception e) {

            try {

                PrintWriter pw = null;

                // Send error message back

                if (out != null) {

                    pw = new PrintWriter(out);

                } else {

                    pw = response.getWriter();

                }

                pw.println("<?xml version=\"1.0\"?>");

                pw.println("<error>");

                pw.println(e.getMessage());

                pw.println("</error>");

                // Close printwriter

                pw.close();

                // Close output stream if out is not null

                if (out != null) {

                    out.close();

                }

            } catch (IOException ioe) {

                logMetacat.error("MetaCatServlet.handleReadInlineDataAction - Problem with the servlet output: "

                        + ioe.getMessage());

            }

            logMetacat.error("MetaCatServlet.handleReadInlineDataAction - General error: "

                    + e.getMessage());

        }

    }

    /**

     * Handle the "read" request of metadata/data files from Metacat or any

     * files from Internet; transformed metadata XML document into HTML

     * presentation if requested; zip files when more than one were requested.

     *

     * @param params the Hashtable of HTTP request parameters

     * @param request the HTTP request object linked to the client

     * @param response the HTTP response object linked to the client

     * @param user the username sent the request

     * @param groups the user's groupnames

     */

    private void handleReadAction(Hashtable<String, String[]> params, HttpServletRequest request,

            HttpServletResponse response, String user, String passWord,

            String[] groups) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        ServletOutputStream out = null;

        ZipOutputStream zout = null;

        PrintWriter pw = null;

        boolean zip = false;

        boolean withInlineData = true;

        try {

            String[] docs = new String[0];

            String docid = "";

            String qformat = "";

            String abstrpath = null;

            // read the params

            if (params.containsKey("docid")) {

                docs = params.get("docid");

            }

            if (params.containsKey("qformat")) {

                qformat = params.get("qformat")[0];

            }

            // the param for only metadata (eml)

            // we don't support read a eml document without inline data now.

            /*if (params.containsKey("inlinedata")) {

                String inlineData = ((String[]) params.get("inlinedata"))[0];

                if (inlineData.equalsIgnoreCase("false")) {

                    withInlineData = false;

                }

            }*/

            if ((docs.length > 1) || qformat.equals("zip")) {

                zip = true;

                out = response.getOutputStream();

                response.setContentType("application/zip"); //MIME type

                zout = new ZipOutputStream(out);

            }

            // go through the list of docs to read

            for (int i = 0; i < docs.length; i++) {

            	String providedFileName = null;

            	if (params.containsKey(docs[i])) {

                    providedFileName = params.get(docs[i])[0];

                }

                try {

                    URL murl = new URL(docs[i]);

                    Hashtable<String,String> murlQueryStr = MetacatUtil.parseQuery(

                            murl.getQuery());

                    // case docid="http://.../?docid=aaa"

                    // or docid="metacat://.../?docid=bbb"

                    if (murlQueryStr.containsKey("docid")) {

                        // get only docid, eliminate the rest

                        docid = murlQueryStr.get("docid");

                        if (zip) {

                            addDocToZip(request, docid, providedFileName, zout, user, groups);

                        } else {

                            readFromMetacat(request, response, docid, qformat,

                                    abstrpath, user, groups, zip, zout,

                                    withInlineData, params);

                        }

                        // case docid="http://.../filename"

                    } else {

                        docid = docs[i];

                        if (zip) {

                            addDocToZip(request, docid, providedFileName, zout, user, groups);

                        } else {

                            readFromURLConnection(response, docid);

                        }

                    }

                } catch (MalformedURLException mue) {

                    docid = docs[i];

                    if (zip) {

                        addDocToZip(request, docid, providedFileName, zout, user, groups);

                    } else {

                        readFromMetacat(request, response, docid, qformat,

                                abstrpath, user, groups, zip, zout,

                                withInlineData, params);

                    }

                }

            }

            if (zip) {

                zout.finish(); //terminate the zip file

                zout.close(); //close the zip stream

            }

        } catch (McdbDocNotFoundException notFoundE) {

            // To handle doc not found exception

            // the docid which didn't be found

            String notFoundDocId = notFoundE.getUnfoundDocId();

            String notFoundRevision = notFoundE.getUnfoundRevision();

            logMetacat.warn("MetaCatServlet.handleReadAction - Missed id: " + notFoundDocId);

            logMetacat.warn("MetaCatServlet.handleReadAction - Missed rev: " + notFoundRevision);

            try {

                // read docid from remote server

                readFromRemoteMetaCat(response, notFoundDocId,

                        notFoundRevision, user, passWord, out, zip, zout);

                // Close zout outputstream

                if (zout != null) {

                    zout.close();

                }

                // close output stream

                if (out != null) {

                    out.close();

                }

            } catch (Exception exc) {

                logMetacat.error("MetaCatServlet.handleReadAction - General error: "

                        + exc.getMessage());

                try {

                    if (out != null) {

                        response.setContentType("text/xml");

                        // Send back error message by printWriter

                        pw = new PrintWriter(out);

                        pw.println("<?xml version=\"1.0\"?>");

                        pw.println("<error>");

                        pw.println(notFoundE.getMessage());

                        pw.println("</error>");

                        pw.close();

                        out.close();

                    } else {

                        response.setContentType("text/xml"); //MIME type

                        // Send back error message if out = null

                        if (pw == null) {

                            // If pw is null, open the respnose

                            pw = response.getWriter();

                        }

                        pw.println("<?xml version=\"1.0\"?>");

                        pw.println("<error>");

                        pw.println(notFoundE.getMessage());

                        pw.println("</error>");

                        pw.close();

                    }

                    // close zout

                    if (zout != null) {

                        zout.close();

                    }

                } catch (IOException ie) {

                    logMetacat.error("MetaCatServlet.handleReadAction - Problem with the servlet output: "

                            + ie.getMessage());

                }

            }

        } catch (Exception e) {

            try {

                if (out != null) {

                    response.setContentType("text/xml"); //MIME type

                    pw = new PrintWriter(out);

                    pw.println("<?xml version=\"1.0\"?>");

                    pw.println("<error>");

                    pw.println(e.getMessage());

                    pw.println("</error>");

                    pw.close();

                    out.close();

                } else {

                    response.setContentType("text/xml"); //MIME type

                    // Send back error message if out = null

                    if (pw == null) {

                        pw = response.getWriter();

                    }

                    pw.println("<?xml version=\"1.0\"?>");

                    pw.println("<error>");

                    pw.println(e.getMessage());

                    pw.println("</error>");

                    pw.close();

                }

                // Close zip output stream

                if (zout != null) {

                    zout.close();

                }

            } catch (Exception e2) {

                logMetacat.error("MetaCatServlet.handleReadAction - Problem with the servlet output: "

                        + e2.getMessage());

                e2.printStackTrace(System.out);

            }

            logMetacat.error("MetaCatServlet.handleReadAction - General error: "

                    + e.getMessage());

            //e.printStackTrace(System.out);

        }

    }

    /** read metadata or data from Metacat

     */

    private void readFromMetacat(HttpServletRequest request,

            HttpServletResponse response, String docid, String qformat,

            String abstrpath, String user, String[] groups, boolean zip,

            ZipOutputStream zout, boolean withInlineData, Hashtable<String, String[]> params)

            throws ClassNotFoundException, IOException, SQLException,

            McdbException, Exception {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        try {

            if (docid.startsWith("urn:")) {

            	docid = LSIDUtil.getDocId(docid, true);               	

            }

            // here is hack for handle docid=john.10(in order to tell mike.jim.10.1

            // mike.jim.10, we require to provide entire docid with rev). But

            // some old client they only provide docid without rev, so we need

            // to handle this suituation. First we will check how many

            // seperator here, if only one, we will append the rev in xml_documents

            // to the id.

            docid = appendRev(docid);

            DocumentImpl doc = new DocumentImpl(docid);

            //check the permission for read

            if (!DocumentImpl.hasReadPermission(user, groups, docid)) {

                Exception e = new Exception("User " + user

                        + " does not have permission"

                        + " to read the document with the docid " + docid);

                throw e;

            }

            if (doc.getRootNodeID() == 0) {

                // this is data file

                String filepath = PropertyService.getProperty("application.datafilepath");

                if (!filepath.endsWith("/")) {

                    filepath += "/";

                }

                String filename = filepath + docid;

                FileInputStream fin = null;

                fin = new FileInputStream(filename);

                //MIME type

                String contentType = getServletContext().getMimeType(filename);

                if (contentType == null) {

                    ContentTypeProvider provider = new ContentTypeProvider(

                            docid);

                    contentType = provider.getContentType();

                    logMetacat.info("MetaCatServlet.readFromMetacat - Final contenttype is: "

                            + contentType);

                }

                response.setContentType(contentType);

                // if we decide to use "application/octet-stream" for all data

                // returns

                // response.setContentType("application/octet-stream");

                // check for the existence of a metadatadocid parameter,

                // if this is sent, then send a filename which contains both

                // the metadata docid and the data docid, so the link with

                // metadata is explicitly encoded in the filename.

                String metadatadocid = null;

                Vector<String> nameparts = new Vector<String>();

                if(params.containsKey("metadatadocid")) {

                    metadatadocid = params.get("metadatadocid")[0];

                }

                if (metadatadocid != null && !metadatadocid.equals("")) {

                    nameparts.add(metadatadocid);

                }

                // we'll always have the docid, include it in the name

                String doctype = doc.getDoctype();

								// TODO: fix this to lookup the associated FGDC metadata document,

								// and grab the doctype tag for it.  These should be set to something 

								// consistent, not 'metadata' as it stands...

                //if (!doctype.equals("metadata")) {

                //    nameparts.add(docid);

                //} 

								nameparts.add(docid);

                // Set the name of the data file to the entity name plus docid,

                // or if that is unavailable, use the docid alone

                String docname = doc.getDocname();

                if (docname != null && !docname.equals("")) {

                    nameparts.add(docname); 

                }

                // combine the name elements with a dash, using a 'join' equivalent

                String outputname = null;

                String delimiter = "-";

                Iterator<String> iter = nameparts.iterator();

                StringBuffer buffer = new StringBuffer(iter.next());

                while (iter.hasNext()) buffer.append(delimiter).append(iter.next());

                outputname = buffer.toString();    

                response.setHeader("Content-Disposition",

                        "attachment; filename=\"" + outputname + "\"");

                try {

                    ServletOutputStream out = response.getOutputStream();

                    byte[] buf = new byte[4 * 1024]; // 4K buffer

                    int b = fin.read(buf);

                    while (b != -1) {

                        out.write(buf, 0, b);

                        b = fin.read(buf);

                    }

                } finally {

                    if (fin != null) fin.close();

                }

            } else {

                // this is metadata doc

            	ServletOutputStream streamOut = response.getOutputStream();

                if (qformat.equals("xml") || qformat.equals("")) {

                    // if equals "", that means no qformat is specified. hence

                    // by default the document should be returned in xml format

                    // set content type first

                    response.setContentType("text/xml"); //MIME type

                    response.setHeader("Content-Disposition",

                            "attachment; filename=" + docid + ".xml");

                    // Try to get the metadata file from disk. If it isn't

					// found, create it from the db and write it to disk then.

					try {

						PrintWriter out = new PrintWriter(streamOut);

						doc.toXml(out, user, groups, withInlineData);				

					} catch (Exception e) {

						// any exceptions in reading the xml from disc, and we go back to the

						// old way of creating the xml directly.

						logMetacat.error("MetaCatServlet.readFromMetacat - could not read from document file " + docid 

								+ ": " + e.getMessage());

						PrintWriter out = new PrintWriter(streamOut);

						doc.toXmlFromDb(out, user, groups, withInlineData);

					}

                } else {

                	// TODO MCD, this should read from disk as well?

                    //*** This is a metadata doc, to be returned in a skin/custom format.

                    //*** Add param to indicate if public has read access or not.

                    if (!user.equals("public")) {

                        if (DocumentImpl.hasReadPermission("public", null, docid))

                            params.put("publicRead", new String[] {"true"});

                        else

                            params.put("publicRead", new String[] {"false"});

                    }

                    response.setContentType("text/html"); //MIME type

                    PrintWriter out = new PrintWriter(streamOut);

                    // Look up the document type

                    String doctype = doc.getDoctype();

                    // Transform the document to the new doctype

                    DBTransform dbt = new DBTransform();

                    dbt.transformXMLDocument(doc.toString(user, groups,

                            withInlineData), doctype, "-//W3C//HTML//EN",

                            qformat, out, params, null);

                }

            }

            EventLog.getInstance().log(request.getRemoteAddr(), user,

                    docid, "read");

        } catch (Exception except) {

            throw except;

        }

    }

    /**

     * read data from URLConnection

     */

    private void readFromURLConnection(HttpServletResponse response,

            String docid) throws IOException, MalformedURLException {

        ServletOutputStream out = response.getOutputStream();

        String contentType = getServletContext().getMimeType(docid); //MIME

        // type

        if (contentType == null) {

            if (docid.endsWith(".xml")) {

                contentType = "text/xml";

            } else if (docid.endsWith(".css")) {

                contentType = "text/css";

            } else if (docid.endsWith(".dtd")) {

                contentType = "text/plain";

            } else if (docid.endsWith(".xsd")) {

                contentType = "text/xml";

            } else if (docid.endsWith("/")) {

                contentType = "text/html";

            } else {

                File f = new File(docid);

                if (f.isDirectory()) {

                    contentType = "text/html";

                } else {

                    contentType = "application/octet-stream";

                }

            }

        }

        response.setContentType(contentType);

        // if we decide to use "application/octet-stream" for all data returns

        // response.setContentType("application/octet-stream");

        // this is http url

        URL url = new URL(docid);

        BufferedInputStream bis = null;

        try {

            bis = new BufferedInputStream(url.openStream());

            byte[] buf = new byte[4 * 1024]; // 4K buffer

            int b = bis.read(buf);

            while (b != -1) {

                out.write(buf, 0, b);

                b = bis.read(buf);

            }

        } finally {

            if (bis != null) bis.close();

        }

    }

    /**

     * read file/doc and write to ZipOutputStream

     *

     * @param docid

     * @param zout

     * @param user

     * @param groups

     * @throws ClassNotFoundException

     * @throws IOException

     * @throws SQLException

     * @throws McdbException

     * @throws Exception

     */

    private void addDocToZip(HttpServletRequest request, String docid, String providedFileName,

            ZipOutputStream zout, String user, String[] groups) throws

            ClassNotFoundException, IOException, SQLException, McdbException,

            Exception {

        byte[] bytestring = null;

        ZipEntry zentry = null;

        try {

            URL url = new URL(docid);

            // this http url; read from URLConnection; add to zip

            //use provided file name if we have one

            if (providedFileName != null && providedFileName.length() > 1) {

            	zentry = new ZipEntry(providedFileName);

            }

            else {

            	zentry = new ZipEntry(docid);

            }

            zout.putNextEntry(zentry);

            BufferedInputStream bis = null;

            try {

                bis = new BufferedInputStream(url.openStream());

                byte[] buf = new byte[4 * 1024]; // 4K buffer

                int b = bis.read(buf);

                while (b != -1) {

                    zout.write(buf, 0, b);

                    b = bis.read(buf);

                }

            } finally {

                if (bis != null) bis.close();

            }

            zout.closeEntry();

        } catch (MalformedURLException mue) {

            // this is metacat doc (data file or metadata doc)

            try {

                DocumentImpl doc = new DocumentImpl(docid);

                //check the permission for read

                if (!DocumentImpl.hasReadPermission(user, groups, docid)) {

                    Exception e = new Exception("User " + user

                            + " does not have "

                            + "permission to read the document with the docid "

                            + docid);

                    throw e;

                }

                if (doc.getRootNodeID() == 0) {

                    // this is data file; add file to zip

                    String filepath = PropertyService.getProperty("application.datafilepath");

                    if (!filepath.endsWith("/")) {

                        filepath += "/";

                    }

                    String filename = filepath + docid;

                    FileInputStream fin = null;

                    fin = new FileInputStream(filename);

                    try {

                    	//use provided file name if we have one

                        if (providedFileName != null && providedFileName.length() > 1) {

                        	zentry = new ZipEntry(providedFileName);

                        }

                        else {

                        	zentry = new ZipEntry(docid);

                        }

                        zout.putNextEntry(zentry);

                        byte[] buf = new byte[4 * 1024]; // 4K buffer

                        int b = fin.read(buf);

                        while (b != -1) {

                            zout.write(buf, 0, b);

                            b = fin.read(buf);

                        }

                    } finally {

                        if (fin != null) fin.close();

                    }

                    zout.closeEntry();

                } else {

                    // this is metadata doc; add doc to zip

                    bytestring = doc.toString().getBytes();

                    //use provided file name if given

                    if (providedFileName != null && providedFileName.length() > 1) {

                    	zentry = new ZipEntry(providedFileName);

                    }

                    else {

                    	zentry = new ZipEntry(docid + ".xml");

                    }

                    zentry.setSize(bytestring.length);

                    zout.putNextEntry(zentry);

                    zout.write(bytestring, 0, bytestring.length);

                    zout.closeEntry();

                }

                EventLog.getInstance().log(request.getRemoteAddr(), user,

                        docid, "read");

            } catch (Exception except) {

                throw except;

            }

        }

    }

    /**

     * If metacat couldn't find a data file or document locally, it will read

     * this docid from its home server. This is for the replication feature

     */

    private void readFromRemoteMetaCat(HttpServletResponse response,

            String docid, String rev, String user, String password,

            ServletOutputStream out, boolean zip, ZipOutputStream zout)

            throws Exception {

        // Create a object of RemoteDocument, "" is for zipEntryPath

        RemoteDocument remoteDoc = new RemoteDocument(docid, rev, user,

                password, "");

        String docType = remoteDoc.getDocType();

        // Only read data file

        if (docType.equals("BIN")) {

            // If it is zip format

            if (zip) {

                remoteDoc.readDocumentFromRemoteServerByZip(zout);

            } else {

                if (out == null) {

                    out = response.getOutputStream();

                }

                response.setContentType("application/octet-stream");

                remoteDoc.readDocumentFromRemoteServer(out);

            }

        } else {

            throw new Exception("Docid: " + docid + "." + rev

                    + " couldn't find");

        }

    }

    /**

     * Handle the database putdocument request and write an XML document to the

     * database connection

     */

    private void handleInsertOrUpdateAction(HttpServletRequest request,

            HttpServletResponse response, PrintWriter out, Hashtable<String, String[]> params,

            String user, String[] groups) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        DBConnection dbConn = null;

        int serialNumber = -1;

        String output = "";

        String qformat = null;

        if(params.containsKey("qformat"))

        {

          qformat = params.get("qformat")[0];

        }

        if(params.get("docid") == null){

            out.println("<?xml version=\"1.0\"?>");

            out.println("<error>");

            out.println("Docid not specified");

            out.println("</error>");

            logMetacat.error("MetaCatServlet.handleInsertOrUpdateAction - Docid not specified");

            return;

        }