Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2010 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *

 *   '$Author: jones $'

 *     '$Date: 2010-02-03 17:58:12 -0900 (Wed, 03 Feb 2010) $'

 * '$Revision: 5211 $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

import java.io.BufferedInputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileReader;

import java.io.IOException;

import java.io.PrintWriter;

import java.io.StringReader;

import java.net.MalformedURLException;

import java.net.URL;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Timestamp;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Enumeration;

import java.util.HashMap;

import java.util.Hashtable;

import java.util.Iterator;

import java.util.Map;

import java.util.Timer;

import java.util.Vector;

import java.util.zip.ZipEntry;

import java.util.zip.ZipOutputStream;

import javax.servlet.ServletContext;

import javax.servlet.ServletOutputStream;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import javax.servlet.http.HttpSession;

import org.apache.log4j.Logger;

import org.ecoinformatics.eml.EMLParser;

import au.com.bytecode.opencsv.CSVWriter;

import com.oreilly.servlet.multipart.FilePart;

import com.oreilly.servlet.multipart.MultipartParser;

import com.oreilly.servlet.multipart.ParamPart;

import com.oreilly.servlet.multipart.Part;

import edu.ucsb.nceas.metacat.accesscontrol.AccessControlException;

import edu.ucsb.nceas.metacat.accesscontrol.AccessControlForSingleFile;

import edu.ucsb.nceas.metacat.accesscontrol.AccessControlInterface;

import edu.ucsb.nceas.metacat.cart.CartManager;

import edu.ucsb.nceas.metacat.database.DBConnectionPool;

import edu.ucsb.nceas.metacat.dataquery.DataQuery;

import edu.ucsb.nceas.metacat.properties.PropertyService;

import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;

import edu.ucsb.nceas.metacat.service.SessionService;

import edu.ucsb.nceas.metacat.service.XMLSchemaService;

import edu.ucsb.nceas.metacat.shared.MetacatUtilException;

import edu.ucsb.nceas.metacat.shared.ServiceException;

import edu.ucsb.nceas.metacat.spatial.SpatialHarvester;

import edu.ucsb.nceas.metacat.spatial.SpatialQuery;

import edu.ucsb.nceas.metacat.util.AuthUtil;

import edu.ucsb.nceas.metacat.util.DocumentUtil;

import edu.ucsb.nceas.metacat.util.MetacatUtil;

import edu.ucsb.nceas.metacat.util.RequestUtil;

import edu.ucsb.nceas.metacat.util.SystemUtil;

import edu.ucsb.nceas.utilities.FileUtil;

import edu.ucsb.nceas.utilities.LSIDUtil;

import edu.ucsb.nceas.utilities.ParseLSIDException;

import edu.ucsb.nceas.utilities.PropertyNotFoundException;

 * General entry point for the Metacat server which is called from various

 * mechanisms such as the standard MetacatServlet class and the various web

 * service servlets such as RestServlet class.  All application logic should be

 * encapsulated in this class, and the calling classes should only contain

 * parameter marshaling and demarshaling code, delegating all else to this

 * MetacatHandler instance.

 * @author Matthew Jones

 */

    private static boolean _sitemapScheduled = false;

    // Constants -- these should be final in a servlet

    private static final String PROLOG = "<?xml version=\"1.0\"?>";

    private static final String SUCCESS = "<success>";

    private static final String SUCCESSCLOSE = "</success>";

    private static final String ERROR = "<error>";

    private static final String ERRORCLOSE = "</error>";

	private Timer timer;

    }

    protected void handleDataquery(

            Hashtable<String, String[]> params,

            HttpServletResponse response,

            String sessionId) throws PropertyNotFoundException, IOException {

        DataQuery dq = null;

        if (sessionId != null) {

            dq = new DataQuery(sessionId);

        }

        else {

            dq = new DataQuery();

        }

        String dataqueryXML = (params.get("dataquery"))[0];

        ResultSet rs = null;

        try {

            rs = dq.executeQuery(dataqueryXML);

        } catch (Exception e) {

            //probably need to do something here

            e.printStackTrace();

            return;

        }

        //process the result set

        String qformat = "csv";

        String[] temp = params.get("qformat");

        if (temp != null && temp.length > 0) {

            qformat = temp[0];

        }

        String fileName = "query-results." + qformat;

        //get the results as csv file

        if (qformat != null && qformat.equalsIgnoreCase("csv")) {

            response.setContentType("text/csv");

            //response.setContentType("application/csv");

            response.setHeader("Content-Disposition", "attachment; filename=" + fileName);

            Writer writer = new OutputStreamWriter(response.getOutputStream());

            CSVWriter csv = new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);

            try {

                csv.writeAll(rs, true);

                csv.flush();

                response.flushBuffer();

            } catch (SQLException e) {

                e.printStackTrace();

            }

            return;

        }

    }

    protected void handleEditCart(

            Hashtable<String, String[]> params,

            HttpServletResponse response,

            String sessionId) throws PropertyNotFoundException, IOException {

        CartManager cm = null;

        if (sessionId != null) {

            cm = new CartManager(sessionId);

        }

        else {

            cm = new CartManager();

        }

        String editOperation = (params.get("operation"))[0];

        String[] docids = params.get("docid");

        String[] field = params.get("field");

        String[] path = params.get("path");

        Map<String,String> fields = null;

        if (field != null && path != null) {

            fields = new HashMap<String,String>();

            fields.put(field[0], path[0]);

        }

        //TODO handle attribute map (metadata fields)

        cm.editCart(editOperation, docids, fields);

    }

    // ///////////////////////////// METACAT SPATIAL ///////////////////////////

    /**

     * handles all spatial queries -- these queries may include any of the

     * queries supported by the WFS / WMS standards

     *

     * handleSQuery(out, params, response, username, groupnames, sess_id);

     */

    protected void handleSpatialQuery(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletResponse response,

            String username, String[] groupnames,

            String sess_id) throws PropertyNotFoundException {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        if ( !PropertyService.getProperty("spatial.runSpatialOption").equals("true") ) {

            response.setContentType("text/html");

            out.println("<html> Metacat Spatial Option is turned off </html>");

            out.close();

            return ;

        }

        /*

         * Perform spatial query against spatial cache

         */

        float _xmax = Float.valueOf( (params.get("xmax"))[0] ).floatValue();

        float _ymax = Float.valueOf( (params.get("ymax"))[0] ).floatValue();

        float _xmin = Float.valueOf( (params.get("xmin"))[0] ).floatValue();

        float _ymin = Float.valueOf( (params.get("ymin"))[0] ).floatValue();

        SpatialQuery sq = new SpatialQuery();

        Vector<String> docids = sq.filterByBbox( _xmin, _ymin, _xmax, _ymax );

        // logMetacat.info(" --- Spatial Query completed. Passing on the SQuery

        // handler");

        // logMetacat.warn("\n\n ******* after spatial query, we've got " +

        // docids.size() + " docids \n\n");

        /*

         * Create an array matching docids

         */

        String [] docidArray = new String[docids.size()];

        docids.toArray(docidArray);

        /*

         * Create squery string

         */

        String squery = DocumentIdQuery.createDocidQuery( docidArray );

        // logMetacat.info("-----------\n" + squery + "\n------------------");

        String[] queryArray = new String[1];

        queryArray[0] = squery;

        params.put("query", queryArray);

        /*

         * Determine qformat

         */

        String[] qformatArray = new String[1];

        try {

            String _skin = (params.get("skin"))[0];

            qformatArray[0] = _skin;

        } catch (java.lang.NullPointerException e) {

            // should be "default" but keep this for backwards compatibility

            // with knp site

            logMetacat.warn("MetaCatServlet.handleSpatialQuery - No SKIN specified for metacat actions=spatial_query... defaulting to 'knp' skin !\n");

            qformatArray[0] = "knp";

        }

        params.put("qformat", qformatArray);

        // change the action

        String[] actionArray = new String[1];

        actionArray[0] = "squery";

        params.put("action", actionArray);

        /*

         * Pass the docids to the DBQuery contructor

         */

        // This is a hack to get the empty result set to show...

        // Otherwise dbquery sees no docidOverrides and does a full % percent

        // query

        if (docids.size() == 0)

            docids.add("");

        DBQuery queryobj = new DBQuery(docids);

        queryobj.findDocuments(response, out, params, username, groupnames, sess_id);

    }

    // LOGIN & LOGOUT SECTION

    /**

     * Handle the login request. Create a new session object. Do user

     * authentication through the session.

     */

    public void handleLoginAction(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletRequest request, HttpServletResponse response) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        AuthSession sess = null;

        if(params.get("username") == null){

            response.setContentType("text/xml");

            out.println("<?xml version=\"1.0\"?>");

            out.println("<error>");

            out.println("Username not specified");

            out.println("</error>");

            return;

        }

        // }

        if(params.get("password") == null){

            response.setContentType("text/xml");

            out.println("<?xml version=\"1.0\"?>");

            out.println("<error>");

            out.println("Password not specified");

            out.println("</error>");

            return;

        }

        String un = (params.get("username"))[0];

        logMetacat.info("MetaCatServlet.handleLoginAction - user " + un + " is trying to login");

        String pw = (params.get("password"))[0];

        String qformat = "xml";

        if (params.get("qformat") != null) {

            qformat = (params.get("qformat"))[0];

        }

        try {

            sess = new AuthSession();

        } catch (Exception e) {

            String errorMsg = "MetacatServlet.handleLoginAction - Problem in MetacatServlet.handleLoginAction() authenicating session: "

                + e.getMessage();

            logMetacat.error(errorMsg);

            out.println(errorMsg);

        }

        boolean isValid = sess.authenticate(request, un, pw);

        //if it is authernticate is true, store the session

        if (isValid) {

            HttpSession session = sess.getSessions();

            String id = session.getId();

            logMetacat.debug("MetaCatServlet.handleLoginAction - Store session id " + id

                    + " which has username" + session.getAttribute("username")

                    + " into hash in login method");

            try {

                SessionService.registerSession(id,

                        (String) session.getAttribute("username"),

                        (String[]) session.getAttribute("groupnames"),

                        (String) session.getAttribute("password"),

                        (String) session.getAttribute("name"));

            } catch (ServiceException se) {

                String errorMsg = "MetacatServlet.handleLoginAction - service problem registering session: "

                        + se.getMessage();

                logMetacat.error("MetaCatServlet.handleLoginAction - " + errorMsg);

                out.println(errorMsg);

            }

        }

        // format and transform the output

        if (qformat.equals("xml")) {

            response.setContentType("text/xml");

            out.println(sess.getMessage());

        } else {

            try {

                DBTransform trans = new DBTransform();

                response.setContentType("text/html");

                trans.transformXMLDocument(sess.getMessage(),

                        "-//NCEAS//login//EN", "-//W3C//HTML//EN", qformat,

                        out, null, null);

            } catch (Exception e) {

                logMetacat.error("MetaCatServlet.handleLoginAction - General error"

                        + e.getMessage());

        }

    }

    /**

     * Handle the logout request. Close the connection.

     */

    public void handleLogoutAction(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletRequest request, HttpServletResponse response) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        String qformat = "xml";

        if(params.get("qformat") != null){

            qformat = params.get("qformat")[0];

        }

        // close the connection

        HttpSession sess = request.getSession(false);

        logMetacat.info("MetaCatServlet.handleLogoutAction - After get session in logout request");

        if (sess != null) {

            logMetacat.info("MetaCatServlet.handleLogoutAction - The session id " + sess.getId()

            + " will be invalidate in logout action");

            logMetacat.info("MetaCatServlet.handleLogoutAction - The session contains user "

                    + sess.getAttribute("username")

                    + " will be invalidate in logout action");

            sess.invalidate();

            SessionService.unRegisterSession(sess.getId());

        }

        // produce output

        StringBuffer output = new StringBuffer();

        output.append("<?xml version=\"1.0\"?>");

        output.append("<logout>");

        output.append("User logged out");

        output.append("</logout>");

        //format and transform the output

        if (qformat.equals("xml")) {

            response.setContentType("text/xml");

            out.println(output.toString());

        } else {

            try {

                DBTransform trans = new DBTransform();

                response.setContentType("text/html");

                trans.transformXMLDocument(output.toString(),

                        "-//NCEAS//login//EN", "-//W3C//HTML//EN", qformat,

                        out, null, null);

            } catch (Exception e) {

                logMetacat.error(

                        "MetaCatServlet.handleLogoutAction - General error: "

                        + e.getMessage());

        }

    }

    // END OF LOGIN & LOGOUT SECTION

    // SQUERY & QUERY SECTION

    /**

     * Retreive the squery xml, execute it and display it

     *

     * @param out the output stream to the client

     * @param params the Hashtable of parameters that should be included in the

     *            squery.

     * @param response the response object linked to the client

     * @param conn the database connection

     */

    protected void handleSQuery(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletResponse response, String user, String[] groups,

            String sessionid) throws PropertyNotFoundException {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        long squeryWarnLimit = Long.parseLong(PropertyService.getProperty("database.squeryTimeWarnLimit"));

        long startTime = System.currentTimeMillis();

        DBQuery queryobj = new DBQuery();

        queryobj.findDocuments(response, out, params, user, groups, sessionid);

        long outPutTime = System.currentTimeMillis();

        long runTime = outPutTime - startTime;

        if (runTime > squeryWarnLimit) {

            logMetacat.warn("MetaCatServlet.handleSQuery - Long running squery.  Total time: " + runTime +

                    " ms, squery: " + ((String[])params.get("query"))[0]);

        }

        logMetacat.debug("MetaCatServlet.handleSQuery - squery: " + ((String[])params.get("query"))[0] +

                " ran in " + runTime + " ms");

    }

    /**

     * Create the xml query, execute it and display the results.

     *

     * @param out the output stream to the client

     * @param params the Hashtable of parameters that should be included in the

     *            squery.

     * @param response the response object linked to the client

     */

    protected void handleQuery(PrintWriter out, Hashtable<String, String[]> params,

            HttpServletResponse response, String user, String[] groups,

            String sessionid) throws PropertyNotFoundException {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        long queryWarnLimit = Long.parseLong(PropertyService.getProperty("database.queryTimeWarnLimit"));

        //create the query and run it

        String xmlquery = DBQuery.createSQuery(params);

        String[] queryArray = new String[1];

        queryArray[0] = xmlquery;

        params.put("query", queryArray);

        long startTime = System.currentTimeMillis();

        DBQuery queryobj = new DBQuery();

        queryobj.findDocuments(response, out, params, user, groups, sessionid);

        long outPutTime = System.currentTimeMillis();

        long runTime = outPutTime -startTime;

        if (runTime > queryWarnLimit) {

            logMetacat.warn("MetaCatServlet.handleQuery - Long running squery.  Total time: " + runTime +

                    " ms, squery: " + ((String[])params.get("query"))[0]);

        }

        logMetacat.debug("MetaCatServlet.handleQuery - query: " + ((String[])params.get("query"))[0] +

                " ran in " + runTime + " ms");

    }

    // END OF SQUERY & QUERY SECTION

    //Export section

    /**

     * Handle the "export" request of data package from Metacat in zip format

     *

     * @param params the Hashtable of HTTP request parameters

     * @param response the HTTP response object linked to the client

     * @param user the username sent the request

     * @param groups the user's groupnames

     */

    protected void handleExportAction(Hashtable<String, String[]> params,

            HttpServletResponse response,

            String user, String[] groups, String passWord) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        // Output stream

        ServletOutputStream out = null;

        // Zip output stream

        ZipOutputStream zOut = null;

        DBQuery queryObj = null;

        String[] docs = new String[10];

        String docId = "";

        try {

            // read the params

            if (params.containsKey("docid")) {

                docs = params.get("docid");

            }

            // Create a DBuery to handle export

            queryObj = new DBQuery();

            // Get the docid

            docId = docs[0];

            // Make sure the client specify docid

            if (docId == null || docId.equals("")) {

                response.setContentType("text/xml"); //MIME type

                // Get a printwriter

                PrintWriter pw = response.getWriter();

                // Send back message

                pw.println("<?xml version=\"1.0\"?>");

                pw.println("<error>");

                pw.println("You didn't specify requested docid");

                pw.println("</error>");

                // Close printwriter

                pw.close();

                return;

            }

            // Get output stream

            out = response.getOutputStream();

            response.setContentType("application/zip"); //MIME type

            response.setHeader("Content-Disposition",

                    "attachment; filename="

                    + docId + ".zip"); // Set the name of the zip file

            zOut = new ZipOutputStream(out);

            zOut = queryObj

                    .getZippedPackage(docId, out, user, groups, passWord);

            zOut.finish(); //terminate the zip file

            zOut.close(); //close the zip stream

        } catch (Exception e) {

            try {

                response.setContentType("text/xml"); //MIME type

                // Send error message back

                if (out != null) {

                    PrintWriter pw = new PrintWriter(out);

                    pw.println("<?xml version=\"1.0\"?>");

                    pw.println("<error>");

                    pw.println(e.getMessage());

                    pw.println("</error>");

                    // Close printwriter

                    pw.close();

                    // Close output stream

                    out.close();

                }

                // Close zip output stream

                if (zOut != null) {

                    zOut.close();

                }

            } catch (IOException ioe) {

                logMetacat.error("MetaCatServlet.handleExportAction - Problem with the servlet output: "

                        + ioe.getMessage());

            logMetacat.error("MetaCatServlet.handleExportAction - General error: "

                    + e.getMessage());

            e.printStackTrace(System.out);

        }

    }

    /**

     * In eml2 document, the xml can have inline data and data was stripped off

     * and store in file system. This action can be used to read inline data

     * only

     *

     * @param params the Hashtable of HTTP request parameters

     * @param response the HTTP response object linked to the client

     * @param user the username sent the request

     * @param groups the user's groupnames

     */

    protected void handleReadInlineDataAction(Hashtable<String, String[]> params,

            HttpServletRequest request, HttpServletResponse response,

            String user, String passWord, String[] groups) {

        Logger logMetacat = Logger.getLogger(MetaCatServlet.class);

        String[] docs = new String[10];

        String inlineDataId = null;

        String docId = "";

        ServletOutputStream out = null;

        try {

            // read the params

            if (params.containsKey("inlinedataid")) {

                docs = params.get("inlinedataid");

            }

            // Get the docid

            inlineDataId = docs[0];

            // Make sure the client specify docid

            if (inlineDataId == null || inlineDataId.equals("")) {

                throw new Exception("You didn't specify requested inlinedataid"); }

            // check for permission

            docId =

                DocumentUtil.getDocIdWithoutRevFromInlineDataID(inlineDataId);

            PermissionController controller = new PermissionController(docId);

            // check top level read permission

            if (!controller.hasPermission(user, groups,

                    AccessControlInterface.READSTRING)) {

                throw new Exception("User " + user

                        + " doesn't have permission " + " to read document "

                        + docId);

            } else {

                //check data access level

                try {

                    Hashtable<String,String> unReadableInlineDataList =

                            PermissionController.getUnReadableInlineDataIdList(docId,

                            user, groups, false);

                    String inlineDataIdWithoutRev = DocumentUtil.getInlineDataIdWithoutRev(inlineDataId);

                    if (unReadableInlineDataList.containsValue(inlineDataIdWithoutRev)) {

                        throw new Exception("User " + user

                                + " doesn't have permission " + " to read inlinedata "

                                + inlineDataId);

                    }//if

                }//try

                catch (Exception e) {

                    throw e;

                }//catch

            }//else

            // Get output stream

            out = response.getOutputStream();

            // read the inline data from the file

            String inlinePath = PropertyService.getProperty("application.inlinedatafilepath");

            File lineData = new File(inlinePath, inlineDataId);

            FileInputStream input = new FileInputStream(lineData);

            byte[] buffer = new byte[4 * 1024];

            int bytes = input.read(buffer);

            while (bytes != -1) {

                out.write(buffer, 0, bytes);

                bytes = input.read(buffer);

            }

            out.close();

            EventLog.getInstance().log(request.getRemoteAddr(), user,

                    inlineDataId, "readinlinedata");

        } catch (Exception e) {

            try {

                PrintWriter pw = null;

                // Send error message back

                if (out != null) {

                    pw = new PrintWriter(out);

                } else {

                    pw = response.getWriter();

                }

                pw.println("<?xml version=\"1.0\"?>");

                pw.println("<error>");

                pw.println(e.getMessage());

                pw.println("</error>");

                // Close printwriter

                pw.close();

                // Close output stream if out is not null

                if (out != null) {

                    out.close();

                }

            } catch (IOException ioe) {

                logMetacat.error("MetaCatServlet.handleReadInlineDataAction - Problem with the servlet output: "

                        + ioe.getMessage());