Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2000-2011 Regents of the University of California and the

 *              National Center for Ecological Analysis and Synthesis

 *

 *   '$Author: leinfelder $'

 *     '$Date: 2012-11-29 16:52:29 -0800 (Thu, 29 Nov 2012) $'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat.index;

import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStreamWriter;

import java.io.StringWriter;

import java.io.Writer;

import java.net.MalformedURLException;

import java.net.URL;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.Collection;

import java.util.Hashtable;

import java.util.List;

import java.util.Map;

import java.util.Set;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrServer;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;

import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.params.AppendedSolrParams;

import org.apache.solr.common.params.ModifiableSolrParams;

import org.apache.solr.common.params.SolrParams;

import org.apache.solr.common.util.NamedList;

import org.apache.solr.core.CoreContainer;

import org.apache.solr.core.SolrConfig;

import org.apache.solr.core.SolrCore;

import org.apache.solr.request.LocalSolrQueryRequest;

import org.apache.solr.response.CSVResponseWriter;

import org.apache.solr.response.JSONResponseWriter;

import org.apache.solr.response.PHPResponseWriter;

import org.apache.solr.response.PHPSerializedResponseWriter;

import org.apache.solr.response.PythonResponseWriter;

import org.apache.solr.response.QueryResponseWriter;

import org.apache.solr.response.RubyResponseWriter;

import org.apache.solr.response.SolrQueryResponse;

import org.apache.solr.response.VelocityResponseWriter;

import org.apache.solr.response.XMLResponseWriter;

import org.apache.solr.schema.IndexSchema;

import org.apache.solr.schema.SchemaField;

import org.apache.solr.servlet.SolrRequestParsers;

import org.dataone.configuration.Settings;

import org.dataone.service.types.v1.Subject;

import org.dataone.service.types.v1_1.QueryEngineDescription;

import org.dataone.service.types.v1_1.QueryField;

import org.dataone.service.util.Constants;

import org.xml.sax.InputSource;

import org.xml.sax.SAXException;

import edu.ucsb.nceas.metacat.DBTransform;

import edu.ucsb.nceas.metacat.MetaCatServlet;

import edu.ucsb.nceas.metacat.common.SolrServerFactory;

import edu.ucsb.nceas.utilities.PropertyNotFoundException;

/**

 * This class will query the solr server and return the result.

 * @author tao

 *

 */

public class MetacatSolrIndex {

    public static final String SOLRQUERY = "solr";

    public static final String SOLR_HOME_PROPERTY_NAME = "solr.homeDir";

    public static final String SOLR_CONFIG_FILE_NAME_PROPERTY_NAME = "solr.configFileName";

    public static final String SOLR_COLLECTION_NAME_PROPERTY_NAME = "solr.collectionName";

    public static final String SOLR_SERVER_CLASSNAME_PROPERTY_NAME = "solr.server.classname";

    private static final String WT = "wt";//the property name to specify the return type

    private static final String XML = "xml";

    private static final String JSON = "json";

    private static final String PYTHON = "python";

    private static final String RUBY = "ruby";

    private static final String PHP = "php";

    private static final String PHPS = "phps";

    private static final String VELOCITY = "velocity";

    private static final String CSV ="csv";

    private static List<String> supportedWriterTypes = null;

    private static final String VERSION = "3.4";

    private static final String SOLRCONFDIR = "/conf";

    private static final String SOLRSCHEMAFILEPATH = SOLRCONFDIR+"/schema.xml";

    private static final String FILTERQUERY = "fq";

    private static final String READPERMISSION = "readPermission";

    private static final String RIGHTSHOLDER = "rightsHolder";

    private static final String OPENPARENTHESE = "(";

    private static final String CLOSEPARENTHESE = ")";

    private static final String COLON = ":";

    private static final String OR = "OR";

    //private static final String SOLRCONFIGPATH = "/conf/solrconfig.xml";

    private static Log log = LogFactory.getLog(MetacatSolrIndex.class);

    private CoreContainer coreContainer = null;

    private SolrServer solrServer = null;

    //private String wt = null;//specify the return format.

    private String collectionName = null;

    private String solrBaseURL = null;

    private boolean isEmbeddedSolrServer = true;

    private static MetacatSolrIndex  solrIndex = null;

    public static MetacatSolrIndex getInstance() throws Exception {

        if (solrIndex == null) {

            solrIndex = new MetacatSolrIndex();

        }

        return solrIndex;

    }

    /**

     * Constructor

     * @throws SAXException 

     * @throws IOException 

     * @throws ParserConfigurationException 

     */

    private MetacatSolrIndex() throws Exception {

    	// these are handled directly by solr

    	supportedWriterTypes = new ArrayList<String>();

    	supportedWriterTypes.add(CSV);

    	supportedWriterTypes.add(JSON);

    	supportedWriterTypes.add(PHP);

    	supportedWriterTypes.add(PHPS);

    	supportedWriterTypes.add(RUBY);

    	supportedWriterTypes.add(VELOCITY);

    	supportedWriterTypes.add(PYTHON);

    	supportedWriterTypes.add(XML);

        generateSolrServer();

    }

    /*

     * Generate the embedded solr server

     */

    private void generateSolrServer() throws Exception {

        solrServer = SolrServerFactory.createSolrServer();

        if(solrServer instanceof EmbeddedSolrServer) {

            isEmbeddedSolrServer = true;

            coreContainer = SolrServerFactory.getCoreContainer();

            collectionName = SolrServerFactory.getCollectionName();

        } else {

            isEmbeddedSolrServer = false;

            CommonsHttpSolrServer httpServer = (CommonsHttpSolrServer)solrServer;

            solrBaseURL = httpServer.getBaseURL();

        }

    }

    /**

     * Query the solr server

     * @param query  the solr query

     * @param authorizedSubjects the authorized subjects in this query session

     * @return the result as the InputStream

     * @throws SolrServerException 

     * @throws ClassNotFoundException 

     * @throws SQLException 

     * @throws PropertyNotFoundException 

     */

    public InputStream query(String query, Set<Subject>authorizedSubjects) throws SolrServerException, IOException, PropertyNotFoundException, SQLException, ClassNotFoundException {

        if(authorizedSubjects == null || authorizedSubjects.isEmpty()) {

            throw new SolrServerException("MetacatSolrIndex.query - There is no any authorized subjects(even the public user) in this query session.");

        }

        if(isEmbeddedSolrServer) {

            return queryEmbedded(query, authorizedSubjects);

        } else {

            return queryHttp(query, authorizedSubjects);

        }

    }

    /**

     * Query a solr embedded server

     * @param query  the solr query

     * @return the result as the InputStream

     * @throws SolrServerException 

     * @throws ClassNotFoundException 

     * @throws SQLException 

     * @throws PropertyNotFoundException 

     */

    private InputStream queryEmbedded(String query, Set<Subject>subjects) throws SolrServerException, IOException, PropertyNotFoundException, SQLException, ClassNotFoundException {

        InputStream inputStream = null;

        SolrParams solrParams = SolrRequestParsers.parseQueryString(query);

        solrParams = appendAccessFilterParams(solrParams, subjects);

        String wt = solrParams.get(WT);

        // handle normal and skin-based queries

        if (wt == null ||supportedWriterTypes.contains(wt)) {

        	// just handle as normal solr query

	        QueryResponse response = solrServer.query(solrParams);

	        inputStream = transformResults(solrParams, response, wt);

        }

        else {

        	// assume it is a skin name

        	String qformat = wt;

        	// perform the solr query using wt=XML

        	wt = XML;

        	ModifiableSolrParams msp = new ModifiableSolrParams(solrParams);

        	msp.set(WT, wt);

        	QueryResponse response = solrServer.query(msp);

	        inputStream = transformResults(msp, response, wt);

	        // apply the stylesheet (XML->HTML)

	        DBTransform transformer = new DBTransform();

	        String documentContent = IOUtils.toString(inputStream, "UTF-8");

			String sourceType = "solr";

			String targetType = "-//W3C//HTML//EN";

			ByteArrayOutputStream baos = new ByteArrayOutputStream();

			Writer writer = new OutputStreamWriter(baos , "UTF-8");

			// TODO: include more params?

			Hashtable<String, String[]> params = new Hashtable<String, String[]>();

			params.put("qformat", new String[] {qformat});

			transformer.transformXMLDocument(

	        		documentContent , 

	        		sourceType, 

	        		targetType , 

	        		qformat, 

	        		writer, 

	        		params, 

	        		null //sessionid

	        		);

			// finally, get the HTML back

			inputStream = new ByteArrayInputStream(baos.toByteArray());	

        }

        return inputStream;

    }

    /*

     * Query a http server. We directly build the url to send the http server.

     * The reason we don't use the method is:

     * QueryResponse response = solrServer.query(solrParams);

     * When we transform the QueryReponse object to the InputStream object, we need to have SolrCore object 

     * which is not available for the SolrHttpServer.

     * 

     */

    private InputStream queryHttp(String query, Set<Subject>subjects) throws IOException {

        StringBuffer accessFilter = generateAccessFilterParamsString(subjects);

        if(accessFilter != null && accessFilter.length() != 0) {

            query = solrBaseURL+"/select?"+query+"&"+FILTERQUERY+"="+accessFilter.toString();

        }

        URL url = new URL(query);

        return url.openStream();

    }

    /*

     * Append the access filter query to the params

     */

    private SolrParams appendAccessFilterParams(SolrParams solrParams, Set<Subject>subjects) {

        SolrParams append = null;

        if(solrParams != null) {

            StringBuffer query = generateAccessFilterParamsString(subjects);      

            if(query != null && query.length() != 0) {

                log.info("=================== fq query is "+query.toString());

                NamedList fq = new NamedList();

                fq.add(FILTERQUERY, query.toString());

                SolrParams fqParam = SolrParams.toSolrParams(fq);

                append = new AppendedSolrParams(solrParams, fqParam);

            } else {

                append = solrParams;

            }

        }

        return append;

    }

    private StringBuffer generateAccessFilterParamsString(Set<Subject>subjects) {

        StringBuffer query = new StringBuffer();

        boolean first = true;

        if(subjects != null) {

            for(Subject subject : subjects) {

                if(subject != null) {

                    String subjectName = subject.getValue();

                    if(subjectName != null && !subjectName.trim().equals("")) {

                        if(first) {

                            first = false;

                            query.append(OPENPARENTHESE+READPERMISSION+COLON+"\""+subjectName+"\""+CLOSEPARENTHESE);

                            if(!subjectName.equals(Constants.SUBJECT_PUBLIC) && !subjectName.equals(Constants.SUBJECT_AUTHENTICATED_USER)) {

                                query.append(OR+OPENPARENTHESE+RIGHTSHOLDER+COLON+"\""+subjectName+"\""+CLOSEPARENTHESE);

                            }

                        } else {

                            query.append(OR + OPENPARENTHESE+READPERMISSION+COLON+"\""+subjectName+"\""+CLOSEPARENTHESE);

                            if(!subjectName.equals(Constants.SUBJECT_PUBLIC) && !subjectName.equals(Constants.SUBJECT_AUTHENTICATED_USER)) {

                                query.append(OR + OPENPARENTHESE+RIGHTSHOLDER+COLON+"\""+subjectName+"\""+CLOSEPARENTHESE);

                            }

                        }

                    }

                }

            }

        }

        return query;

    }

    /*

     * Transform the Queryresponse to the InputStream

     */

    private InputStream transformResults(SolrParams request, QueryResponse response, String wt) throws SolrServerException, IOException {

        //InputStream stream = null;

        QueryResponseWriter writer = generateResponseWriter(wt);

        Writer results = new StringWriter();

        SolrQueryResponse sResponse = new SolrQueryResponse();

        sResponse.setAllValues(response.getResponse());

        SolrCore core =coreContainer.getCore(collectionName);

        writer.write(results, new LocalSolrQueryRequest(core, request), sResponse);

        return new ByteArrayInputStream(results.toString().getBytes(MetaCatServlet.DEFAULT_ENCODING));

    }

    /* Here is the list of the handler class to handle different format.

     * <queryResponseWriter name="xml" default="true" class="solr.XMLResponseWriter" />

     * <queryResponseWriter name="json" class="solr.JSONResponseWriter"/>

     * <queryResponseWriter name="python" class="solr.PythonResponseWriter"/>

     * <queryResponseWriter name="ruby" class="solr.RubyResponseWriter"/>

     * <queryResponseWriter name="php" class="solr.PHPResponseWriter"/>

     * <queryResponseWriter name="phps" class="solr.PHPSerializedResponseWriter"/>

     * <queryResponseWriter name="velocity" class="solr.VelocityResponseWriter"/>

     * <queryResponseWriter name="csv" class="solr.CSVResponseWriter"/>

     */

    private QueryResponseWriter generateResponseWriter(String wt) throws SolrServerException {

        QueryResponseWriter writer = null;

        if(wt == null || wt.trim().equals("") || wt.equals(XML)) {

            writer = new XMLResponseWriter();

        } else if(wt.equals(JSON)) {

            writer = new JSONResponseWriter();

        } else if(wt.equals(PYTHON)) {

            writer = new PythonResponseWriter();

        } else if(wt.equals(RUBY)) {

            writer = new RubyResponseWriter();

        } else if(wt.equals(PHP)) {

            writer = new PHPResponseWriter();

        } else if(wt.equals(PHPS)) {

            writer = new PHPSerializedResponseWriter();

        } else if(wt.equals(VELOCITY)) {

            writer = new VelocityResponseWriter();

        } else if(wt.equals(CSV)) {

            writer = new CSVResponseWriter();

        } else {

            throw new SolrServerException("Metacat doesn't support this response format "+wt);

        }

        return writer;

    }

}