Revision 7718
Added by Jing Tao over 11 years ago
src/edu/ucsb/nceas/metacat/index/MetacatSolrIndex.java | ||
---|---|---|
52 | 52 |
import org.apache.solr.core.CoreContainer; |
53 | 53 |
import org.apache.solr.core.SolrCore; |
54 | 54 |
import org.apache.solr.servlet.SolrRequestParsers; |
55 |
import org.dataone.service.exceptions.NotFound; |
|
56 |
import org.dataone.service.exceptions.NotImplemented; |
|
57 |
import org.dataone.service.exceptions.UnsupportedType; |
|
55 | 58 |
import org.dataone.service.types.v1.Subject; |
56 | 59 |
import org.dataone.service.util.Constants; |
57 | 60 |
import org.xml.sax.SAXException; |
... | ... | |
60 | 63 |
import edu.ucsb.nceas.metacat.common.SolrQueryResponseTransformer; |
61 | 64 |
import edu.ucsb.nceas.metacat.common.SolrQueryResponseWriterFactory; |
62 | 65 |
import edu.ucsb.nceas.metacat.common.SolrServerFactory; |
66 |
import edu.ucsb.nceas.metacat.common.query.SolrQueryService; |
|
67 |
import edu.ucsb.nceas.metacat.common.query.SolrQueryServiceController; |
|
63 | 68 |
import edu.ucsb.nceas.utilities.PropertyNotFoundException; |
64 | 69 |
|
65 | 70 |
|
... | ... | |
76 | 81 |
public static final String SOLR_CONFIG_FILE_NAME_PROPERTY_NAME = "solr.configFileName"; |
77 | 82 |
public static final String SOLR_COLLECTION_NAME_PROPERTY_NAME = "solr.collectionName"; |
78 | 83 |
public static final String SOLR_SERVER_CLASSNAME_PROPERTY_NAME = "solr.server.classname"; |
79 |
private static final String WT = "wt";//the property name to specify the return type |
|
84 |
|
|
80 | 85 |
|
81 |
private static List<String> supportedWriterTypes = null; |
|
82 |
private static final String VERSION = "3.4"; |
|
83 |
private static final String SOLRCONFDIR = "/conf"; |
|
84 |
private static final String SOLRSCHEMAFILEPATH = SOLRCONFDIR+"/schema.xml"; |
|
85 |
private static final String FILTERQUERY = "fq"; |
|
86 |
private static final String READPERMISSION = "readPermission"; |
|
87 |
private static final String RIGHTSHOLDER = "rightsHolder"; |
|
88 |
private static final String OPENPARENTHESE = "("; |
|
89 |
private static final String CLOSEPARENTHESE = ")"; |
|
90 |
private static final String COLON = ":"; |
|
91 |
private static final String OR = "OR"; |
|
92 |
//private static final String SOLRCONFIGPATH = "/conf/solrconfig.xml"; |
|
93 |
|
|
94 | 86 |
private static Log log = LogFactory.getLog(MetacatSolrIndex.class); |
95 |
private CoreContainer coreContainer = null; |
|
96 |
private SolrServer solrServer = null; |
|
97 |
//private String wt = null;//specify the return format. |
|
98 |
private String collectionName = null; |
|
99 |
private String solrBaseURL = null; |
|
100 |
private boolean isEmbeddedSolrServer = true; |
|
101 | 87 |
private static MetacatSolrIndex solrIndex = null; |
102 | 88 |
|
103 | 89 |
public static MetacatSolrIndex getInstance() throws Exception { |
... | ... | |
114 | 100 |
* @throws ParserConfigurationException |
115 | 101 |
*/ |
116 | 102 |
private MetacatSolrIndex() throws Exception { |
117 |
// these are handled directly by solr |
|
118 |
supportedWriterTypes = new ArrayList<String>(); |
|
119 |
supportedWriterTypes.add(SolrQueryResponseWriterFactory.CSV); |
|
120 |
supportedWriterTypes.add(SolrQueryResponseWriterFactory.JSON); |
|
121 |
supportedWriterTypes.add(SolrQueryResponseWriterFactory.PHP); |
|
122 |
supportedWriterTypes.add(SolrQueryResponseWriterFactory.PHPS); |
|
123 |
supportedWriterTypes.add(SolrQueryResponseWriterFactory.RUBY); |
|
124 |
supportedWriterTypes.add(SolrQueryResponseWriterFactory.VELOCITY); |
|
125 |
supportedWriterTypes.add(SolrQueryResponseWriterFactory.PYTHON); |
|
126 |
supportedWriterTypes.add(SolrQueryResponseWriterFactory.XML); |
|
127 |
|
|
128 |
generateSolrServer(); |
|
103 |
|
|
129 | 104 |
} |
130 | 105 |
|
131 | 106 |
|
132 |
/* |
|
133 |
* Generate the embedded solr server |
|
134 |
*/ |
|
135 |
private void generateSolrServer() throws Exception { |
|
136 |
solrServer = SolrServerFactory.createSolrServer(); |
|
137 |
if(solrServer instanceof EmbeddedSolrServer) { |
|
138 |
isEmbeddedSolrServer = true; |
|
139 |
coreContainer = SolrServerFactory.getCoreContainer(); |
|
140 |
collectionName = SolrServerFactory.getCollectionName(); |
|
141 |
} else { |
|
142 |
isEmbeddedSolrServer = false; |
|
143 |
CommonsHttpSolrServer httpServer = (CommonsHttpSolrServer)solrServer; |
|
144 |
solrBaseURL = httpServer.getBaseURL(); |
|
145 |
} |
|
146 |
|
|
147 |
|
|
148 |
} |
|
149 | 107 |
|
108 |
|
|
150 | 109 |
/** |
151 | 110 |
* Query the solr server |
152 | 111 |
* @param query the solr query |
... | ... | |
158 | 117 |
* @throws PropertyNotFoundException |
159 | 118 |
* @throws SAXException |
160 | 119 |
* @throws ParserConfigurationException |
120 |
* @throws UnsupportedType |
|
121 |
* @throws NotFound |
|
122 |
* @throws NotImplemented |
|
161 | 123 |
*/ |
162 |
public InputStream query(String query, Set<Subject>authorizedSubjects) throws SolrServerException, IOException, PropertyNotFoundException, SQLException, ClassNotFoundException, ParserConfigurationException, SAXException { |
|
124 |
public InputStream query(String query, Set<Subject>authorizedSubjects) throws SolrServerException, IOException, PropertyNotFoundException, SQLException, |
|
125 |
ClassNotFoundException, ParserConfigurationException, SAXException, NotImplemented, NotFound, UnsupportedType { |
|
163 | 126 |
if(authorizedSubjects == null || authorizedSubjects.isEmpty()) { |
164 | 127 |
throw new SolrServerException("MetacatSolrIndex.query - There is no any authorized subjects(even the public user) in this query session."); |
165 | 128 |
} |
166 |
if(isEmbeddedSolrServer) { |
|
167 |
return queryEmbedded(query, authorizedSubjects); |
|
168 |
} else { |
|
169 |
return queryHttp(query, authorizedSubjects); |
|
170 |
} |
|
171 |
} |
|
172 |
/** |
|
173 |
* Query a solr embedded server |
|
174 |
* @param query the solr query |
|
175 |
* @return the result as the InputStream |
|
176 |
* @throws SolrServerException |
|
177 |
* @throws ClassNotFoundException |
|
178 |
* @throws SQLException |
|
179 |
* @throws PropertyNotFoundException |
|
180 |
* @throws SAXException |
|
181 |
* @throws ParserConfigurationException |
|
182 |
*/ |
|
183 |
private InputStream queryEmbedded(String query, Set<Subject>subjects) throws SolrServerException, IOException, PropertyNotFoundException, SQLException, ClassNotFoundException, ParserConfigurationException, SAXException { |
|
184 | 129 |
InputStream inputStream = null; |
185 | 130 |
SolrParams solrParams = SolrRequestParsers.parseQueryString(query); |
186 |
solrParams = appendAccessFilterParams(solrParams, subjects); |
|
187 |
String wt = solrParams.get(WT); |
|
188 |
|
|
189 |
//reload the core before query. Only after reloading the core, the query result can reflect the change made in metacat-index module. |
|
190 |
coreContainer.reload(collectionName); |
|
191 |
|
|
192 |
SolrCore core = coreContainer.getCore(collectionName); |
|
193 |
SolrQueryResponseTransformer solrTransformer = new SolrQueryResponseTransformer(core); |
|
131 |
String wt = solrParams.get(SolrQueryService.WT); |
|
194 | 132 |
// handle normal and skin-based queries |
195 |
if (wt == null ||supportedWriterTypes.contains(wt)) {
|
|
196 |
// just handle as normal solr query
|
|
197 |
QueryResponse response = solrServer.query(solrParams); |
|
198 |
inputStream = solrTransformer.transformResults(solrParams, response, wt);
|
|
133 |
if (SolrQueryService.isSupportedWT(wt)) {
|
|
134 |
// just handle as normal solr query
|
|
135 |
|
|
136 |
inputStream = SolrQueryServiceController.getInstance().query(query, solrParams, authorizedSubjects);
|
|
199 | 137 |
} |
200 | 138 |
else { |
201 |
// assume it is a skin name
|
|
202 |
String qformat = wt;
|
|
203 |
|
|
204 |
// perform the solr query using wt=XML
|
|
205 |
wt = SolrQueryResponseWriterFactory.XML;
|
|
206 |
ModifiableSolrParams msp = new ModifiableSolrParams(solrParams);
|
|
207 |
msp.set(WT, wt);
|
|
208 |
QueryResponse response = solrServer.query(msp);
|
|
209 |
inputStream = solrTransformer.transformResults(msp, response, wt);
|
|
210 |
|
|
211 |
// apply the stylesheet (XML->HTML)
|
|
212 |
DBTransform transformer = new DBTransform();
|
|
213 |
String documentContent = IOUtils.toString(inputStream, "UTF-8");
|
|
214 |
String sourceType = "solr";
|
|
215 |
String targetType = "-//W3C//HTML//EN";
|
|
216 |
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
217 |
Writer writer = new OutputStreamWriter(baos , "UTF-8");
|
|
218 |
// TODO: include more params?
|
|
219 |
Hashtable<String, String[]> params = new Hashtable<String, String[]>();
|
|
220 |
params.put("qformat", new String[] {qformat});
|
|
221 |
transformer.transformXMLDocument(
|
|
222 |
documentContent ,
|
|
223 |
sourceType,
|
|
224 |
targetType ,
|
|
225 |
qformat,
|
|
226 |
writer,
|
|
227 |
params,
|
|
228 |
null //sessionid
|
|
229 |
);
|
|
230 |
|
|
231 |
// finally, get the HTML back
|
|
232 |
inputStream = new ByteArrayInputStream(baos.toByteArray());
|
|
139 |
// assume it is a skin name
|
|
140 |
String qformat = wt;
|
|
141 |
|
|
142 |
// perform the solr query using wt=XML
|
|
143 |
wt = SolrQueryResponseWriterFactory.XML;
|
|
144 |
ModifiableSolrParams msp = new ModifiableSolrParams(solrParams);
|
|
145 |
msp.set(SolrQueryService.WT, wt);
|
|
146 |
String newQuery = msp.toString();
|
|
147 |
inputStream = SolrQueryServiceController.getInstance().query(newQuery,msp, authorizedSubjects);
|
|
148 |
|
|
149 |
// apply the stylesheet (XML->HTML)
|
|
150 |
DBTransform transformer = new DBTransform();
|
|
151 |
String documentContent = IOUtils.toString(inputStream, "UTF-8");
|
|
152 |
String sourceType = "solr";
|
|
153 |
String targetType = "-//W3C//HTML//EN";
|
|
154 |
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
155 |
Writer writer = new OutputStreamWriter(baos , "UTF-8");
|
|
156 |
// TODO: include more params?
|
|
157 |
Hashtable<String, String[]> params = new Hashtable<String, String[]>();
|
|
158 |
params.put("qformat", new String[] {qformat});
|
|
159 |
transformer.transformXMLDocument(
|
|
160 |
documentContent ,
|
|
161 |
sourceType,
|
|
162 |
targetType ,
|
|
163 |
qformat,
|
|
164 |
writer,
|
|
165 |
params,
|
|
166 |
null //sessionid
|
|
167 |
);
|
|
168 |
|
|
169 |
// finally, get the HTML back
|
|
170 |
inputStream = new ByteArrayInputStream(baos.toByteArray());
|
|
233 | 171 |
} |
234 | 172 |
|
235 | 173 |
return inputStream; |
174 |
|
|
236 | 175 |
} |
237 |
|
|
238 |
/* |
|
239 |
* Query a http server. We directly build the url to send the http server. |
|
240 |
* The reason we don't use the method is: |
|
241 |
* QueryResponse response = solrServer.query(solrParams); |
|
242 |
* When we transform the QueryReponse object to the InputStream object, we need to have SolrCore object |
|
243 |
* which is not available for the SolrHttpServer. |
|
244 |
* |
|
245 |
*/ |
|
246 |
private InputStream queryHttp(String query, Set<Subject>subjects) throws IOException { |
|
247 |
StringBuffer accessFilter = generateAccessFilterParamsString(subjects); |
|
248 |
if(accessFilter != null && accessFilter.length() != 0) { |
|
249 |
query = solrBaseURL+"/select?"+query+"&"+FILTERQUERY+"="+accessFilter.toString(); |
|
250 |
} |
|
251 |
URL url = new URL(query); |
|
252 |
return url.openStream(); |
|
253 |
} |
|
254 |
|
|
255 |
|
|
256 |
/* |
|
257 |
* Append the access filter query to the params |
|
258 |
*/ |
|
259 |
private SolrParams appendAccessFilterParams(SolrParams solrParams, Set<Subject>subjects) { |
|
260 |
SolrParams append = null; |
|
261 |
if(solrParams != null) { |
|
262 |
StringBuffer query = generateAccessFilterParamsString(subjects); |
|
263 |
if(query != null && query.length() != 0) { |
|
264 |
log.info("=================== fq query is "+query.toString()); |
|
265 |
NamedList fq = new NamedList(); |
|
266 |
fq.add(FILTERQUERY, query.toString()); |
|
267 |
SolrParams fqParam = SolrParams.toSolrParams(fq); |
|
268 |
append = new AppendedSolrParams(solrParams, fqParam); |
|
269 |
} else { |
|
270 |
append = solrParams; |
|
271 |
} |
|
272 |
} |
|
273 |
return append; |
|
274 |
} |
|
275 |
|
|
276 |
private StringBuffer generateAccessFilterParamsString(Set<Subject>subjects) { |
|
277 |
StringBuffer query = new StringBuffer(); |
|
278 |
boolean first = true; |
|
279 |
if(subjects != null) { |
|
280 |
for(Subject subject : subjects) { |
|
281 |
if(subject != null) { |
|
282 |
String subjectName = subject.getValue(); |
|
283 |
if(subjectName != null && !subjectName.trim().equals("")) { |
|
284 |
if(first) { |
|
285 |
first = false; |
|
286 |
query.append(OPENPARENTHESE+READPERMISSION+COLON+"\""+subjectName+"\""+CLOSEPARENTHESE); |
|
287 |
if(!subjectName.equals(Constants.SUBJECT_PUBLIC) && !subjectName.equals(Constants.SUBJECT_AUTHENTICATED_USER)) { |
|
288 |
query.append(OR+OPENPARENTHESE+RIGHTSHOLDER+COLON+"\""+subjectName+"\""+CLOSEPARENTHESE); |
|
289 |
} |
|
290 |
} else { |
|
291 |
query.append(OR + OPENPARENTHESE+READPERMISSION+COLON+"\""+subjectName+"\""+CLOSEPARENTHESE); |
|
292 |
if(!subjectName.equals(Constants.SUBJECT_PUBLIC) && !subjectName.equals(Constants.SUBJECT_AUTHENTICATED_USER)) { |
|
293 |
query.append(OR + OPENPARENTHESE+RIGHTSHOLDER+COLON+"\""+subjectName+"\""+CLOSEPARENTHESE); |
|
294 |
} |
|
295 |
} |
|
296 |
} |
|
297 |
|
|
298 |
} |
|
299 |
|
|
300 |
} |
|
301 |
} |
|
302 |
return query; |
|
303 |
} |
|
304 |
|
|
305 |
|
|
306 |
|
|
307 |
|
|
176 |
|
|
308 | 177 |
} |
Also available in: Unified diff
Use the SolrQueryServiceController class to handle the query.