Revision 7728
Added by Jing Tao over 11 years ago
| metacat-common/src/main/java/edu/ucsb/nceas/metacat/common/query/EmbeddedSolrQueryService.java | ||
|---|---|---|
| 64 | 64 |
private CoreContainer coreContainer = null; |
| 65 | 65 |
private String collectionName = null; |
| 66 | 66 |
private SolrCore solrCore = null; |
| 67 |
private IndexSchema schema = null; |
|
| 67 |
|
|
| 68 | 68 |
|
| 69 | 69 |
/** |
| 70 | 70 |
* Constructor. |
| ... | ... | |
| 151 | 151 |
* @return |
| 152 | 152 |
*/ |
| 153 | 153 |
public List<String> getValidSchemaField() {
|
| 154 |
if (validSolrFieldNames != null && validSolrFieldNames.isEmpty()) {
|
|
| 155 |
//System.out.println("the valid file name is\n"+validSolrFieldNames);
|
|
| 156 |
return validSolrFieldNames; |
|
| 157 |
} else {
|
|
| 158 |
validSolrFieldNames = new ArrayList<String>(); |
|
| 159 |
if(fieldMap != null) {
|
|
| 160 |
Set<String> fieldNames = fieldMap.keySet(); |
|
| 161 |
for(String fieldName : fieldNames) {
|
|
| 162 |
SchemaField field = fieldMap.get(fieldName); |
|
| 163 |
//remove the field which is the target field of a CopyField. |
|
| 164 |
if(field != null && !schema.isCopyFieldTarget(field)) {
|
|
| 165 |
validSolrFieldNames.add(fieldName); |
|
| 166 |
} |
|
| 167 |
} |
|
| 168 |
} |
|
| 169 |
//System.out.println("the valid file name is\n"+validSolrFieldNames);
|
|
| 170 |
return validSolrFieldNames; |
|
| 171 |
} |
|
| 172 |
|
|
| 173 |
|
|
| 174 |
|
|
| 154 |
return super.getValidSchemaFields(); |
|
| 175 | 155 |
} |
| 176 | 156 |
|
| 177 | 157 |
/** |
| ... | ... | |
| 188 | 168 |
if (null != solrSpecVersion) {
|
| 189 | 169 |
try {
|
| 190 | 170 |
XML.escapeCharData(solrSpecVersion, tmp); |
| 171 |
solrSpecVersion = tmp.toString(); |
|
| 191 | 172 |
} catch (IOException e) {
|
| 192 | 173 |
e.printStackTrace(); |
| 193 | 174 |
} |
| 194 |
solrSpecVersion = tmp.toString(); |
|
| 195 | 175 |
} |
| 196 | 176 |
if (solrSpecVersion == null || solrSpecVersion.trim().equals("")) {
|
| 197 | 177 |
solrSpecVersion = UNKNOWN; |
| metacat-common/src/main/java/edu/ucsb/nceas/metacat/common/query/HttpSolrQueryService.java | ||
|---|---|---|
| 30 | 30 |
import java.io.InputStream; |
| 31 | 31 |
import java.net.MalformedURLException; |
| 32 | 32 |
import java.net.URL; |
| 33 |
import java.util.ArrayList; |
|
| 34 |
import java.util.HashMap; |
|
| 33 | 35 |
import java.util.List; |
| 34 | 36 |
import java.util.Map; |
| 35 | 37 |
import java.util.Set; |
| 38 |
import java.util.Vector; |
|
| 36 | 39 |
|
| 37 | 40 |
import javax.xml.parsers.DocumentBuilder; |
| 38 | 41 |
import javax.xml.parsers.DocumentBuilderFactory; |
| 39 | 42 |
import javax.xml.parsers.ParserConfigurationException; |
| 40 | 43 |
import javax.xml.xpath.XPathConstants; |
| 44 |
import javax.xml.xpath.XPathExpressionException; |
|
| 41 | 45 |
import javax.xml.xpath.XPathFactory; |
| 42 | 46 |
|
| 43 | 47 |
import org.apache.commons.codec.net.URLCodec; |
| ... | ... | |
| 47 | 51 |
import org.apache.solr.client.solrj.util.ClientUtils; |
| 48 | 52 |
|
| 49 | 53 |
import org.apache.solr.common.params.SolrParams; |
| 54 |
import org.apache.solr.core.SolrConfig; |
|
| 55 |
import org.apache.solr.schema.FieldType; |
|
| 56 |
import org.apache.solr.schema.IndexSchema; |
|
| 50 | 57 |
import org.apache.solr.schema.SchemaField; |
| 58 |
import org.apache.solr.schema.TextField; |
|
| 51 | 59 |
import org.dataone.configuration.Settings; |
| 52 | 60 |
import org.dataone.service.exceptions.NotFound; |
| 53 | 61 |
import org.dataone.service.exceptions.NotImplemented; |
| 54 | 62 |
import org.dataone.service.types.v1.Subject; |
| 63 |
import org.w3c.dom.Attr; |
|
| 55 | 64 |
import org.w3c.dom.Document; |
| 65 |
import org.w3c.dom.Element; |
|
| 56 | 66 |
import org.w3c.dom.Node; |
| 57 | 67 |
import org.w3c.dom.NodeList; |
| 68 |
import org.xml.sax.Attributes; |
|
| 69 |
import org.xml.sax.InputSource; |
|
| 58 | 70 |
import org.xml.sax.SAXException; |
| 59 | 71 |
|
| 60 | 72 |
|
| ... | ... | |
| 68 | 80 |
private static final String SELECTIONPHASE = "/select"; |
| 69 | 81 |
private static final String SOLR_SYSTEMINFO_URLAPPENDIX = "solr.systeminfo.urlappendix"; |
| 70 | 82 |
private static final String SOLR_SCHEMA_URLAPPENDIX = "sorl.schema.urlappendix"; |
| 83 |
private static final String SOLR_CONFIG_URLAPPENDIX = "solr.config.urlappendix"; |
|
| 71 | 84 |
private static final String SPEC_PATH = "//str[@name=\"solr-spec-version\"]"; |
| 85 |
private static final String FIELDS_PATH = "//fields//field"; |
|
| 86 |
private static final String COPY_FIELDS_PATH = "//copyField"; |
|
| 87 |
private static final String DEST = "dest"; |
|
| 88 |
private static final String TRUE = "true"; |
|
| 72 | 89 |
|
| 73 | 90 |
private String solrServerBaseURL = null; |
| 74 | 91 |
private CommonsHttpSolrServer httpSolrServer = null; |
| ... | ... | |
| 153 | 170 |
/** |
| 154 | 171 |
* Get the fields list of the index schema |
| 155 | 172 |
* @return |
| 173 |
* @throws SAXException |
|
| 174 |
* @throws IOException |
|
| 175 |
* @throws ParserConfigurationException |
|
| 176 |
* @throws MalformedURLException |
|
| 156 | 177 |
* @throws Exception |
| 157 | 178 |
*/ |
| 158 |
public Map<String, SchemaField> getIndexSchemaFields() throws Exception {
|
|
| 159 |
return null; |
|
| 179 |
public Map<String, SchemaField> getIndexSchemaFields() throws MalformedURLException, ParserConfigurationException, IOException, SAXException {
|
|
| 180 |
if(fieldMap == null || fieldMap.isEmpty()) {
|
|
| 181 |
getIndexSchemaFieldFromServer(); |
|
| 182 |
} |
|
| 183 |
return fieldMap; |
|
| 160 | 184 |
} |
| 161 | 185 |
|
| 162 | 186 |
|
| ... | ... | |
| 164 | 188 |
/** |
| 165 | 189 |
* Get the list of the valid field name (moved the fields names of the CopyFieldTarget). |
| 166 | 190 |
* @return |
| 191 |
* @throws SAXException |
|
| 192 |
* @throws IOException |
|
| 193 |
* @throws ParserConfigurationException |
|
| 194 |
* @throws MalformedURLException |
|
| 167 | 195 |
*/ |
| 168 |
public List<String> getValidSchemaField() {
|
|
| 169 |
return null; |
|
| 196 |
public List<String> getValidSchemaField() throws MalformedURLException, ParserConfigurationException, IOException, SAXException {
|
|
| 197 |
if(fieldMap == null || fieldMap.isEmpty()) {
|
|
| 198 |
getIndexSchemaFields(); |
|
| 199 |
} |
|
| 200 |
return super.getValidSchemaFields(); |
|
| 170 | 201 |
} |
| 171 | 202 |
|
| 203 |
|
|
| 204 |
/* |
|
| 205 |
* Get the fieldMap from the http server. |
|
| 206 |
* @throws MalformedURLException |
|
| 207 |
* @throws ParserConfigurationException |
|
| 208 |
* @throws IOException |
|
| 209 |
* @throws SAXException |
|
| 210 |
*/ |
|
| 211 |
private void getIndexSchemaFieldFromServer() throws MalformedURLException, ParserConfigurationException, IOException, SAXException {
|
|
| 212 |
SolrConfig config = new SolrConfig("dataone", new InputSource(getSolrConfig()));
|
|
| 213 |
schema = new IndexSchema(config, "dataone", new InputSource(getSchema())); |
|
| 214 |
fieldMap = schema.getFields(); |
|
| 215 |
} |
|
| 216 |
|
|
| 217 |
/* |
|
| 218 |
* Parse the schema.xml and get the validSolrFieldName list |
|
| 219 |
*/ |
|
| 220 |
/*private void parseSchema() throws MalformedURLException, ParserConfigurationException, SAXException, IOException, XPathExpressionException {
|
|
| 221 |
validSolrFieldNames = new ArrayList<String>(); |
|
| 222 |
Map<String, SchemaField> fieldMap = new HashMap<String, SchemaField>(); |
|
| 223 |
Document schema = transformInputStreamToDoc(getSchema()); |
|
| 224 |
Vector<String>copyFieldTargetNames = new Vector<String>(); |
|
| 225 |
NodeList copyFields = (NodeList) XPathFactory.newInstance().newXPath() |
|
| 226 |
.evaluate(COPY_FIELDS_PATH, schema, XPathConstants.NODESET); |
|
| 227 |
if(copyFields != null) {
|
|
| 228 |
for(int i=0; i<copyFields.getLength(); i++) {
|
|
| 229 |
Element copyField = (Element)copyFields.item(i); |
|
| 230 |
String target = copyField.getAttribute(DEST); |
|
| 231 |
if(target != null && !target.trim().equals("")) {
|
|
| 232 |
copyFieldTargetNames.add(target); |
|
| 233 |
} |
|
| 234 |
} |
|
| 235 |
} |
|
| 236 |
NodeList fields = (NodeList) XPathFactory.newInstance().newXPath() |
|
| 237 |
.evaluate(FIELDS_PATH, schema, XPathConstants.NODESET); |
|
| 238 |
if(fields!= null) {
|
|
| 239 |
for(int i=0; i<fields.getLength(); i++) {
|
|
| 240 |
Element fieldElement = (Element) fields.item(i); |
|
| 241 |
String name = fieldElement.getAttribute("name");
|
|
| 242 |
if(name != null && !name.trim().equals("")) {
|
|
| 243 |
if(!copyFieldTargetNames.contains(name)) {
|
|
| 244 |
validSolrFieldNames.add(name); |
|
| 245 |
} |
|
| 246 |
} |
|
| 247 |
} |
|
| 248 |
} |
|
| 249 |
}*/ |
|
| 250 |
|
|
| 251 |
|
|
| 252 |
/* |
|
| 253 |
* Get the SolrConfig InputStream. |
|
| 254 |
* @return |
|
| 255 |
* @throws MalformedURLException |
|
| 256 |
* @throws IOException |
|
| 257 |
*/ |
|
| 258 |
private InputStream getSolrConfig() throws MalformedURLException, IOException {
|
|
| 259 |
String solrConfigAppendix = Settings.getConfiguration().getString(SOLR_CONFIG_URLAPPENDIX); |
|
| 260 |
String configURL = solrServerBaseURL+solrConfigAppendix; |
|
| 261 |
return (new URL(configURL)).openStream(); |
|
| 262 |
} |
|
| 263 |
/* |
|
| 264 |
* Get the schema InputStream from the url which is specified in the metacat.properties and transform it to a Document. |
|
| 265 |
*/ |
|
| 266 |
private InputStream getSchema() throws MalformedURLException, IOException {
|
|
| 267 |
String schemaURLAppendix = Settings.getConfiguration().getString(SOLR_SCHEMA_URLAPPENDIX); |
|
| 268 |
String schemaURL = solrServerBaseURL+schemaURLAppendix; |
|
| 269 |
return (new URL(schemaURL)).openStream(); |
|
| 270 |
} |
|
| 271 |
|
|
| 172 | 272 |
/** |
| 173 | 273 |
* Get the version of the solr server. |
| 174 | 274 |
* @return |
| metacat-common/src/main/java/edu/ucsb/nceas/metacat/common/query/SolrQueryService.java | ||
|---|---|---|
| 37 | 37 |
import org.apache.solr.common.params.AppendedSolrParams; |
| 38 | 38 |
import org.apache.solr.common.params.SolrParams; |
| 39 | 39 |
import org.apache.solr.common.util.NamedList; |
| 40 |
import org.apache.solr.schema.IndexSchema; |
|
| 40 | 41 |
import org.apache.solr.schema.SchemaField; |
| 41 | 42 |
import org.dataone.service.types.v1.Subject; |
| 42 | 43 |
import org.dataone.service.util.Constants; |
| ... | ... | |
| 65 | 66 |
private static Log log = LogFactory.getLog(SolrQueryService.class); |
| 66 | 67 |
private static List<String> supportedWriterTypes = null; |
| 67 | 68 |
|
| 69 |
protected IndexSchema schema = null; |
|
| 68 | 70 |
protected Map<String, SchemaField> fieldMap = null; |
| 69 | 71 |
protected List<String> validSolrFieldNames = null; |
| 70 | 72 |
protected String solrSpecVersion = null; |
| ... | ... | |
| 120 | 122 |
* Get the list of the valid field name (moved the fields names of the CopyFieldTarget). |
| 121 | 123 |
* @return |
| 122 | 124 |
*/ |
| 123 |
public abstract List<String> getValidSchemaField(); |
|
| 125 |
protected List<String> getValidSchemaFields() {
|
|
| 126 |
if (validSolrFieldNames != null && !validSolrFieldNames.isEmpty()) {
|
|
| 127 |
//System.out.println("the valid file name is\n"+validSolrFieldNames);
|
|
| 128 |
return validSolrFieldNames; |
|
| 129 |
} else {
|
|
| 130 |
validSolrFieldNames = new ArrayList<String>(); |
|
| 131 |
if(fieldMap != null) {
|
|
| 132 |
Set<String> fieldNames = fieldMap.keySet(); |
|
| 133 |
for(String fieldName : fieldNames) {
|
|
| 134 |
SchemaField field = fieldMap.get(fieldName); |
|
| 135 |
//remove the field which is the target field of a CopyField. |
|
| 136 |
if(field != null && !schema.isCopyFieldTarget(field)) {
|
|
| 137 |
validSolrFieldNames.add(fieldName); |
|
| 138 |
} |
|
| 139 |
} |
|
| 140 |
} |
|
| 141 |
//System.out.println("the valid file name is\n"+validSolrFieldNames);
|
|
| 142 |
return validSolrFieldNames; |
|
| 143 |
} |
|
| 144 |
} |
|
| 124 | 145 |
|
| 125 | 146 |
/** |
| 126 | 147 |
* If the solr server supports the specified wt. |
| metacat-common/src/main/java/edu/ucsb/nceas/metacat/common/query/SolrQueryServiceController.java | ||
|---|---|---|
| 28 | 28 |
|
| 29 | 29 |
import java.io.IOException; |
| 30 | 30 |
import java.io.InputStream; |
| 31 |
import java.net.MalformedURLException; |
|
| 32 |
import java.util.List; |
|
| 33 |
import java.util.Map; |
|
| 31 | 34 |
import java.util.Set; |
| 32 | 35 |
|
| 33 | 36 |
import javax.xml.parsers.ParserConfigurationException; |
| ... | ... | |
| 38 | 41 |
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; |
| 39 | 42 |
import org.apache.solr.common.params.SolrParams; |
| 40 | 43 |
import org.apache.solr.core.CoreContainer; |
| 44 |
import org.apache.solr.schema.SchemaField; |
|
| 41 | 45 |
import org.dataone.service.exceptions.NotFound; |
| 42 | 46 |
import org.dataone.service.exceptions.NotImplemented; |
| 43 | 47 |
import org.dataone.service.exceptions.UnsupportedType; |
| ... | ... | |
| 128 | 132 |
return httpQueryService.getSolrServerVersion(); |
| 129 | 133 |
} |
| 130 | 134 |
} |
| 135 |
|
|
| 136 |
/** |
|
| 137 |
* Get the fields list of the index schema |
|
| 138 |
* @return |
|
| 139 |
* @throws SAXException |
|
| 140 |
* @throws IOException |
|
| 141 |
* @throws ParserConfigurationException |
|
| 142 |
* @throws MalformedURLException |
|
| 143 |
* @throws Exception |
|
| 144 |
*/ |
|
| 145 |
public Map<String, SchemaField> getIndexSchemaFields() throws MalformedURLException, ParserConfigurationException, IOException, SAXException {
|
|
| 146 |
if(isEmbeddedSolrServer) {
|
|
| 147 |
return embeddedQueryService.getIndexSchemaFields(); |
|
| 148 |
} else {
|
|
| 149 |
return httpQueryService.getIndexSchemaFields(); |
|
| 150 |
} |
|
| 151 |
} |
|
| 152 |
|
|
| 153 |
|
|
| 154 |
|
|
| 155 |
/** |
|
| 156 |
* Get the list of the valid field name (moved the fields names of the CopyFieldTarget). |
|
| 157 |
* @return |
|
| 158 |
* @throws SAXException |
|
| 159 |
* @throws IOException |
|
| 160 |
* @throws ParserConfigurationException |
|
| 161 |
* @throws MalformedURLException |
|
| 162 |
*/ |
|
| 163 |
public List<String> getValidSchemaFields() throws MalformedURLException, ParserConfigurationException, IOException, SAXException {
|
|
| 164 |
if(isEmbeddedSolrServer) {
|
|
| 165 |
return embeddedQueryService.getValidSchemaFields(); |
|
| 166 |
} else{
|
|
| 167 |
return httpQueryService.getValidSchemaField(); |
|
| 168 |
} |
|
| 169 |
|
|
| 170 |
} |
|
| 131 | 171 |
|
| 132 | 172 |
|
| 133 | 173 |
} |
Also available in: Unified diff
Add the getIndexFields and getValidIndexFields methods.