| 1 |
4872
|
leinfelder
|
<%@page
|
| 2 |
|
|
import="java.util.Iterator"%><%@page
|
| 3 |
|
|
import="java.util.ArrayList"%><%@page
|
| 4 |
|
|
import="edu.ucsb.nceas.utilities.OrderedMap"%><%@page
|
| 5 |
|
|
import="java.util.List"%><%@page
|
| 6 |
|
|
import="java.util.Enumeration"%><%@page
|
| 7 |
|
|
import="java.sql.SQLException"%><%@page
|
| 8 |
|
|
import="org.ecoinformatics.datamanager.transpose.DataTranspose"%><%@page
|
| 9 |
|
|
import="au.com.bytecode.opencsv.CSVWriter"%><%@page
|
| 10 |
|
|
import="java.io.OutputStreamWriter"%><%@page
|
| 11 |
|
|
import="java.io.Writer"%><%@page
|
| 12 |
|
|
import="java.sql.ResultSet"%><%@page
|
| 13 |
|
|
import="edu.ucsb.nceas.metacat.dataquery.DataQuery"%><%@page
|
| 14 |
|
|
import="java.io.IOException"%><%@page
|
| 15 |
|
|
import="edu.ucsb.nceas.utilities.PropertyNotFoundException"%><%@page
|
| 16 |
|
|
import="java.util.Hashtable"%><%@ page
|
| 17 |
|
|
language="java" %><%
|
| 18 |
|
|
/**
|
| 19 |
|
|
*
|
| 20 |
|
|
* '$RCSfile$'
|
| 21 |
|
|
* Copyright: 2008 Regents of the University of California and the
|
| 22 |
|
|
* National Center for Ecological Analysis and Synthesis
|
| 23 |
|
|
* '$Author: leinfelder $'
|
| 24 |
|
|
* '$Date: 2008-08-22 16:48:56 -0700 (Fri, 22 Aug 2008) $'
|
| 25 |
|
|
* '$Revision: 4305 $'
|
| 26 |
|
|
*
|
| 27 |
|
|
* This program is free software; you can redistribute it and/or modify
|
| 28 |
|
|
* it under the terms of the GNU General Public License as published by
|
| 29 |
|
|
* the Free Software Foundation; either version 2 of the License, or
|
| 30 |
|
|
* (at your option) any later version.
|
| 31 |
|
|
*
|
| 32 |
|
|
* This program is distributed in the hope that it will be useful,
|
| 33 |
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
| 34 |
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
| 35 |
|
|
* GNU General Public License for more details.
|
| 36 |
|
|
|
| 37 |
|
|
* You should have received a copy of the GNU General Public License
|
| 38 |
|
|
* along with this program; if not, write to the Free Software
|
| 39 |
|
|
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
| 40 |
|
|
*/
|
| 41 |
|
|
%><%
|
| 42 |
|
|
Hashtable params = getParams(request);
|
| 43 |
|
|
handleDataquery(params,response,request.getSession().getId());
|
| 44 |
|
|
%><%!
|
| 45 |
|
|
private Hashtable getParams(HttpServletRequest request) {
|
| 46 |
|
|
Hashtable params = new Hashtable();
|
| 47 |
|
|
Enumeration<String> paramlist = (Enumeration<String>)request.getParameterNames();
|
| 48 |
|
|
while (paramlist.hasMoreElements()) {
|
| 49 |
|
|
|
| 50 |
|
|
String name = paramlist.nextElement();
|
| 51 |
|
|
String[] value = request.getParameterValues(name);
|
| 52 |
|
|
params.put(name, value);
|
| 53 |
|
|
}
|
| 54 |
|
|
|
| 55 |
|
|
return params;
|
| 56 |
|
|
}
|
| 57 |
|
|
%><%!
|
| 58 |
4874
|
leinfelder
|
private List transpose(ResultSet rs, int idCol, int pivotCol, List pivotAttributes, boolean omitIdValues) throws SQLException {
|
| 59 |
4875
|
leinfelder
|
//map keyed by id column - data
|
| 60 |
4872
|
leinfelder
|
OrderedMap table = new OrderedMap();
|
| 61 |
4875
|
leinfelder
|
//track all the data columns
|
| 62 |
4872
|
leinfelder
|
OrderedMap widestRow = new OrderedMap();
|
| 63 |
4875
|
leinfelder
|
//map keyed by the pivot column - metadata
|
| 64 |
|
|
OrderedMap headerRows = new OrderedMap();
|
| 65 |
|
|
|
| 66 |
4883
|
leinfelder
|
//vocab columns
|
| 67 |
|
|
String vocabNameCol = "qmetadatavocabulary";
|
| 68 |
|
|
String vocabValueCol = "qmetadatavalue";
|
| 69 |
|
|
//maps for the vocab lists
|
| 70 |
|
|
OrderedMap vocabNames = new OrderedMap();
|
| 71 |
|
|
OrderedMap vocabValues = new OrderedMap();
|
| 72 |
|
|
//all vocab names/values
|
| 73 |
|
|
List allVocabNames = new ArrayList();
|
| 74 |
|
|
List allVocabValues = new ArrayList();
|
| 75 |
|
|
|
| 76 |
4872
|
leinfelder
|
int colCount = rs.getMetaData().getColumnCount();
|
| 77 |
|
|
String idColName = rs.getMetaData().getColumnName(idCol);
|
| 78 |
|
|
|
| 79 |
|
|
while (rs.next()) {
|
| 80 |
|
|
String id = rs.getString(idCol);
|
| 81 |
|
|
String pivotValue = rs.getString(pivotCol);
|
| 82 |
|
|
|
| 83 |
4875
|
leinfelder
|
//look up the data row we are working on
|
| 84 |
4872
|
leinfelder
|
OrderedMap row = (OrderedMap) table.get(id);
|
| 85 |
|
|
if (row == null) {
|
| 86 |
|
|
row = new OrderedMap();
|
| 87 |
|
|
}
|
| 88 |
4875
|
leinfelder
|
//look up the metadata row we are working on
|
| 89 |
|
|
OrderedMap metadataRow = (OrderedMap) table.get(pivotValue);
|
| 90 |
|
|
if (metadataRow == null) {
|
| 91 |
|
|
metadataRow = new OrderedMap();
|
| 92 |
|
|
}
|
| 93 |
4872
|
leinfelder
|
|
| 94 |
|
|
//get the values for this pivot
|
| 95 |
|
|
for (int i = 1; i <= colCount; i++) {
|
| 96 |
|
|
if (i != pivotCol) {
|
| 97 |
|
|
String colName = rs.getMetaData().getColumnName(i);
|
| 98 |
4875
|
leinfelder
|
String value = rs.getString(i);
|
| 99 |
4876
|
leinfelder
|
//clean up the value
|
| 100 |
4878
|
leinfelder
|
if (value != null) {
|
| 101 |
|
|
value = value.replaceAll("\n", " ");
|
| 102 |
4881
|
leinfelder
|
value = value.replaceAll("\\s+", " ");
|
| 103 |
4878
|
leinfelder
|
value = value.replaceAll("<html>", " ");
|
| 104 |
|
|
value = value.replaceAll("</html>", " ");
|
| 105 |
|
|
value = value.replaceAll("<head>", " ");
|
| 106 |
|
|
value = value.replaceAll("</head>", " ");
|
| 107 |
|
|
value = value.replaceAll("<body>", " ");
|
| 108 |
|
|
value = value.replaceAll("</body>", " ");
|
| 109 |
|
|
}
|
| 110 |
4874
|
leinfelder
|
//do we include this column in the pivot?
|
| 111 |
|
|
if (pivotAttributes.contains(colName)) {
|
| 112 |
|
|
//annotate the column name with the pivot column value if not the id column
|
| 113 |
|
|
if (i != idCol) {
|
| 114 |
|
|
colName = pivotValue + "_" + colName;
|
| 115 |
|
|
}
|
| 116 |
|
|
row.put(colName, value);
|
| 117 |
4872
|
leinfelder
|
}
|
| 118 |
4875
|
leinfelder
|
else {
|
| 119 |
4885
|
leinfelder
|
if (colName.startsWith(vocabNameCol) || colName.startsWith(vocabValueCol)) {
|
| 120 |
|
|
//don't add it to the normal metadata
|
| 121 |
|
|
}
|
| 122 |
|
|
else {
|
| 123 |
|
|
metadataRow.put(colName, value);
|
| 124 |
|
|
}
|
| 125 |
4875
|
leinfelder
|
}
|
| 126 |
4883
|
leinfelder
|
//names
|
| 127 |
|
|
if (colName.startsWith(vocabNameCol)) {
|
| 128 |
|
|
List list = (List) vocabNames.get(pivotValue);
|
| 129 |
|
|
if (list == null) {
|
| 130 |
|
|
list = new ArrayList();
|
| 131 |
|
|
}
|
| 132 |
|
|
list.add(value);
|
| 133 |
|
|
vocabNames.put(pivotValue, list);
|
| 134 |
|
|
allVocabNames.add(value);
|
| 135 |
|
|
}
|
| 136 |
|
|
//values
|
| 137 |
|
|
if (colName.startsWith(vocabValueCol)) {
|
| 138 |
|
|
List list = (List) vocabValues.get(pivotValue);
|
| 139 |
|
|
if (list == null) {
|
| 140 |
|
|
list = new ArrayList();
|
| 141 |
|
|
}
|
| 142 |
|
|
list.add(value);
|
| 143 |
|
|
vocabValues.put(pivotValue, list);
|
| 144 |
|
|
allVocabValues.add(value);
|
| 145 |
|
|
}
|
| 146 |
4872
|
leinfelder
|
}
|
| 147 |
|
|
}
|
| 148 |
4875
|
leinfelder
|
//track the data columns - the values are junk
|
| 149 |
4872
|
leinfelder
|
widestRow.putAll(row);
|
| 150 |
|
|
|
| 151 |
|
|
//put the row back (or maybe it's the first time)
|
| 152 |
4875
|
leinfelder
|
table.put(id, row);
|
| 153 |
|
|
|
| 154 |
|
|
//put the metadata header back
|
| 155 |
|
|
headerRows.put(pivotValue, metadataRow);
|
| 156 |
|
|
|
| 157 |
4872
|
leinfelder
|
}
|
| 158 |
|
|
|
| 159 |
4883
|
leinfelder
|
/** Construct the table structure for returning **/
|
| 160 |
|
|
|
| 161 |
4872
|
leinfelder
|
//now make it into a list
|
| 162 |
|
|
List retTable = new ArrayList();
|
| 163 |
|
|
|
| 164 |
4875
|
leinfelder
|
//map keyed by metadata labels
|
| 165 |
|
|
OrderedMap metadataHeaders = new OrderedMap();
|
| 166 |
|
|
|
| 167 |
|
|
//do the data header - drives the other columns - based on widest entry
|
| 168 |
4872
|
leinfelder
|
List header = new ArrayList(widestRow.keySet());
|
| 169 |
4875
|
leinfelder
|
|
| 170 |
|
|
//do the metadata header rows (basically rotate them around)
|
| 171 |
|
|
Iterator headerIter = header.iterator();
|
| 172 |
4886
|
leinfelder
|
String lastValue = "";
|
| 173 |
4875
|
leinfelder
|
while (headerIter.hasNext()) {
|
| 174 |
|
|
String column = (String) headerIter.next();
|
| 175 |
|
|
//get the pivotValue part of column name
|
| 176 |
|
|
String pivotValue = column;
|
| 177 |
|
|
try {
|
| 178 |
|
|
pivotValue = column.substring(0, column.indexOf("_"));
|
| 179 |
|
|
}
|
| 180 |
|
|
catch (Exception e) {}
|
| 181 |
|
|
//look up the row from the metadata - keyed by pivot value
|
| 182 |
|
|
OrderedMap metadataRow = (OrderedMap) headerRows.get(pivotValue);
|
| 183 |
|
|
if (metadataRow != null) {
|
| 184 |
|
|
//go through the values
|
| 185 |
|
|
Iterator metadataIter = metadataRow.keySet().iterator();
|
| 186 |
|
|
while (metadataIter.hasNext()) {
|
| 187 |
|
|
String key = (String) metadataIter.next();
|
| 188 |
|
|
String value = (String) metadataRow.get(key);
|
| 189 |
|
|
OrderedMap newMetadataRow = (OrderedMap) metadataHeaders.get(key);
|
| 190 |
|
|
if (newMetadataRow == null) {
|
| 191 |
|
|
newMetadataRow = new OrderedMap();
|
| 192 |
|
|
}
|
| 193 |
4886
|
leinfelder
|
//if it's the same as the last one, just use null value
|
| 194 |
|
|
if (lastValue.equals(pivotValue)) {
|
| 195 |
|
|
value = null;
|
| 196 |
|
|
}
|
| 197 |
4875
|
leinfelder
|
newMetadataRow.put(column, value);
|
| 198 |
|
|
metadataHeaders.put(key, newMetadataRow);
|
| 199 |
|
|
}
|
| 200 |
|
|
}
|
| 201 |
4886
|
leinfelder
|
|
| 202 |
|
|
lastValue = pivotValue;
|
| 203 |
|
|
|
| 204 |
4875
|
leinfelder
|
}
|
| 205 |
|
|
|
| 206 |
|
|
//make metadata rows as list/arrays on the reteurn table
|
| 207 |
|
|
Iterator metadataLabelIter = metadataHeaders.keySet().iterator();
|
| 208 |
|
|
while (metadataLabelIter.hasNext()) {
|
| 209 |
|
|
String label = (String) metadataLabelIter.next();
|
| 210 |
|
|
OrderedMap row = (OrderedMap) metadataHeaders.get(label);
|
| 211 |
|
|
List rowValues = new ArrayList(row.values());
|
| 212 |
|
|
rowValues.add(0, label);
|
| 213 |
|
|
retTable.add(rowValues.toArray(new String[0]));
|
| 214 |
|
|
}
|
| 215 |
|
|
|
| 216 |
4883
|
leinfelder
|
//create the special vocab matrix rows
|
| 217 |
|
|
List vocabTable = new ArrayList();
|
| 218 |
|
|
List uniqueVocabs = new ArrayList();
|
| 219 |
|
|
for (int i = 0; i < allVocabNames.size(); i++) {
|
| 220 |
|
|
List vocabRow = new ArrayList();
|
| 221 |
|
|
String vocabName = (String) allVocabNames.get(i);
|
| 222 |
|
|
String vocabValue = (String) allVocabValues.get(i);
|
| 223 |
|
|
String key = vocabName + "/" + vocabValue;
|
| 224 |
|
|
//check if we've processed this already, skip if so
|
| 225 |
|
|
if (uniqueVocabs.contains(key)) {
|
| 226 |
|
|
continue;
|
| 227 |
|
|
}
|
| 228 |
|
|
uniqueVocabs.add(key);
|
| 229 |
|
|
//TODO: expand the column count by one for _everything_
|
| 230 |
|
|
vocabRow.add(key);
|
| 231 |
|
|
//vocabRow.add(vocabName);
|
| 232 |
|
|
//vocabRow.add(vocabValue);
|
| 233 |
|
|
//go through the questions now, again
|
| 234 |
|
|
headerIter = header.iterator();
|
| 235 |
|
|
while (headerIter.hasNext()) {
|
| 236 |
|
|
String column = (String) headerIter.next();
|
| 237 |
|
|
//get the pivotValue part of column name if it exists
|
| 238 |
|
|
String pivotValue = null;
|
| 239 |
|
|
try {
|
| 240 |
|
|
pivotValue = column.substring(0, column.indexOf("_"));
|
| 241 |
|
|
}
|
| 242 |
|
|
catch (Exception e) {}
|
| 243 |
|
|
if (pivotValue == null) {
|
| 244 |
|
|
continue;
|
| 245 |
|
|
}
|
| 246 |
|
|
//check to see if this question has that keyword
|
| 247 |
|
|
List names = (List) vocabNames.get(pivotValue);
|
| 248 |
|
|
List values = (List) vocabValues.get(pivotValue);
|
| 249 |
|
|
if (names != null && names.indexOf(vocabName) > -1 && names.indexOf(vocabName) == values.indexOf(vocabValue) ) {
|
| 250 |
|
|
vocabRow.add("true");
|
| 251 |
|
|
}
|
| 252 |
|
|
else {
|
| 253 |
|
|
vocabRow.add("false");
|
| 254 |
|
|
}
|
| 255 |
|
|
}
|
| 256 |
|
|
//put the row on
|
| 257 |
|
|
vocabTable.add(vocabRow.toArray(new String[0]));
|
| 258 |
|
|
}
|
| 259 |
|
|
|
| 260 |
|
|
//put the vocab matrix on the table
|
| 261 |
|
|
retTable.addAll(vocabTable);
|
| 262 |
|
|
|
| 263 |
4875
|
leinfelder
|
//put the data header row on the table
|
| 264 |
4872
|
leinfelder
|
retTable.add(header.toArray(new String[0]));
|
| 265 |
|
|
|
| 266 |
4875
|
leinfelder
|
//now the value rows in the table
|
| 267 |
4872
|
leinfelder
|
Iterator rowIter = table.values().iterator();
|
| 268 |
|
|
int rowCount = 1;
|
| 269 |
|
|
while (rowIter.hasNext()) {
|
| 270 |
|
|
OrderedMap rowMap = (OrderedMap) rowIter.next();
|
| 271 |
|
|
List row = new ArrayList();
|
| 272 |
|
|
//iterate over the widest row's columns
|
| 273 |
|
|
Iterator columnIter = widestRow.keySet().iterator();
|
| 274 |
|
|
while (columnIter.hasNext()) {
|
| 275 |
|
|
Object key = columnIter.next();
|
| 276 |
|
|
Object value = rowMap.get(key);
|
| 277 |
|
|
//hide the value used for Ids - just increment row
|
| 278 |
|
|
if (key.equals(idColName) && omitIdValues) {
|
| 279 |
|
|
value = String.valueOf(rowCount);
|
| 280 |
|
|
}
|
| 281 |
|
|
row.add(value);
|
| 282 |
|
|
}
|
| 283 |
|
|
rowCount++;
|
| 284 |
|
|
retTable.add(row.toArray(new String[0]));
|
| 285 |
|
|
}
|
| 286 |
|
|
|
| 287 |
|
|
return retTable;
|
| 288 |
|
|
}
|
| 289 |
|
|
%><%!
|
| 290 |
|
|
private void handleDataquery(
|
| 291 |
|
|
Hashtable<String, String[]> params,
|
| 292 |
|
|
HttpServletResponse response,
|
| 293 |
|
|
String sessionId) throws PropertyNotFoundException, IOException {
|
| 294 |
|
|
|
| 295 |
|
|
DataQuery dq = null;
|
| 296 |
|
|
if (sessionId != null) {
|
| 297 |
|
|
dq = new DataQuery(sessionId);
|
| 298 |
|
|
}
|
| 299 |
|
|
else {
|
| 300 |
|
|
dq = new DataQuery();
|
| 301 |
|
|
}
|
| 302 |
|
|
|
| 303 |
|
|
String dataqueryXML = (params.get("dataquery"))[0];
|
| 304 |
|
|
|
| 305 |
|
|
ResultSet rs = null;
|
| 306 |
|
|
try {
|
| 307 |
|
|
rs = dq.executeQuery(dataqueryXML);
|
| 308 |
|
|
} catch (Exception e) {
|
| 309 |
|
|
//probably need to do something here
|
| 310 |
|
|
e.printStackTrace();
|
| 311 |
|
|
return;
|
| 312 |
|
|
}
|
| 313 |
|
|
|
| 314 |
|
|
//process the result set
|
| 315 |
|
|
String qformat = "csv";
|
| 316 |
|
|
String[] temp = params.get("qformat");
|
| 317 |
|
|
if (temp != null && temp.length > 0) {
|
| 318 |
|
|
qformat = temp[0];
|
| 319 |
|
|
}
|
| 320 |
|
|
String fileName = "query-results." + qformat;
|
| 321 |
|
|
|
| 322 |
|
|
boolean transpose = false;
|
| 323 |
|
|
temp = params.get("transpose");
|
| 324 |
|
|
if (temp != null && temp.length > 0) {
|
| 325 |
|
|
transpose = Boolean.parseBoolean(temp[0]);
|
| 326 |
|
|
}
|
| 327 |
|
|
int observation = 0;
|
| 328 |
|
|
temp = params.get("observation");
|
| 329 |
|
|
if (temp != null && temp.length > 0) {
|
| 330 |
|
|
observation = Integer.parseInt(temp[0]);
|
| 331 |
|
|
}
|
| 332 |
|
|
int pivot = 0;
|
| 333 |
|
|
temp = params.get("pivot");
|
| 334 |
|
|
if (temp != null && temp.length > 0) {
|
| 335 |
|
|
pivot = Integer.parseInt(temp[0]);
|
| 336 |
|
|
}
|
| 337 |
|
|
|
| 338 |
|
|
//get the results as csv file
|
| 339 |
|
|
if (qformat != null && qformat.equalsIgnoreCase("csv")) {
|
| 340 |
|
|
response.setContentType("text/csv");
|
| 341 |
|
|
//response.setContentType("application/csv");
|
| 342 |
|
|
response.setHeader("Content-Disposition", "attachment; filename=" + fileName);
|
| 343 |
|
|
|
| 344 |
|
|
Writer writer = new OutputStreamWriter(response.getOutputStream());
|
| 345 |
4876
|
leinfelder
|
//CSVWriter csv = new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);
|
| 346 |
|
|
CSVWriter csv = new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.DEFAULT_ESCAPE_CHARACTER);
|
| 347 |
4872
|
leinfelder
|
try {
|
| 348 |
|
|
if (transpose) {
|
| 349 |
4874
|
leinfelder
|
List pivotCols = new ArrayList();
|
| 350 |
|
|
pivotCols.add("studentid");
|
| 351 |
|
|
pivotCols.add("score");
|
| 352 |
|
|
pivotCols.add("response");
|
| 353 |
|
|
List transposedTable = transpose(rs, observation, pivot, pivotCols, true);
|
| 354 |
4872
|
leinfelder
|
csv.writeAll(transposedTable);
|
| 355 |
|
|
} else {
|
| 356 |
|
|
csv.writeAll(rs, true);
|
| 357 |
|
|
}
|
| 358 |
|
|
|
| 359 |
|
|
csv.flush();
|
| 360 |
|
|
response.flushBuffer();
|
| 361 |
|
|
|
| 362 |
|
|
rs.close();
|
| 363 |
|
|
|
| 364 |
|
|
} catch (SQLException e) {
|
| 365 |
|
|
e.printStackTrace();
|
| 366 |
|
|
}
|
| 367 |
|
|
|
| 368 |
|
|
return;
|
| 369 |
|
|
}
|
| 370 |
|
|
|
| 371 |
|
|
}
|
| 372 |
|
|
%>
|