Project

General

Profile

1 155 jones
/**
2 203 jones
 *  '$RCSfile$'
3 2043 sgarg
 *    Purpose: A Class that searches a relational DB for elements and
4 203 jones
 *             attributes that have free text matches a query string,
5 2043 sgarg
 *             or structured query matches to a path specified node in the
6
 *             XML hierarchy.  It returns a result set consisting of the
7 203 jones
 *             document ID for each document that satisfies the query
8
 *  Copyright: 2000 Regents of the University of California and the
9
 *             National Center for Ecological Analysis and Synthesis
10
 *    Authors: Matt Jones
11 155 jones
 *
12 203 jones
 *   '$Author$'
13
 *     '$Date$'
14
 * '$Revision$'
15 669 jones
 *
16
 * This program is free software; you can redistribute it and/or modify
17
 * it under the terms of the GNU General Public License as published by
18
 * the Free Software Foundation; either version 2 of the License, or
19
 * (at your option) any later version.
20
 *
21
 * This program is distributed in the hope that it will be useful,
22
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
23
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
24
 * GNU General Public License for more details.
25
 *
26
 * You should have received a copy of the GNU General Public License
27
 * along with this program; if not, write to the Free Software
28
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
29 155 jones
 */
30
31 607 bojilova
package edu.ucsb.nceas.metacat;
32 155 jones
33 5752 leinfelder
import java.io.BufferedWriter;
34
import java.io.File;
35
import java.io.FileInputStream;
36
import java.io.FileOutputStream;
37
import java.io.IOException;
38
import java.io.InputStream;
39
import java.io.InputStreamReader;
40
import java.io.OutputStreamWriter;
41
import java.io.Reader;
42
import java.io.StringReader;
43
import java.io.StringWriter;
44
import java.io.Writer;
45 2074 jones
import java.sql.PreparedStatement;
46
import java.sql.ResultSet;
47
import java.sql.SQLException;
48 5752 leinfelder
import java.util.Enumeration;
49
import java.util.Hashtable;
50
import java.util.Iterator;
51
import java.util.StringTokenizer;
52
import java.util.Vector;
53
import java.util.zip.ZipEntry;
54
import java.util.zip.ZipOutputStream;
55 2074 jones
56 940 tao
import javax.servlet.ServletOutputStream;
57 2087 tao
import javax.servlet.http.HttpServletResponse;
58 155 jones
59 2663 sgarg
import org.apache.log4j.Logger;
60 2087 tao
61 5090 daigle
import edu.ucsb.nceas.metacat.accesscontrol.AccessControlInterface;
62 5015 daigle
import edu.ucsb.nceas.metacat.database.DBConnection;
63
import edu.ucsb.nceas.metacat.database.DBConnectionPool;
64 5030 daigle
import edu.ucsb.nceas.metacat.properties.PropertyService;
65 4589 daigle
import edu.ucsb.nceas.metacat.util.AuthUtil;
66 5025 daigle
import edu.ucsb.nceas.metacat.util.DocumentUtil;
67 4698 daigle
import edu.ucsb.nceas.metacat.util.MetacatUtil;
68 2074 jones
import edu.ucsb.nceas.morpho.datapackage.Triple;
69
import edu.ucsb.nceas.morpho.datapackage.TripleCollection;
70 4080 daigle
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
71 2074 jones
72 2912 harris
73 2043 sgarg
/**
74 2075 jones
 * A Class that searches a relational DB for elements and attributes that have
75
 * free text matches a query string, or structured query matches to a path
76
 * specified node in the XML hierarchy. It returns a result set consisting of
77
 * the document ID for each document that satisfies the query
78 155 jones
 */
79 2075 jones
public class DBQuery
80
{
81 155 jones
82 2075 jones
    static final int ALL = 1;
83 2043 sgarg
84 2075 jones
    static final int WRITE = 2;
85 2043 sgarg
86 2075 jones
    static final int READ = 4;
87 5490 berkley
88
    private String qformat = "xml";
89 6035 leinfelder
90
    // are we combining the query with docid list and, if so, using INTERSECT or UNION?
91
    private String operator = null;
92 155 jones
93 2075 jones
    //private Connection conn = null;
94
    private String parserName = null;
95 706 bojilova
96 2663 sgarg
    private Logger logMetacat = Logger.getLogger(DBQuery.class);
97
98 2912 harris
    /** true if the metacat spatial option is installed **/
99
    private final boolean METACAT_SPATIAL = true;
100
101 3392 tao
    /** useful if you just want to grab a list of docids. Since the docids can be very long,
102
         it is a vector of vector  **/
103 3047 perry
    Vector docidOverride = new Vector();
104 3340 tao
105
    // a hash table serves as query reuslt cache. Key of hashtable
106 3342 tao
    // is a query string and value is result xml string
107 3340 tao
    private static Hashtable queryResultCache = new Hashtable();
108
109
    // Capacity of the query result cache
110 4080 daigle
    private static final int QUERYRESULTCACHESIZE;
111
    static {
112
    	int qryRsltCacheSize = 0;
113
    	try {
114 4212 daigle
    		qryRsltCacheSize = Integer.parseInt(PropertyService.getProperty("database.queryresultCacheSize"));
115 4080 daigle
    	} catch (PropertyNotFoundException pnfe) {
116
    		System.err.println("Could not get QUERYRESULTCACHESIZE property in static block: "
117
					+ pnfe.getMessage());
118
    	}
119
    	QUERYRESULTCACHESIZE = qryRsltCacheSize;
120
    }
121
122 3047 perry
123 3368 tao
    // Size of page for non paged query
124
    private static final int NONPAGESIZE = 99999999;
125 2075 jones
    /**
126
     * the main routine used to test the DBQuery utility.
127
     * <p>
128
     * Usage: java DBQuery <xmlfile>
129 5752 leinfelder
     * NOTE: encoding should be provided for best results
130 2075 jones
     * @param xmlfile the filename of the xml file containing the query
131
     */
132
    static public void main(String[] args)
133
    {
134 706 bojilova
135 2075 jones
        if (args.length < 1) {
136
            System.err.println("Wrong number of arguments!!!");
137
            System.err.println("USAGE: java DBQuery [-t] [-index] <xmlfile>");
138
            return;
139
        } else {
140
            try {
141 706 bojilova
142 2075 jones
                int i = 0;
143
                boolean showRuntime = false;
144
                boolean useXMLIndex = false;
145
                if (args[i].equals("-t")) {
146
                    showRuntime = true;
147
                    i++;
148
                }
149
                if (args[i].equals("-index")) {
150
                    useXMLIndex = true;
151
                    i++;
152
                }
153
                String xmlfile = args[i];
154 706 bojilova
155 2075 jones
                // Time the request if asked for
156
                double startTime = System.currentTimeMillis();
157 2043 sgarg
158 2075 jones
                // Open a connection to the database
159
                //Connection dbconn = util.openDBConnection();
160 2043 sgarg
161 2075 jones
                double connTime = System.currentTimeMillis();
162 2043 sgarg
163 2075 jones
                // Execute the query
164 2752 jones
                DBQuery queryobj = new DBQuery();
165 5752 leinfelder
                Reader xml = new InputStreamReader(new FileInputStream(new File(xmlfile)));
166 2075 jones
                Hashtable nodelist = null;
167 2087 tao
                //nodelist = queryobj.findDocuments(xml, null, null, useXMLIndex);
168 2043 sgarg
169 2075 jones
                // Print the reulting document listing
170
                StringBuffer result = new StringBuffer();
171
                String document = null;
172
                String docid = null;
173
                result.append("<?xml version=\"1.0\"?>\n");
174
                result.append("<resultset>\n");
175 2043 sgarg
176 2075 jones
                if (!showRuntime) {
177
                    Enumeration doclist = nodelist.keys();
178
                    while (doclist.hasMoreElements()) {
179
                        docid = (String) doclist.nextElement();
180
                        document = (String) nodelist.get(docid);
181
                        result.append("  <document>\n    " + document
182
                                + "\n  </document>\n");
183
                    }
184 706 bojilova
185 2075 jones
                    result.append("</resultset>\n");
186
                }
187
                // Time the request if asked for
188
                double stopTime = System.currentTimeMillis();
189
                double dbOpenTime = (connTime - startTime) / 1000;
190
                double readTime = (stopTime - connTime) / 1000;
191
                double executionTime = (stopTime - startTime) / 1000;
192
                if (showRuntime) {
193
                    System.out.print("  " + executionTime);
194
                    System.out.print("  " + dbOpenTime);
195
                    System.out.print("  " + readTime);
196
                    System.out.print("  " + nodelist.size());
197
                    System.out.println();
198
                }
199
                //System.out.println(result);
200
                //write into a file "result.txt"
201
                if (!showRuntime) {
202
                    File f = new File("./result.txt");
203 5752 leinfelder
                    Writer fw = new OutputStreamWriter(new FileOutputStream(f));
204 2075 jones
                    BufferedWriter out = new BufferedWriter(fw);
205
                    out.write(result.toString());
206
                    out.flush();
207
                    out.close();
208
                    fw.close();
209
                }
210 2043 sgarg
211 2075 jones
            } catch (Exception e) {
212
                System.err.println("Error in DBQuery.main");
213
                System.err.println(e.getMessage());
214
                e.printStackTrace(System.err);
215
            }
216
        }
217
    }
218 2043 sgarg
219 2075 jones
    /**
220
     * construct an instance of the DBQuery class
221 2087 tao
     *
222 2075 jones
     * <p>
223
     * Generally, one would call the findDocuments() routine after creating an
224
     * instance to specify the search query
225
     * </p>
226 2087 tao
     *
227
228 2075 jones
     * @param parserName the fully qualified name of a Java class implementing
229
     *            the org.xml.sax.XMLReader interface
230
     */
231 4080 daigle
    public DBQuery() throws PropertyNotFoundException
232 2075 jones
    {
233 4213 daigle
        String parserName = PropertyService.getProperty("xml.saxparser");
234 2752 jones
        this.parserName = parserName;
235 2075 jones
    }
236 2043 sgarg
237 3047 perry
    /**
238
     *
239
     * Construct an instance of DBQuery Class
240
     * BUT accept a docid Vector that will supersede
241
     * the query.printSQL() method
242
     *
243
     * If a docid Vector is passed in,
244
     * the docids will be used to create a simple IN query
245
     * without the multiple subselects of the printSQL() method
246
     *
247
     * Using this constructor, we just check for
248
     * a docidOverride Vector in the findResultDoclist() method
249
     *
250
     * @param docids List of docids to display in the resultset
251
     */
252 4080 daigle
    public DBQuery(Vector docids) throws PropertyNotFoundException
253 3047 perry
    {
254 3392 tao
    	// since the query will be too long to be handled, so we divided the
255
    	// docids vector into couple vectors.
256 4212 daigle
    	int size = (new Integer(PropertyService.getProperty("database.appResultsetSize"))).intValue();
257 5165 daigle
    	logMetacat.info("DBQuery.DBQuery - The size of select doicds is "+docids.size());
258
    	logMetacat.info("DBQuery.DBQuery - The application result size in metacat.properties is "+size);
259 3392 tao
    	Vector subset = new Vector();
260
    	if (docids != null && docids.size() > size)
261
    	{
262
    		int index = 0;
263
    		for (int i=0; i< docids.size(); i++)
264
    		{
265
266
    			if (index < size)
267
    			{
268
    				subset.add(docids.elementAt(i));
269
    				index ++;
270
    			}
271
    			else
272
    			{
273
    				docidOverride.add(subset);
274
    				subset = new Vector();
275
    				subset.add(docids.elementAt(i));
276
    			    index = 1;
277
    			}
278
    		}
279
    		if (!subset.isEmpty())
280
    		{
281
    			docidOverride.add(subset);
282
    		}
283
284
    	}
285
    	else
286
    	{
287
    		this.docidOverride.add(docids);
288
    	}
289
290 4213 daigle
        String parserName = PropertyService.getProperty("xml.saxparser");
291 3047 perry
        this.parserName = parserName;
292
    }
293 2087 tao
294
  /**
295
   * Method put the search result set into out printerwriter
296
   * @param resoponse the return response
297
   * @param out the output printer
298
   * @param params the paratermer hashtable
299
   * @param user the user name (it maybe different to the one in param)
300
   * @param groups the group array
301
   * @param sessionid  the sessionid
302
   */
303
  public void findDocuments(HttpServletResponse response,
304 5752 leinfelder
                                       Writer out, Hashtable params,
305 2087 tao
                                       String user, String[] groups,
306 4080 daigle
                                       String sessionid) throws PropertyNotFoundException
307 2087 tao
  {
308 4173 daigle
    boolean useXMLIndex = (new Boolean(PropertyService.getProperty("database.usexmlindex")))
309 2087 tao
               .booleanValue();
310
    findDocuments(response, out, params, user, groups, sessionid, useXMLIndex);
311
312
  }
313
314
315 2075 jones
    /**
316 2087 tao
     * Method put the search result set into out printerwriter
317
     * @param resoponse the return response
318
     * @param out the output printer
319
     * @param params the paratermer hashtable
320
     * @param user the user name (it maybe different to the one in param)
321
     * @param groups the group array
322
     * @param sessionid  the sessionid
323 2075 jones
     */
324 2087 tao
    public void findDocuments(HttpServletResponse response,
325 5752 leinfelder
                                         Writer out, Hashtable params,
326 2087 tao
                                         String user, String[] groups,
327
                                         String sessionid, boolean useXMLIndex)
328 2075 jones
    {
329 3211 berkley
      int pagesize = 0;
330
      int pagestart = 0;
331 5165 daigle
      long transferWarnLimit = 0;
332 3211 berkley
333
      if(params.containsKey("pagesize") && params.containsKey("pagestart"))
334
      {
335
        String pagesizeStr = ((String[])params.get("pagesize"))[0];
336
        String pagestartStr = ((String[])params.get("pagestart"))[0];
337
        if(pagesizeStr != null && pagestartStr != null)
338
        {
339
          pagesize = (new Integer(pagesizeStr)).intValue();
340
          pagestart = (new Integer(pagestartStr)).intValue();
341
        }
342
      }
343
344 3780 daigle
      String xmlquery = null;
345
      String qformat = null;
346 2087 tao
      // get query and qformat
347 3780 daigle
      try {
348
    	xmlquery = ((String[])params.get("query"))[0];
349 2168 tao
350 5165 daigle
        logMetacat.info("DBQuery.findDocuments - SESSIONID: " + sessionid);
351
        logMetacat.info("DBQuery.findDocuments - xmlquery: " + xmlquery);
352 3780 daigle
        qformat = ((String[])params.get("qformat"))[0];
353 5165 daigle
        logMetacat.info("DBQuery.findDocuments - qformat: " + qformat);
354 3780 daigle
      }
355
      catch (Exception ee)
356
      {
357 5165 daigle
        logMetacat.error("DBQuery.findDocuments - Couldn't retrieve xmlquery or qformat value from "
358 3780 daigle
                  +"params hashtable in DBQuery.findDocuments: "
359
                  + ee.getMessage());
360
      }
361 2168 tao
      // Get the XML query and covert it into a SQL statment
362
      QuerySpecification qspec = null;
363
      if ( xmlquery != null)
364
      {
365
         xmlquery = transformQuery(xmlquery);
366
         try
367
         {
368
           qspec = new QuerySpecification(xmlquery,
369
                                          parserName,
370 4212 daigle
                                          PropertyService.getProperty("document.accNumSeparator"));
371 2168 tao
         }
372
         catch (Exception ee)
373
         {
374 5165 daigle
           logMetacat.error("DBQuery.findDocuments - error generating QuerySpecification object: "
375 2663 sgarg
                                    + ee.getMessage());
376 2168 tao
         }
377
      }
378 2087 tao
379 2168 tao
380
381 5025 daigle
      if (qformat != null && qformat.equals(MetacatUtil.XMLFORMAT))
382 2087 tao
      {
383
        //xml format
384 5491 berkley
        if(response != null)
385
        {
386
            response.setContentType("text/xml");
387
        }
388 5490 berkley
        createResultDocument(xmlquery, qspec, out, user, groups, useXMLIndex,
389
          pagesize, pagestart, sessionid, qformat);
390 2087 tao
      }//if
391
      else
392
      {
393
        //knb format, in this case we will get whole result and sent it out
394 3257 berkley
        response.setContentType("text/html");
395 5752 leinfelder
        Writer nonout = null;
396 2168 tao
        StringBuffer xml = createResultDocument(xmlquery, qspec, nonout, user,
397 3211 berkley
                                                groups, useXMLIndex, pagesize,
398 5490 berkley
                                                pagestart, sessionid, qformat);
399 2658 sgarg
400 2087 tao
        //transfer the xml to html
401
        try
402
        {
403 5165 daigle
         long startHTMLTransform = System.currentTimeMillis();
404 2087 tao
         DBTransform trans = new DBTransform();
405
         response.setContentType("text/html");
406 2787 sgarg
407 3219 berkley
         // if the user is a moderator, then pass a param to the
408 2787 sgarg
         // xsl specifying the fact
409 4589 daigle
         if(AuthUtil.isModerator(user, groups)){
410 2787 sgarg
        	 params.put("isModerator", new String[] {"true"});
411
         }
412
413 2087 tao
         trans.transformXMLDocument(xml.toString(), "-//NCEAS//resultset//EN",
414
                                 "-//W3C//HTML//EN", qformat, out, params,
415
                                 sessionid);
416 5165 daigle
         long transformRunTime = System.currentTimeMillis() - startHTMLTransform;
417
418
         transferWarnLimit = Long.parseLong(PropertyService.getProperty("dbquery.transformTimeWarnLimit"));
419
420
         if (transformRunTime > transferWarnLimit) {
421
         	logMetacat.warn("DBQuery.findDocuments - The time to transfrom resultset from xml to html format is "
422
                  		                             + transformRunTime);
423
         }
424 4698 daigle
          MetacatUtil.writeDebugToFile("---------------------------------------------------------------------------------------------------------------Transfrom xml to html  "
425 5165 daigle
                             + transformRunTime);
426
          MetacatUtil.writeDebugToDelimiteredFile(" " + transformRunTime, false);
427 2087 tao
        }
428
        catch(Exception e)
429
        {
430 5165 daigle
         logMetacat.error("DBQuery.findDocuments - Error in MetaCatServlet.transformResultset:"
431 2663 sgarg
                                +e.getMessage());
432 2087 tao
         }
433
434
      }//else
435
436 3219 berkley
  }
437 5490 berkley
438
439 3220 tao
440
  /**
441
   * Transforms a hashtable of documents to an xml or html result and sent
442
   * the content to outputstream. Keep going untill hastable is empty. stop it.
443
   * add the QuerySpecification as parameter is for ecogrid. But it is duplicate
444
   * to xmlquery String
445
   * @param xmlquery
446
   * @param qspec
447
   * @param out
448
   * @param user
449
   * @param groups
450
   * @param useXMLIndex
451
   * @param sessionid
452
   * @return
453
   */
454
    public StringBuffer createResultDocument(String xmlquery,
455
                                              QuerySpecification qspec,
456 5752 leinfelder
                                              Writer out,
457 3220 tao
                                              String user, String[] groups,
458
                                              boolean useXMLIndex)
459
    {
460 5490 berkley
    	return createResultDocument(xmlquery,qspec,out, user,groups, useXMLIndex, 0, 0,"", qformat);
461 3220 tao
    }
462 2043 sgarg
463 2087 tao
  /*
464
   * Transforms a hashtable of documents to an xml or html result and sent
465 2168 tao
   * the content to outputstream. Keep going untill hastable is empty. stop it.
466
   * add the QuerySpecification as parameter is for ecogrid. But it is duplicate
467
   * to xmlquery String
468 2087 tao
   */
469 2168 tao
  public StringBuffer createResultDocument(String xmlquery,
470
                                            QuerySpecification qspec,
471 5752 leinfelder
                                            Writer out,
472 2087 tao
                                            String user, String[] groups,
473 3211 berkley
                                            boolean useXMLIndex, int pagesize,
474 5490 berkley
                                            int pagestart, String sessionid,
475
                                            String qformat)
476 2087 tao
  {
477
    DBConnection dbconn = null;
478
    int serialNumber = -1;
479
    StringBuffer resultset = new StringBuffer();
480 3219 berkley
481
    //try to get the cached version first
482 4080 daigle
    // Hashtable sessionHash = MetaCatServlet.getSessionHash();
483
    // HttpSession sess = (HttpSession)sessionHash.get(sessionid);
484 3219 berkley
485 3220 tao
486 2087 tao
    resultset.append("<?xml version=\"1.0\"?>\n");
487
    resultset.append("<resultset>\n");
488 3257 berkley
    resultset.append("  <pagestart>" + pagestart + "</pagestart>\n");
489
    resultset.append("  <pagesize>" + pagesize + "</pagesize>\n");
490
    resultset.append("  <nextpage>" + (pagestart + 1) + "</nextpage>\n");
491
    resultset.append("  <previouspage>" + (pagestart - 1) + "</previouspage>\n");
492
493 2087 tao
    resultset.append("  <query>" + xmlquery + "</query>");
494 3219 berkley
    //send out a new query
495 2087 tao
    if (out != null)
496 2075 jones
    {
497 5752 leinfelder
    	try {
498
    	  out.write(resultset.toString());
499
		} catch (IOException e) {
500
			logMetacat.error(e.getMessage(), e);
501
		}
502 2075 jones
    }
503 2168 tao
    if (qspec != null)
504 2087 tao
    {
505 2168 tao
      try
506
      {
507 2043 sgarg
508 2168 tao
        //checkout the dbconnection
509
        dbconn = DBConnectionPool.getDBConnection("DBQuery.findDocuments");
510
        serialNumber = dbconn.getCheckOutSerialNumber();
511 2087 tao
512 2168 tao
        //print out the search result
513
        // search the doc list
514 3392 tao
        Vector givenDocids = new Vector();
515
        StringBuffer resultContent = new StringBuffer();
516
        if (docidOverride == null || docidOverride.size() == 0)
517
        {
518 5165 daigle
        	logMetacat.debug("DBQuery.createResultDocument - Not in map query");
519 3392 tao
        	resultContent = findResultDoclist(qspec, out, user, groups,
520
                    dbconn, useXMLIndex, pagesize, pagestart,
521 5490 berkley
                    sessionid, givenDocids, qformat);
522 3392 tao
        }
523
        else
524
        {
525 5165 daigle
        	logMetacat.debug("DBQuery.createResultDocument - In map query");
526 3392 tao
        	// since docid can be too long to be handled. We divide it into several parts
527
        	for (int i= 0; i<docidOverride.size(); i++)
528
        	{
529 5165 daigle
        	   logMetacat.debug("DBQuery.createResultDocument - in loop===== "+i);
530 3392 tao
        		givenDocids = (Vector)docidOverride.elementAt(i);
531
        		StringBuffer subset = findResultDoclist(qspec, out, user, groups,
532
                        dbconn, useXMLIndex, pagesize, pagestart,
533 5490 berkley
                        sessionid, givenDocids, qformat);
534 3392 tao
        		resultContent.append(subset);
535
        	}
536
        }
537
538 3342 tao
        resultset.append(resultContent);
539 2168 tao
      } //try
540
      catch (IOException ioe)
541
      {
542 5165 daigle
        logMetacat.error("DBQuery.createResultDocument - IO error: " + ioe.getMessage());
543 2168 tao
      }
544
      catch (SQLException e)
545
      {
546 5165 daigle
        logMetacat.error("DBQuery.createResultDocument - SQL Error: " + e.getMessage());
547 2168 tao
      }
548
      catch (Exception ee)
549
      {
550 5165 daigle
        logMetacat.error("DBQuery.createResultDocument - General exception: "
551 2663 sgarg
                                 + ee.getMessage());
552 3219 berkley
        ee.printStackTrace();
553 2168 tao
      }
554
      finally
555
      {
556
        DBConnectionPool.returnDBConnection(dbconn, serialNumber);
557
      } //finally
558
    }//if
559 2087 tao
    String closeRestultset = "</resultset>";
560
    resultset.append(closeRestultset);
561
    if (out != null)
562
    {
563 5752 leinfelder
      try {
564
		out.write(closeRestultset);
565
		} catch (IOException e) {
566
			logMetacat.error(e.getMessage(), e);
567
		}
568 2087 tao
    }
569 2168 tao
570 3221 berkley
    //default to returning the whole resultset
571 2087 tao
    return resultset;
572
  }//createResultDocuments
573 2043 sgarg
574 2087 tao
    /*
575
     * Find the doc list which match the query
576
     */
577
    private StringBuffer findResultDoclist(QuerySpecification qspec,
578 5752 leinfelder
                                      Writer out,
579 2087 tao
                                      String user, String[]groups,
580 3211 berkley
                                      DBConnection dbconn, boolean useXMLIndex,
581 5490 berkley
                                      int pagesize, int pagestart, String sessionid,
582
                                      Vector givenDocids, String qformat)
583 2087 tao
                                      throws Exception
584
    {
585 3342 tao
      StringBuffer resultsetBuffer = new StringBuffer();
586 3219 berkley
      String query = null;
587
      int count = 0;
588
      int index = 0;
589 3246 berkley
      ResultDocumentSet docListResult = new ResultDocumentSet();
590 3219 berkley
      PreparedStatement pstmt = null;
591
      String docid = null;
592
      String docname = null;
593
      String doctype = null;
594
      String createDate = null;
595
      String updateDate = null;
596
      StringBuffer document = null;
597 3262 berkley
      boolean lastpage = false;
598 3219 berkley
      int rev = 0;
599
      double startTime = 0;
600 3368 tao
      int offset = 1;
601 5165 daigle
      long startSelectionTime = System.currentTimeMillis();
602 3219 berkley
      ResultSet rs = null;
603 3368 tao
604
605
      // this is a hack for offset. in postgresql 7, if the returned docid list is too long,
606
      //the extend query which base on the docid will be too long to be run. So we
607
      // have to cut them into different parts. Page query don't need it somehow.
608
      if (out == null)
609 2091 tao
      {
610
        // for html page, we put everything into one page
611 2421 sgarg
        offset =
612 4212 daigle
            (new Integer(PropertyService.getProperty("database.webResultsetSize"))).intValue();
613 2091 tao
      }
614
      else
615
      {
616
          offset =
617 4212 daigle
              (new Integer(PropertyService.getProperty("database.appResultsetSize"))).intValue();
618 3368 tao
      }
619 2421 sgarg
620 3047 perry
      /*
621
       * Check the docidOverride Vector
622
       * if defined, we bypass the qspec.printSQL() method
623
       * and contruct a simpler query based on a
624
       * list of docids rather than a bunch of subselects
625
       */
626 3392 tao
      if ( givenDocids == null || givenDocids.size() == 0 ) {
627 3047 perry
          query = qspec.printSQL(useXMLIndex);
628
      } else {
629 6035 leinfelder
    	  // condition for the docids
630
    	  StringBuffer docidCondition = new StringBuffer();
631
    	  docidCondition.append( " docid IN (" );
632 3392 tao
          for (int i = 0; i < givenDocids.size(); i++) {
633 6035 leinfelder
        	  docidCondition.append("'");
634
        	  docidCondition.append( (String)givenDocids.elementAt(i) );
635
        	  docidCondition.append("'");
636
        	  if (i < givenDocids.size()-1) {
637
        		  docidCondition.append(",");
638
        	  }
639 3047 perry
          }
640 6035 leinfelder
          docidCondition.append( ") " );
641
642
    	  // include the docids, either exclusively, or in conjuction with the query
643
    	  if (operator == null) {
644
    		  query = "SELECT docid, docname, doctype, date_created, date_updated, rev FROM xml_documents WHERE";
645
              query = query + docidCondition.toString();
646
    	  } else {
647
    		  // start with the keyword query, but add conditions
648
              query = qspec.printSQL(useXMLIndex);
649
              String myOperator = "";
650
              if (!query.endsWith("WHERE")) {
651
	              if (operator.equalsIgnoreCase(QueryGroup.UNION)) {
652
	            	  myOperator =  " OR ";
653
	              }
654
	              else {
655
	            	  myOperator =  " AND ";
656
	              }
657
              }
658
              query = query + myOperator + docidCondition.toString();
659
660
    	  }
661 3047 perry
      }
662 2087 tao
      String ownerQuery = getOwnerQuery(user);
663 4574 daigle
      //logMetacat.debug("query: " + query);
664 5165 daigle
      logMetacat.debug("DBQuery.findResultDoclist - owner query: " + ownerQuery);
665 2087 tao
      // if query is not the owner query, we need to check the permission
666
      // otherwise we don't need (owner has all permission by default)
667
      if (!query.equals(ownerQuery))
668
      {
669
        // set user name and group
670
        qspec.setUserName(user);
671
        qspec.setGroup(groups);
672
        // Get access query
673
        String accessQuery = qspec.getAccessQuery();
674 2366 sgarg
        if(!query.endsWith("WHERE")){
675
            query = query + accessQuery;
676
        } else {
677
            query = query + accessQuery.substring(4, accessQuery.length());
678
        }
679 3309 tao
680 2087 tao
      }
681 5165 daigle
      logMetacat.debug("DBQuery.findResultDoclist - final selection query: " + query);
682 3357 tao
      String selectionAndExtendedQuery = null;
683 3342 tao
      // we only get cache for public
684
      if (user != null && user.equalsIgnoreCase("public")
685 4212 daigle
     		 && pagesize == 0 && PropertyService.getProperty("database.queryCacheOn").equals("true"))
686 3342 tao
      {
687 3357 tao
    	  selectionAndExtendedQuery = query +qspec.getReturnDocList()+qspec.getReturnFieldList();
688
   	      String cachedResult = getResultXMLFromCache(selectionAndExtendedQuery);
689 5165 daigle
   	      logMetacat.debug("DBQuery.findResultDoclist - The key of query cache is " + selectionAndExtendedQuery);
690 3342 tao
   	      //System.out.println("==========the string from cache is "+cachedResult);
691
   	      if (cachedResult != null)
692
   	      {
693 5165 daigle
   	    	logMetacat.info("DBQuery.findResultDoclist - result from cache !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
694 3342 tao
   	    	 if (out != null)
695
   	         {
696 5752 leinfelder
   	             out.write(cachedResult);
697 3342 tao
   	         }
698
   	    	 resultsetBuffer.append(cachedResult);
699
   	    	 return resultsetBuffer;
700
   	      }
701
      }
702
703 3219 berkley
      startTime = System.currentTimeMillis() / 1000;
704 2087 tao
      pstmt = dbconn.prepareStatement(query);
705 3219 berkley
      rs = pstmt.executeQuery();
706 3246 berkley
707 2087 tao
      double queryExecuteTime = System.currentTimeMillis() / 1000;
708 5165 daigle
      logMetacat.debug("DBQuery.findResultDoclist - Time to execute select docid query is "
709 2663 sgarg
                    + (queryExecuteTime - startTime));
710 4698 daigle
      MetacatUtil.writeDebugToFile("\n\n\n\n\n\nExecute selection query  "
711 3271 tao
              + (queryExecuteTime - startTime));
712 4698 daigle
      MetacatUtil.writeDebugToDelimiteredFile(""+(queryExecuteTime - startTime), false);
713 3246 berkley
714 3247 berkley
      boolean tableHasRows = rs.next();
715 3246 berkley
716
      if(pagesize == 0)
717
      { //this makes sure we get all results if there is no paging
718 3368 tao
        pagesize = NONPAGESIZE;
719
        pagestart = NONPAGESIZE;
720 3246 berkley
      }
721
722
      int currentIndex = 0;
723 2087 tao
      while (tableHasRows)
724
      {
725 5165 daigle
        logMetacat.debug("DBQuery.findResultDoclist - getting result: " + currentIndex);
726 2087 tao
        docid = rs.getString(1).trim();
727 5165 daigle
        logMetacat.debug("DBQuery.findResultDoclist -  processing: " + docid);
728 2087 tao
        docname = rs.getString(2);
729
        doctype = rs.getString(3);
730 5165 daigle
        logMetacat.debug("DBQuery.findResultDoclist - processing: " + doctype);
731 2087 tao
        createDate = rs.getString(4);
732
        updateDate = rs.getString(5);
733
        rev = rs.getInt(6);
734 3246 berkley
735 3307 tao
         Vector returndocVec = qspec.getReturnDocList();
736
       if (returndocVec.size() == 0 || returndocVec.contains(doctype))
737 2087 tao
        {
738 5165 daigle
          logMetacat.debug("DBQuery.findResultDoclist - NOT Back tracing now...");
739 2087 tao
           document = new StringBuffer();
740 2043 sgarg
741 2087 tao
           String completeDocid = docid
742 4212 daigle
                            + PropertyService.getProperty("document.accNumSeparator");
743 2087 tao
           completeDocid += rev;
744
           document.append("<docid>").append(completeDocid).append("</docid>");
745
           if (docname != null)
746
           {
747
               document.append("<docname>" + docname + "</docname>");
748 3219 berkley
           }
749
           if (doctype != null)
750
           {
751
              document.append("<doctype>" + doctype + "</doctype>");
752
           }
753
           if (createDate != null)
754
           {
755
               document.append("<createdate>" + createDate + "</createdate>");
756
           }
757
           if (updateDate != null)
758
           {
759
             document.append("<updatedate>" + updateDate + "</updatedate>");
760
           }
761
           // Store the document id and the root node id
762 3246 berkley
763
           docListResult.addResultDocument(
764
             new ResultDocument(docid, (String) document.toString()));
765 5165 daigle
           logMetacat.info("DBQuery.findResultDoclist - real result: " + docid);
766 3246 berkley
           currentIndex++;
767 3219 berkley
           count++;
768 2087 tao
        }//else
769 3246 berkley
770 2087 tao
        // when doclist reached the offset number, send out doc list and empty
771
        // the hash table
772 3368 tao
        if (count == offset && pagesize == NONPAGESIZE)
773 3246 berkley
        { //if pagesize is not 0, do this later.
774 2087 tao
          //reset count
775 3262 berkley
          //logMetacat.warn("############doing subset cache");
776 2087 tao
          count = 0;
777 3246 berkley
          handleSubsetResult(qspec, resultsetBuffer, out, docListResult,
778 5490 berkley
                              user, groups,dbconn, useXMLIndex, qformat);
779 3246 berkley
          //reset docListResult
780
          docListResult = new ResultDocumentSet();
781 3368 tao
        }
782 3246 berkley
783 5165 daigle
       logMetacat.debug("DBQuery.findResultDoclist - currentIndex: " + currentIndex);
784
       logMetacat.debug("DBQuery.findResultDoclist - page comparator: " + (pagesize * pagestart) + pagesize);
785 3246 berkley
       if(currentIndex >= ((pagesize * pagestart) + pagesize))
786
       {
787
         ResultDocumentSet pagedResultsHash = new ResultDocumentSet();
788
         for(int i=pagesize*pagestart; i<docListResult.size(); i++)
789
         {
790
           pagedResultsHash.put(docListResult.get(i));
791
         }
792
793
         docListResult = pagedResultsHash;
794
         break;
795
       }
796 2087 tao
       // Advance to the next record in the cursor
797
       tableHasRows = rs.next();
798 3246 berkley
       if(!tableHasRows)
799
       {
800 3262 berkley
         ResultDocumentSet pagedResultsHash = new ResultDocumentSet();
801
         //get the last page of information then break
802 3368 tao
         if(pagesize != NONPAGESIZE)
803 3262 berkley
         {
804
           for(int i=pagesize*pagestart; i<docListResult.size(); i++)
805
           {
806
             pagedResultsHash.put(docListResult.get(i));
807
           }
808
           docListResult = pagedResultsHash;
809
         }
810
811
         lastpage = true;
812 3246 berkley
         break;
813
       }
814 2087 tao
     }//while
815 3246 berkley
816 2087 tao
     rs.close();
817
     pstmt.close();
818 5165 daigle
     long docListTime = System.currentTimeMillis() - startSelectionTime;
819
     long docListWarnLimit = Long.parseLong(PropertyService.getProperty("dbquery.findDocListTimeWarnLimit"));
820
     if (docListTime > docListWarnLimit) {
821
    	 logMetacat.warn("DBQuery.findResultDoclist - Total time to get docid list is: "
822
                          + docListTime);
823
     }
824 4698 daigle
     MetacatUtil.writeDebugToFile("---------------------------------------------------------------------------------------------------------------Total selection: "
825 5165 daigle
             + docListTime);
826
     MetacatUtil.writeDebugToDelimiteredFile(" "+ docListTime, false);
827 2087 tao
     //if docListResult is not empty, it need to be sent.
828 3246 berkley
     if (docListResult.size() != 0)
829 2087 tao
     {
830 3342 tao
831 2087 tao
       handleSubsetResult(qspec,resultsetBuffer, out, docListResult,
832 5490 berkley
                              user, groups,dbconn, useXMLIndex, qformat);
833 2087 tao
     }
834 2091 tao
835 3262 berkley
     resultsetBuffer.append("\n<lastpage>" + lastpage + "</lastpage>\n");
836
     if (out != null)
837
     {
838 5752 leinfelder
         out.write("\n<lastpage>" + lastpage + "</lastpage>\n");
839 3262 berkley
     }
840 3342 tao
841
     // now we only cached none-paged query and user is public
842
     if (user != null && user.equalsIgnoreCase("public")
843 4212 daigle
    		 && pagesize == NONPAGESIZE && PropertyService.getProperty("database.queryCacheOn").equals("true"))
844 3342 tao
     {
845
       //System.out.println("the string stored into cache is "+ resultsetBuffer.toString());
846 3357 tao
  	   storeQueryResultIntoCache(selectionAndExtendedQuery, resultsetBuffer.toString());
847 3342 tao
     }
848 3262 berkley
849 2087 tao
     return resultsetBuffer;
850
    }//findReturnDoclist
851 2043 sgarg
852
853 2087 tao
    /*
854
     * Send completed search hashtable(part of reulst)to output stream
855
     * and buffer into a buffer stream
856
     */
857
    private StringBuffer handleSubsetResult(QuerySpecification qspec,
858
                                           StringBuffer resultset,
859 5752 leinfelder
                                           Writer out, ResultDocumentSet partOfDoclist,
860 2087 tao
                                           String user, String[]groups,
861 5490 berkley
                                       DBConnection dbconn, boolean useXMLIndex,
862
                                       String qformat)
863 2087 tao
                                       throws Exception
864
   {
865 5165 daigle
     double startReturnFieldTime = System.currentTimeMillis();
866 2424 sgarg
     // check if there is a record in xml_returnfield
867
     // and get the returnfield_id and usage count
868
     int usage_count = getXmlReturnfieldsTableId(qspec, dbconn);
869
     boolean enterRecords = false;
870
871 4212 daigle
     // get value of database.xmlReturnfieldCount
872 4080 daigle
     int count = (new Integer(PropertyService
873 4212 daigle
                            .getProperty("database.xmlReturnfieldCount")))
874 2424 sgarg
                            .intValue();
875 2430 sgarg
876 2446 sgarg
     // set enterRecords to true if usage_count is more than the offset
877 2430 sgarg
     // specified in metacat.properties
878 2424 sgarg
     if(usage_count > count){
879
         enterRecords = true;
880
     }
881 3257 berkley
882 2421 sgarg
     if(returnfield_id < 0){
883 5165 daigle
         logMetacat.warn("DBQuery.handleSubsetResult - Error in getting returnfield id from"
884 2663 sgarg
                                  + "xml_returnfield table");
885 3227 berkley
         enterRecords = false;
886 2421 sgarg
     }
887
888
     // get the hashtable containing the docids that already in the
889
     // xml_queryresult table
890 5165 daigle
     logMetacat.info("DBQuery.handleSubsetResult - size of partOfDoclist before"
891 2421 sgarg
                             + " docidsInQueryresultTable(): "
892 2663 sgarg
                             + partOfDoclist.size());
893 5165 daigle
     long startGetReturnValueFromQueryresultable = System.currentTimeMillis();
894 2421 sgarg
     Hashtable queryresultDocList = docidsInQueryresultTable(returnfield_id,
895
                                                        partOfDoclist, dbconn);
896
897
     // remove the keys in queryresultDocList from partOfDoclist
898
     Enumeration _keys = queryresultDocList.keys();
899
     while (_keys.hasMoreElements()){
900 3246 berkley
         partOfDoclist.remove((String)_keys.nextElement());
901 2421 sgarg
     }
902 5165 daigle
903
     long queryResultReturnValuetime = System.currentTimeMillis() - startGetReturnValueFromQueryresultable;
904
     long queryResultWarnLimit =
905
    	 Long.parseLong(PropertyService.getProperty("dbquery.findQueryResultsTimeWarnLimit"));
906
907
     if (queryResultReturnValuetime > queryResultWarnLimit) {
908
    	 logMetacat.warn("DBQuery.handleSubsetResult - Time to get return fields from xml_queryresult table is (Part1 in return fields) " +
909
    		 queryResultReturnValuetime);
910
     }
911 4698 daigle
     MetacatUtil.writeDebugToFile("-----------------------------------------Get fields from xml_queryresult(Part1 in return fields) " +
912 5165 daigle
    		 queryResultReturnValuetime);
913
     MetacatUtil.writeDebugToDelimiteredFile(" " + queryResultReturnValuetime,false);
914
915
     long startExtendedQuery = System.currentTimeMillis();
916 2425 sgarg
     // backup the keys-elements in partOfDoclist to check later
917
     // if the doc entry is indexed yet
918
     Hashtable partOfDoclistBackup = new Hashtable();
919 3246 berkley
     Iterator itt = partOfDoclist.getDocids();
920
     while (itt.hasNext()){
921
       Object key = itt.next();
922 2425 sgarg
         partOfDoclistBackup.put(key, partOfDoclist.get(key));
923
     }
924
925 5165 daigle
     logMetacat.info("DBQuery.handleSubsetResult - size of partOfDoclist after"
926 2421 sgarg
                             + " docidsInQueryresultTable(): "
927 2663 sgarg
                             + partOfDoclist.size());
928 2421 sgarg
929
     //add return fields for the documents in partOfDoclist
930
     partOfDoclist = addReturnfield(partOfDoclist, qspec, user, groups,
931 5490 berkley
                                        dbconn, useXMLIndex, qformat);
932 5165 daigle
     long extendedQueryRunTime = startExtendedQuery - System.currentTimeMillis();
933
     long extendedQueryWarnLimit =
934
    	 Long.parseLong(PropertyService.getProperty("dbquery.extendedQueryRunTimeWarnLimit"));
935
936
     if (extendedQueryRunTime > extendedQueryWarnLimit) {
937
    	 logMetacat.warn("DBQuery.handleSubsetResult - Get fields from index and node table (Part2 in return fields) "
938
        		                                          + extendedQueryRunTime);
939
     }
940 4698 daigle
     MetacatUtil.writeDebugToFile("-----------------------------------------Get fields from extened query(Part2 in return fields) "
941 5165 daigle
             + extendedQueryRunTime);
942 4698 daigle
     MetacatUtil.writeDebugToDelimiteredFile(" "
943 5165 daigle
             + extendedQueryRunTime, false);
944 2421 sgarg
     //add relationship part part docid list for the documents in partOfDocList
945 3730 tao
     //partOfDoclist = addRelationship(partOfDoclist, qspec, dbconn, useXMLIndex);
946 2421 sgarg
947 5165 daigle
     long startStoreReturnField = System.currentTimeMillis();
948 3246 berkley
     Iterator keys = partOfDoclist.getDocids();
949 2087 tao
     String key = null;
950
     String element = null;
951 2421 sgarg
     String query = null;
952 4080 daigle
     int offset = (new Integer(PropertyService
953 4212 daigle
                               .getProperty("database.queryresultStringLength")))
954 2421 sgarg
                               .intValue();
955 3246 berkley
     while (keys.hasNext())
956 2087 tao
     {
957 3246 berkley
         key = (String) keys.next();
958 2421 sgarg
         element = (String)partOfDoclist.get(key);
959 3350 tao
960 2446 sgarg
	 // check if the enterRecords is true, elements is not null, element's
961
         // length is less than the limit of table column and if the document
962 2425 sgarg
         // has been indexed already
963 2446 sgarg
         if(enterRecords && element != null
964 2425 sgarg
		&& element.length() < offset
965
		&& element.compareTo((String) partOfDoclistBackup.get(key)) != 0){
966 2421 sgarg
             query = "INSERT INTO xml_queryresult (returnfield_id, docid, "
967 2446 sgarg
                 + "queryresult_string) VALUES (?, ?, ?)";
968
969 2421 sgarg
             PreparedStatement pstmt = null;
970
             pstmt = dbconn.prepareStatement(query);
971 2446 sgarg
             pstmt.setInt(1, returnfield_id);
972
             pstmt.setString(2, key);
973
             pstmt.setString(3, element);
974 3350 tao
975 2421 sgarg
             dbconn.increaseUsageCount(1);
976 3350 tao
             try
977
             {
978
            	 pstmt.execute();
979
             }
980
             catch(Exception e)
981
             {
982 5165 daigle
            	 logMetacat.warn("DBQuery.handleSubsetResult - couldn't insert the element to xml_queryresult table "+e.getLocalizedMessage());
983 3350 tao
             }
984
             finally
985
             {
986
                pstmt.close();
987
             }
988 2421 sgarg
         }
989 3263 tao
990 2421 sgarg
         // A string with element
991
         String xmlElement = "  <document>" + element + "</document>";
992 3257 berkley
993 2421 sgarg
         //send single element to output
994
         if (out != null)
995
         {
996 5752 leinfelder
             out.write(xmlElement);
997 2421 sgarg
         }
998
         resultset.append(xmlElement);
999
     }//while
1000 3263 tao
1001 5165 daigle
     double storeReturnFieldTime = System.currentTimeMillis() - startStoreReturnField;
1002
     long storeReturnFieldWarnLimit =
1003
    	 Long.parseLong(PropertyService.getProperty("dbquery.storeReturnFieldTimeWarnLimit"));
1004
1005
     if (storeReturnFieldTime > storeReturnFieldWarnLimit) {
1006
    	 logMetacat.warn("DBQuery.handleSubsetResult - Time to store new return fields into xml_queryresult table (Part4 in return fields) "
1007
                   + storeReturnFieldTime);
1008
     }
1009 4698 daigle
     MetacatUtil.writeDebugToFile("-----------------------------------------Insert new record to xml_queryresult(Part4 in return fields) "
1010 5165 daigle
             + storeReturnFieldTime);
1011
     MetacatUtil.writeDebugToDelimiteredFile(" " + storeReturnFieldTime, false);
1012 3263 tao
1013 3246 berkley
     Enumeration keysE = queryresultDocList.keys();
1014
     while (keysE.hasMoreElements())
1015 2421 sgarg
     {
1016 3246 berkley
         key = (String) keysE.nextElement();
1017 2421 sgarg
         element = (String)queryresultDocList.get(key);
1018
         // A string with element
1019
         String xmlElement = "  <document>" + element + "</document>";
1020
         //send single element to output
1021
         if (out != null)
1022
         {
1023 5752 leinfelder
             out.write(xmlElement);
1024 2421 sgarg
         }
1025
         resultset.append(xmlElement);
1026
     }//while
1027 5165 daigle
     double returnFieldTime = System.currentTimeMillis() - startReturnFieldTime;
1028
     long totalReturnFieldWarnLimit =
1029
    	 Long.parseLong(PropertyService.getProperty("dbquery.totalReturnFieldTimeWarnLimit"));
1030
1031
     if (returnFieldTime > totalReturnFieldWarnLimit) {
1032
    	 logMetacat.warn("DBQuery.handleSubsetResult - Total time to get return fields is: "
1033
                           + returnFieldTime);
1034
     }
1035
     MetacatUtil.writeDebugToFile("DBQuery.handleSubsetResult - ---------------------------------------------------------------------------------------------------------------"+
1036
    		 "Total to get return fields  " + returnFieldTime);
1037
     MetacatUtil.writeDebugToDelimiteredFile("DBQuery.handleSubsetResult - "+ returnFieldTime, false);
1038 2421 sgarg
     return resultset;
1039
 }
1040
1041
   /**
1042
    * Get the docids already in xml_queryresult table and corresponding
1043
    * queryresultstring as a hashtable
1044
    */
1045
   private Hashtable docidsInQueryresultTable(int returnfield_id,
1046 3246 berkley
                                              ResultDocumentSet partOfDoclist,
1047 2421 sgarg
                                              DBConnection dbconn){
1048
1049
         Hashtable returnValue = new Hashtable();
1050
         PreparedStatement pstmt = null;
1051
         ResultSet rs = null;
1052
1053
         // get partOfDoclist as string for the query
1054 3246 berkley
         Iterator keylist = partOfDoclist.getDocids();
1055 2421 sgarg
         StringBuffer doclist = new StringBuffer();
1056 3246 berkley
         while (keylist.hasNext())
1057 2421 sgarg
         {
1058
             doclist.append("'");
1059 3246 berkley
             doclist.append((String) keylist.next());
1060 2421 sgarg
             doclist.append("',");
1061
         }//while
1062
1063
1064
         if (doclist.length() > 0)
1065
         {
1066
             doclist.deleteCharAt(doclist.length() - 1); //remove the last comma
1067
1068
             // the query to find out docids from xml_queryresult
1069
             String query = "select docid, queryresult_string from "
1070
                          + "xml_queryresult where returnfield_id = " +
1071
                          returnfield_id +" and docid in ("+ doclist + ")";
1072 5165 daigle
             logMetacat.info("DBQuery.docidsInQueryresultTable - Query to get docids from xml_queryresult:"
1073 2663 sgarg
                                      + query);
1074 2421 sgarg
1075
             try {
1076
                 // prepare and execute the query
1077
                 pstmt = dbconn.prepareStatement(query);
1078
                 dbconn.increaseUsageCount(1);
1079
                 pstmt.execute();
1080
                 rs = pstmt.getResultSet();
1081
                 boolean tableHasRows = rs.next();
1082
                 while (tableHasRows) {
1083
                     // store the returned results in the returnValue hashtable
1084
                     String key = rs.getString(1);
1085
                     String element = rs.getString(2);
1086
1087
                     if(element != null){
1088
                         returnValue.put(key, element);
1089
                     } else {
1090 5165 daigle
                         logMetacat.info("DBQuery.docidsInQueryresultTable - Null elment found ("
1091 2663 sgarg
                         + "DBQuery.docidsInQueryresultTable)");
1092 2421 sgarg
                     }
1093
                     tableHasRows = rs.next();
1094
                 }
1095
                 rs.close();
1096
                 pstmt.close();
1097
             } catch (Exception e){
1098 5165 daigle
                 logMetacat.error("DBQuery.docidsInQueryresultTable - Error getting docids from "
1099
                                          + "queryresult: " + e.getMessage());
1100 2421 sgarg
              }
1101
         }
1102
         return returnValue;
1103
     }
1104
1105
1106
   /**
1107
    * Method to get id from xml_returnfield table
1108
    * for a given query specification
1109
    */
1110 2424 sgarg
   private int returnfield_id;
1111 2421 sgarg
   private int getXmlReturnfieldsTableId(QuerySpecification qspec,
1112
                                           DBConnection dbconn){
1113
       int id = -1;
1114 2424 sgarg
       int count = 1;
1115 2421 sgarg
       PreparedStatement pstmt = null;
1116
       ResultSet rs = null;
1117
       String returnfield = qspec.getSortedReturnFieldString();
1118
1119
       // query for finding the id from xml_returnfield
1120 2446 sgarg
       String query = "SELECT returnfield_id, usage_count FROM xml_returnfield "
1121
            + "WHERE returnfield_string LIKE ?";
1122 5165 daigle
       logMetacat.info("DBQuery.getXmlReturnfieldsTableId - ReturnField Query:" + query);
1123 2421 sgarg
1124
       try {
1125
           // prepare and run the query
1126
           pstmt = dbconn.prepareStatement(query);
1127 2446 sgarg
           pstmt.setString(1,returnfield);
1128 2421 sgarg
           dbconn.increaseUsageCount(1);
1129
           pstmt.execute();
1130
           rs = pstmt.getResultSet();
1131
           boolean tableHasRows = rs.next();
1132
1133
           // if record found then increase the usage count
1134
           // else insert a new record and get the id of the new record
1135
           if(tableHasRows){
1136
               // get the id
1137
               id = rs.getInt(1);
1138 2424 sgarg
               count = rs.getInt(2) + 1;
1139 2421 sgarg
               rs.close();
1140
               pstmt.close();
1141
1142
               // increase the usage count
1143
               query = "UPDATE xml_returnfield SET usage_count ='" + count
1144
                   + "' WHERE returnfield_id ='"+ id +"'";
1145 5165 daigle
               logMetacat.info("DBQuery.getXmlReturnfieldsTableId - ReturnField Table Update:"+ query);
1146 2421 sgarg
1147
               pstmt = dbconn.prepareStatement(query);
1148
               dbconn.increaseUsageCount(1);
1149
               pstmt.execute();
1150
               pstmt.close();
1151
1152
           } else {
1153
               rs.close();
1154
               pstmt.close();
1155
1156
               // insert a new record
1157
               query = "INSERT INTO xml_returnfield (returnfield_string, usage_count)"
1158 2446 sgarg
                   + "VALUES (?, '1')";
1159 5165 daigle
               logMetacat.info("DBQuery.getXmlReturnfieldsTableId - ReturnField Table Insert:"+ query);
1160 2421 sgarg
               pstmt = dbconn.prepareStatement(query);
1161 2446 sgarg
               pstmt.setString(1, returnfield);
1162 2421 sgarg
               dbconn.increaseUsageCount(1);
1163
               pstmt.execute();
1164
               pstmt.close();
1165
1166
               // get the id of the new record
1167 2446 sgarg
               query = "SELECT returnfield_id FROM xml_returnfield "
1168
                   + "WHERE returnfield_string LIKE ?";
1169 5165 daigle
               logMetacat.info("DBQuery.getXmlReturnfieldsTableId - ReturnField query after Insert:" + query);
1170 2421 sgarg
               pstmt = dbconn.prepareStatement(query);
1171 2446 sgarg
               pstmt.setString(1, returnfield);
1172
1173 2421 sgarg
               dbconn.increaseUsageCount(1);
1174
               pstmt.execute();
1175
               rs = pstmt.getResultSet();
1176
               if(rs.next()){
1177
                   id = rs.getInt(1);
1178
               } else {
1179
                   id = -1;
1180
               }
1181
               rs.close();
1182
               pstmt.close();
1183 2087 tao
           }
1184 2091 tao
1185 2421 sgarg
       } catch (Exception e){
1186 5165 daigle
           logMetacat.error("DBQuery.getXmlReturnfieldsTableId - Error getting id from xml_returnfield in "
1187 2421 sgarg
                                     + "DBQuery.getXmlReturnfieldsTableId: "
1188 2663 sgarg
                                     + e.getMessage());
1189 2421 sgarg
           id = -1;
1190
       }
1191 2424 sgarg
1192
       returnfield_id = id;
1193
       return count;
1194 2087 tao
   }
1195 2043 sgarg
1196
1197 2087 tao
    /*
1198
     * A method to add return field to return doclist hash table
1199
     */
1200 3246 berkley
    private ResultDocumentSet addReturnfield(ResultDocumentSet docListResult,
1201 2087 tao
                                      QuerySpecification qspec,
1202
                                      String user, String[]groups,
1203 5490 berkley
                                      DBConnection dbconn, boolean useXMLIndex,
1204
                                      String qformat)
1205 2087 tao
                                      throws Exception
1206
    {
1207
      PreparedStatement pstmt = null;
1208
      ResultSet rs = null;
1209
      String docid = null;
1210
      String fieldname = null;
1211 3635 leinfelder
      String fieldtype = null;
1212 2087 tao
      String fielddata = null;
1213
      String relation = null;
1214
1215
      if (qspec.containsExtendedSQL())
1216
      {
1217
        qspec.setUserName(user);
1218
        qspec.setGroup(groups);
1219
        Vector extendedFields = new Vector(qspec.getReturnFieldList());
1220
        Vector results = new Vector();
1221 3246 berkley
        Iterator keylist = docListResult.getDocids();
1222 2087 tao
        StringBuffer doclist = new StringBuffer();
1223
        Vector parentidList = new Vector();
1224
        Hashtable returnFieldValue = new Hashtable();
1225 3246 berkley
        while (keylist.hasNext())
1226 2087 tao
        {
1227 5490 berkley
          String key = (String)keylist.next();
1228 2087 tao
          doclist.append("'");
1229 5490 berkley
          doclist.append(key);
1230 2087 tao
          doclist.append("',");
1231
        }
1232
        if (doclist.length() > 0)
1233
        {
1234
          Hashtable controlPairs = new Hashtable();
1235
          doclist.deleteCharAt(doclist.length() - 1); //remove the last comma
1236 3248 tao
          boolean tableHasRows = false;
1237 3349 tao
1238 2087 tao
1239
           String extendedQuery =
1240 3248 tao
               qspec.printExtendedSQL(doclist.toString(), useXMLIndex);
1241 5165 daigle
           logMetacat.info("DBQuery.addReturnfield - Extended query: " + extendedQuery);
1242 2376 sgarg
1243 2474 sgarg
           if(extendedQuery != null){
1244 5165 daigle
//        	   long extendedQueryStart = System.currentTimeMillis();
1245 2474 sgarg
               pstmt = dbconn.prepareStatement(extendedQuery);
1246
               //increase dbconnection usage count
1247
               dbconn.increaseUsageCount(1);
1248
               pstmt.execute();
1249
               rs = pstmt.getResultSet();
1250
               tableHasRows = rs.next();
1251
               while (tableHasRows) {
1252
                   ReturnFieldValue returnValue = new ReturnFieldValue();
1253
                   docid = rs.getString(1).trim();
1254
                   fieldname = rs.getString(2);
1255 5490 berkley
1256
                   if(qformat.toLowerCase().trim().equals("xml"))
1257
                   {
1258
                       byte[] b = rs.getBytes(3);
1259 5756 leinfelder
                       fielddata = new String(b, 0, b.length, MetaCatServlet.DEFAULT_ENCODING);
1260 5490 berkley
                   }
1261
                   else
1262
                   {
1263
                       fielddata = rs.getString(3);
1264
                   }
1265
1266
                   //System.out.println("raw fielddata: " + fielddata);
1267 4698 daigle
                   fielddata = MetacatUtil.normalize(fielddata);
1268 5490 berkley
                   //System.out.println("normalized fielddata: " + fielddata);
1269 2474 sgarg
                   String parentId = rs.getString(4);
1270 3635 leinfelder
                   fieldtype = rs.getString(5);
1271 2474 sgarg
                   StringBuffer value = new StringBuffer();
1272 2043 sgarg
1273 3635 leinfelder
                   //handle case when usexmlindex is true differently
1274
                   //at one point merging the nodedata (for large text elements) was
1275
                   //deemed unnecessary - but now it is needed.  but not for attribute nodes
1276 2474 sgarg
                   if (useXMLIndex || !containsKey(parentidList, parentId)) {
1277 3635 leinfelder
                	   //merge node data only for non-ATTRIBUTEs
1278
                	   if (fieldtype != null && !fieldtype.equals("ATTRIBUTE")) {
1279
	                	   //try merging the data
1280
	                	   ReturnFieldValue existingRFV =
1281
	                		   getArrayValue(parentidList, parentId);
1282 5387 berkley
	                	   if (existingRFV != null && !existingRFV.getFieldType().equals("ATTRIBUTE")) {
1283 3635 leinfelder
	                		   fielddata = existingRFV.getFieldValue() + fielddata;
1284
	                	   }
1285
                	   }
1286 5387 berkley
                	   //System.out.println("fieldname: " + fieldname + " fielddata: " + fielddata);
1287 5490 berkley
1288 2474 sgarg
                       value.append("<param name=\"");
1289
                       value.append(fieldname);
1290
                       value.append("\">");
1291
                       value.append(fielddata);
1292
                       value.append("</param>");
1293
                       //set returnvalue
1294
                       returnValue.setDocid(docid);
1295
                       returnValue.setFieldValue(fielddata);
1296 3635 leinfelder
                       returnValue.setFieldType(fieldtype);
1297 2474 sgarg
                       returnValue.setXMLFieldValue(value.toString());
1298
                       // Store it in hastable
1299
                       putInArray(parentidList, parentId, returnValue);
1300
                   }
1301
                   else {
1302 5490 berkley
1303 2474 sgarg
                       // need to merge nodedata if they have same parent id and
1304
                       // node type is text
1305
                       fielddata = (String) ( (ReturnFieldValue)
1306
                                             getArrayValue(
1307
                           parentidList, parentId)).getFieldValue()
1308
                           + fielddata;
1309 5490 berkley
                       //System.out.println("fieldname: " + fieldname + " fielddata: " + fielddata);
1310 2474 sgarg
                       value.append("<param name=\"");
1311
                       value.append(fieldname);
1312
                       value.append("\">");
1313
                       value.append(fielddata);
1314
                       value.append("</param>");
1315
                       returnValue.setDocid(docid);
1316
                       returnValue.setFieldValue(fielddata);
1317 3635 leinfelder
                       returnValue.setFieldType(fieldtype);
1318 2474 sgarg
                       returnValue.setXMLFieldValue(value.toString());
1319
                       // remove the old return value from paretnidList
1320
                       parentidList.remove(parentId);
1321
                       // store the new return value in parentidlit
1322
                       putInArray(parentidList, parentId, returnValue);
1323
                   }
1324
                   tableHasRows = rs.next();
1325
               } //while
1326
               rs.close();
1327
               pstmt.close();
1328 2043 sgarg
1329 2474 sgarg
               // put the merger node data info into doclistReult
1330
               Enumeration xmlFieldValue = (getElements(parentidList)).
1331
                   elements();
1332
               while (xmlFieldValue.hasMoreElements()) {
1333
                   ReturnFieldValue object =
1334
                       (ReturnFieldValue) xmlFieldValue.nextElement();
1335
                   docid = object.getDocid();
1336 3246 berkley
                   if (docListResult.containsDocid(docid)) {
1337 2474 sgarg
                       String removedelement = (String) docListResult.
1338
                           remove(docid);
1339
                       docListResult.
1340 3246 berkley
                           addResultDocument(new ResultDocument(docid,
1341
                               removedelement + object.getXMLFieldValue()));
1342 2474 sgarg
                   }
1343
                   else {
1344 3246 berkley
                       docListResult.addResultDocument(
1345
                         new ResultDocument(docid, object.getXMLFieldValue()));
1346 2474 sgarg
                   }
1347
               } //while
1348 5165 daigle
//               double docListResultEnd = System.currentTimeMillis() / 1000;
1349
//               logMetacat.warn(
1350
//                   "Time to prepare ResultDocumentSet after"
1351
//                   + " execute extended query: "
1352
//                   + (docListResultEnd - extendedQueryEnd));
1353 2474 sgarg
           }
1354 2087 tao
       }//if doclist lenght is great than zero
1355
     }//if has extended query
1356 2043 sgarg
1357 2087 tao
      return docListResult;
1358
    }//addReturnfield
1359 2043 sgarg
1360 3730 tao
1361 2087 tao
  /**
1362
   * removes the <?xml version="1.0"?> tag from the beginning.  This takes a
1363
   * string as a param instead of a hashtable.
1364
   *
1365
   * @param xmlquery a string representing a query.
1366
   */
1367
   private  String transformQuery(String xmlquery)
1368
   {
1369
     xmlquery = xmlquery.trim();
1370
     int index = xmlquery.indexOf("?>");
1371
     if (index != -1)
1372
     {
1373
       return xmlquery.substring(index + 2, xmlquery.length());
1374
     }
1375
     else
1376
     {
1377
       return xmlquery;
1378
     }
1379
   }
1380 3340 tao
1381
   /*
1382 3342 tao
    * Method to store query string and result xml string into query result
1383 3340 tao
    * cache. If the size alreay reache the limitation, the cache will be
1384
    * cleared first, then store them.
1385
    */
1386 3342 tao
   private void storeQueryResultIntoCache(String query, String resultXML)
1387 3340 tao
   {
1388
	   synchronized (queryResultCache)
1389
	   {
1390
		   if (queryResultCache.size() >= QUERYRESULTCACHESIZE)
1391
		   {
1392
			   queryResultCache.clear();
1393
		   }
1394 3342 tao
		   queryResultCache.put(query, resultXML);
1395 3340 tao
1396
	   }
1397
   }
1398
1399
   /*
1400 3342 tao
    * Method to get result xml string from query result cache.
1401
    * Note: the returned string can be null.
1402 3340 tao
    */
1403 3342 tao
   private String getResultXMLFromCache(String query)
1404 3340 tao
   {
1405 3342 tao
	   String resultSet = null;
1406 3340 tao
	   synchronized (queryResultCache)
1407
	   {
1408
          try
1409
          {
1410 5165 daigle
        	 logMetacat.info("DBQuery.getResultXMLFromCache - Get query from cache");
1411 3342 tao
		     resultSet = (String)queryResultCache.get(query);
1412 3340 tao
1413
          }
1414
          catch (Exception e)
1415
          {
1416
        	  resultSet = null;
1417
          }
1418
1419
	   }
1420
	   return resultSet;
1421
   }
1422
1423
   /**
1424
    * Method to clear the query result cache.
1425
    */
1426
   public static void clearQueryResultCache()
1427
   {
1428
	   synchronized (queryResultCache)
1429
	   {
1430
		   queryResultCache.clear();
1431
	   }
1432
   }
1433 2087 tao
1434
1435 2075 jones
    /*
1436
     * A method to search if Vector contains a particular key string
1437
     */
1438
    private boolean containsKey(Vector parentidList, String parentId)
1439
    {
1440 2043 sgarg
1441 2075 jones
        Vector tempVector = null;
1442 2043 sgarg
1443 2075 jones
        for (int count = 0; count < parentidList.size(); count++) {
1444
            tempVector = (Vector) parentidList.get(count);
1445 2360 sgarg
            if (parentId.compareTo((String) tempVector.get(0)) == 0) { return true; }
1446 2075 jones
        }
1447
        return false;
1448 2043 sgarg
    }
1449 3635 leinfelder
1450 2075 jones
    /*
1451
     * A method to put key and value in Vector
1452
     */
1453
    private void putInArray(Vector parentidList, String key,
1454
            ReturnFieldValue value)
1455
    {
1456 2043 sgarg
1457 2075 jones
        Vector tempVector = null;
1458 3635 leinfelder
        //only filter if the field type is NOT an attribute (say, for text)
1459
        String fieldType = value.getFieldType();
1460
        if (fieldType != null && !fieldType.equals("ATTRIBUTE")) {
1461
1462
	        for (int count = 0; count < parentidList.size(); count++) {
1463
	            tempVector = (Vector) parentidList.get(count);
1464
1465
	            if (key.compareTo((String) tempVector.get(0)) == 0) {
1466
	                tempVector.remove(1);
1467
	                tempVector.add(1, value);
1468
	                return;
1469
	            }
1470
	        }
1471 2075 jones
        }
1472 2043 sgarg
1473 2075 jones
        tempVector = new Vector();
1474
        tempVector.add(0, key);
1475
        tempVector.add(1, value);
1476
        parentidList.add(tempVector);
1477
        return;
1478 2043 sgarg
    }
1479
1480 2075 jones
    /*
1481
     * A method to get value in Vector given a key
1482
     */
1483
    private ReturnFieldValue getArrayValue(Vector parentidList, String key)
1484 1353 tao
    {
1485 2043 sgarg
1486 2075 jones
        Vector tempVector = null;
1487 2043 sgarg
1488 5490 berkley
        for (int count = 0; count < parentidList.size(); count++) {
1489 2075 jones
            tempVector = (Vector) parentidList.get(count);
1490 2043 sgarg
1491 5490 berkley
            if (key.compareTo((String) tempVector.get(0)) == 0) { return (ReturnFieldValue) tempVector
1492
                    .get(1); }
1493 2075 jones
        }
1494
        return null;
1495 2045 tao
    }
1496 436 berkley
1497 2075 jones
    /*
1498
     * A method to get enumeration of all values in Vector
1499
     */
1500
    private Vector getElements(Vector parentidList)
1501 342 berkley
    {
1502 2446 sgarg
        Vector enumVector = new Vector();
1503 2075 jones
        Vector tempVector = null;
1504 2043 sgarg
1505 2075 jones
        for (int count = 0; count < parentidList.size(); count++) {
1506
            tempVector = (Vector) parentidList.get(count);
1507 744 jones
1508 2446 sgarg
            enumVector.add(tempVector.get(1));
1509 744 jones
        }
1510 2446 sgarg
        return enumVector;
1511 372 berkley
    }
1512 2043 sgarg
1513 3308 tao
1514 2043 sgarg
1515 2075 jones
    /*
1516
     * A method to create a query to get owner's docid list
1517
     */
1518
    private String getOwnerQuery(String owner)
1519 372 berkley
    {
1520 2075 jones
        if (owner != null) {
1521
            owner = owner.toLowerCase();
1522
        }
1523
        StringBuffer self = new StringBuffer();
1524 2043 sgarg
1525 2075 jones
        self.append("SELECT docid,docname,doctype,");
1526
        self.append("date_created, date_updated, rev ");
1527
        self.append("FROM xml_documents WHERE docid IN (");
1528
        self.append("(");
1529
        self.append("SELECT DISTINCT docid FROM xml_nodes WHERE \n");
1530
        self.append("nodedata LIKE '%%%' ");
1531
        self.append(") \n");
1532
        self.append(") ");
1533
        self.append(" AND (");
1534
        self.append(" lower(user_owner) = '" + owner + "'");
1535
        self.append(") ");
1536
        return self.toString();
1537 342 berkley
    }
1538 2043 sgarg
1539 2075 jones
    /**
1540
     * format a structured query as an XML document that conforms to the
1541
     * pathquery.dtd and is appropriate for submission to the DBQuery
1542
     * structured query engine
1543 2087 tao
     *
1544 2075 jones
     * @param params The list of parameters that should be included in the
1545
     *            query
1546
     */
1547 4080 daigle
    public static String createSQuery(Hashtable params) throws PropertyNotFoundException
1548 342 berkley
    {
1549 2075 jones
        StringBuffer query = new StringBuffer();
1550
        Enumeration elements;
1551
        Enumeration keys;
1552
        String filterDoctype = null;
1553
        String casesensitive = null;
1554
        String searchmode = null;
1555
        Object nextkey;
1556
        Object nextelement;
1557
        //add the xml headers
1558
        query.append("<?xml version=\"1.0\"?>\n");
1559 2091 tao
        query.append("<pathquery version=\"1.2\">\n");
1560 372 berkley
1561 2091 tao
1562
1563 2075 jones
        if (params.containsKey("meta_file_id")) {
1564
            query.append("<meta_file_id>");
1565
            query.append(((String[]) params.get("meta_file_id"))[0]);
1566
            query.append("</meta_file_id>");
1567 372 berkley
        }
1568 2043 sgarg
1569 2075 jones
        if (params.containsKey("returndoctype")) {
1570
            String[] returnDoctypes = ((String[]) params.get("returndoctype"));
1571
            for (int i = 0; i < returnDoctypes.length; i++) {
1572
                String doctype = (String) returnDoctypes[i];
1573 181 jones
1574 2075 jones
                if (!doctype.equals("any") && !doctype.equals("ANY")
1575
                        && !doctype.equals("")) {
1576
                    query.append("<returndoctype>").append(doctype);
1577
                    query.append("</returndoctype>");
1578
                }
1579
            }
1580
        }
1581 181 jones
1582 2075 jones
        if (params.containsKey("filterdoctype")) {
1583
            String[] filterDoctypes = ((String[]) params.get("filterdoctype"));
1584
            for (int i = 0; i < filterDoctypes.length; i++) {
1585
                query.append("<filterdoctype>").append(filterDoctypes[i]);
1586
                query.append("</filterdoctype>");
1587
            }
1588
        }
1589 181 jones
1590 2075 jones
        if (params.containsKey("returnfield")) {
1591
            String[] returnfield = ((String[]) params.get("returnfield"));
1592
            for (int i = 0; i < returnfield.length; i++) {
1593
                query.append("<returnfield>").append(returnfield[i]);
1594
                query.append("</returnfield>");
1595
            }
1596
        }
1597 2043 sgarg
1598 2075 jones
        if (params.containsKey("owner")) {
1599
            String[] owner = ((String[]) params.get("owner"));
1600
            for (int i = 0; i < owner.length; i++) {
1601
                query.append("<owner>").append(owner[i]);
1602
                query.append("</owner>");
1603
            }
1604
        }
1605 181 jones
1606 2075 jones
        if (params.containsKey("site")) {
1607
            String[] site = ((String[]) params.get("site"));
1608
            for (int i = 0; i < site.length; i++) {
1609
                query.append("<site>").append(site[i]);
1610
                query.append("</site>");
1611
            }
1612
        }
1613 2043 sgarg
1614 2075 jones
        //allows the dynamic switching of boolean operators
1615
        if (params.containsKey("operator")) {
1616
            query.append("<querygroup operator=\""
1617
                    + ((String[]) params.get("operator"))[0] + "\">");
1618
        } else { //the default operator is UNION
1619
            query.append("<querygroup operator=\"UNION\">");
1620
        }
1621 940 tao
1622 2075 jones
        if (params.containsKey("casesensitive")) {
1623
            casesensitive = ((String[]) params.get("casesensitive"))[0];
1624
        } else {
1625
            casesensitive = "false";
1626
        }
1627 2043 sgarg
1628 2075 jones
        if (params.containsKey("searchmode")) {
1629
            searchmode = ((String[]) params.get("searchmode"))[0];
1630
        } else {
1631
            searchmode = "contains";
1632 940 tao
        }
1633
1634 2075 jones
        //anyfield is a special case because it does a
1635
        //free text search. It does not have a <pathexpr>
1636
        //tag. This allows for a free text search within the structured
1637
        //query. This is useful if the INTERSECT operator is used.
1638
        if (params.containsKey("anyfield")) {
1639
            String[] anyfield = ((String[]) params.get("anyfield"));
1640
            //allow for more than one value for anyfield
1641
            for (int i = 0; i < anyfield.length; i++) {
1642 4135 berkley
                if (anyfield[i] != null && !anyfield[i].equals("")) {
1643 2075 jones
                    query.append("<queryterm casesensitive=\"" + casesensitive
1644
                            + "\" " + "searchmode=\"" + searchmode
1645
                            + "\"><value>" + anyfield[i]
1646
                            + "</value></queryterm>");
1647
                }
1648
            }
1649 940 tao
        }
1650 2043 sgarg
1651 2075 jones
        //this while loop finds the rest of the parameters
1652
        //and attempts to query for the field specified
1653
        //by the parameter.
1654
        elements = params.elements();
1655
        keys = params.keys();
1656
        while (keys.hasMoreElements() && elements.hasMoreElements()) {
1657
            nextkey = keys.nextElement();
1658
            nextelement = elements.nextElement();
1659 2043 sgarg
1660 2075 jones
            //make sure we aren't querying for any of these
1661
            //parameters since the are already in the query
1662
            //in one form or another.
1663
            Vector ignoredParams = new Vector();
1664
            ignoredParams.add("returndoctype");
1665
            ignoredParams.add("filterdoctype");
1666
            ignoredParams.add("action");
1667
            ignoredParams.add("qformat");
1668
            ignoredParams.add("anyfield");
1669
            ignoredParams.add("returnfield");
1670
            ignoredParams.add("owner");
1671
            ignoredParams.add("site");
1672
            ignoredParams.add("operator");
1673 2091 tao
            ignoredParams.add("sessionid");
1674 3211 berkley
            ignoredParams.add("pagesize");
1675
            ignoredParams.add("pagestart");
1676 4135 berkley
            ignoredParams.add("searchmode");
1677 2043 sgarg
1678 2075 jones
            // Also ignore parameters listed in the properties file
1679
            // so that they can be passed through to stylesheets
1680 4080 daigle
            String paramsToIgnore = PropertyService
1681 4173 daigle
                    .getProperty("database.queryignoredparams");
1682 2075 jones
            StringTokenizer st = new StringTokenizer(paramsToIgnore, ",");
1683
            while (st.hasMoreTokens()) {
1684
                ignoredParams.add(st.nextToken());
1685
            }
1686
            if (!ignoredParams.contains(nextkey.toString())) {
1687
                //allow for more than value per field name
1688
                for (int i = 0; i < ((String[]) nextelement).length; i++) {
1689
                    if (!((String[]) nextelement)[i].equals("")) {
1690
                        query.append("<queryterm casesensitive=\""
1691
                                + casesensitive + "\" " + "searchmode=\""
1692 2087 tao
                                + searchmode + "\">" + "<value>" +
1693 2075 jones
                                //add the query value
1694
                                ((String[]) nextelement)[i]
1695 2087 tao
                                + "</value><pathexpr>" +
1696 2075 jones
                                //add the path to query by
1697
                                nextkey.toString() + "</pathexpr></queryterm>");
1698
                    }
1699
                }
1700
            }
1701
        }
1702
        query.append("</querygroup></pathquery>");
1703
        //append on the end of the xml and return the result as a string
1704
        return query.toString();
1705
    }
1706 2043 sgarg
1707 2075 jones
    /**
1708
     * format a simple free-text value query as an XML document that conforms
1709
     * to the pathquery.dtd and is appropriate for submission to the DBQuery
1710
     * structured query engine
1711 2087 tao
     *
1712 2075 jones
     * @param value the text string to search for in the xml catalog
1713
     * @param doctype the type of documents to include in the result set -- use
1714
     *            "any" or "ANY" for unfiltered result sets
1715
     */
1716
    public static String createQuery(String value, String doctype)
1717 1292 tao
    {
1718 2075 jones
        StringBuffer xmlquery = new StringBuffer();
1719
        xmlquery.append("<?xml version=\"1.0\"?>\n");
1720
        xmlquery.append("<pathquery version=\"1.0\">");
1721 2043 sgarg
1722 2075 jones
        if (!doctype.equals("any") && !doctype.equals("ANY")) {
1723
            xmlquery.append("<returndoctype>");
1724
            xmlquery.append(doctype).append("</returndoctype>");
1725
        }
1726 2043 sgarg
1727 2075 jones
        xmlquery.append("<querygroup operator=\"UNION\">");
1728
        //chad added - 8/14
1729
        //the if statement allows a query to gracefully handle a null
1730
        //query. Without this if a nullpointerException is thrown.
1731
        if (!value.equals("")) {
1732
            xmlquery.append("<queryterm casesensitive=\"false\" ");
1733
            xmlquery.append("searchmode=\"contains\">");
1734
            xmlquery.append("<value>").append(value).append("</value>");
1735
            xmlquery.append("</queryterm>");
1736 1217 tao
        }
1737 2075 jones
        xmlquery.append("</querygroup>");
1738
        xmlquery.append("</pathquery>");
1739 2043 sgarg
1740 2075 jones
        return (xmlquery.toString());
1741
    }
1742 2043 sgarg
1743 2075 jones
    /**
1744
     * format a simple free-text value query as an XML document that conforms
1745
     * to the pathquery.dtd and is appropriate for submission to the DBQuery
1746
     * structured query engine
1747 2087 tao
     *
1748 2075 jones
     * @param value the text string to search for in the xml catalog
1749
     */
1750
    public static String createQuery(String value)
1751 940 tao
    {
1752 2075 jones
        return createQuery(value, "any");
1753 940 tao
    }
1754 2043 sgarg
1755 2075 jones
    /**
1756
     * Check for "READ" permission on @docid for @user and/or @group from DB
1757
     * connection
1758
     */
1759
    private boolean hasPermission(String user, String[] groups, String docid)
1760
            throws SQLException, Exception
1761 940 tao
    {
1762 2075 jones
        // Check for READ permission on @docid for @user and/or @groups
1763
        PermissionController controller = new PermissionController(docid);
1764
        return controller.hasPermission(user, groups,
1765
                AccessControlInterface.READSTRING);
1766
    }
1767 2043 sgarg
1768 2075 jones
    /**
1769
     * Get all docIds list for a data packadge
1770 2087 tao
     *
1771 2075 jones
     * @param dataPackageDocid, the string in docId field of xml_relation table
1772
     */
1773
    private Vector getCurrentDocidListForDataPackage(String dataPackageDocid)
1774 940 tao
    {
1775 2075 jones
        DBConnection dbConn = null;
1776
        int serialNumber = -1;
1777
        Vector docIdList = new Vector();//return value
1778
        PreparedStatement pStmt = null;
1779
        ResultSet rs = null;
1780
        String docIdInSubjectField = null;
1781
        String docIdInObjectField = null;
1782 2043 sgarg
1783 2075 jones
        // Check the parameter
1784
        if (dataPackageDocid == null || dataPackageDocid.equals("")) { return docIdList; }//if
1785 940 tao
1786 2075 jones
        //the query stirng
1787
        String query = "SELECT subject, object from xml_relation where docId = ?";
1788
        try {
1789
            dbConn = DBConnectionPool
1790
                    .getDBConnection("DBQuery.getCurrentDocidListForDataPackage");
1791
            serialNumber = dbConn.getCheckOutSerialNumber();
1792
            pStmt = dbConn.prepareStatement(query);
1793
            //bind the value to query
1794
            pStmt.setString(1, dataPackageDocid);
1795 2043 sgarg
1796 2075 jones
            //excute the query
1797
            pStmt.execute();
1798
            //get the result set
1799
            rs = pStmt.getResultSet();
1800
            //process the result
1801
            while (rs.next()) {
1802
                //In order to get the whole docIds in a data packadge,
1803
                //we need to put the docIds of subject and object field in
1804
                // xml_relation
1805
                //into the return vector
1806
                docIdInSubjectField = rs.getString(1);//the result docId in
1807
                                                      // subject field
1808
                docIdInObjectField = rs.getString(2);//the result docId in
1809
                                                     // object field
1810 940 tao
1811 2075 jones
                //don't put the duplicate docId into the vector
1812
                if (!docIdList.contains(docIdInSubjectField)) {
1813
                    docIdList.add(docIdInSubjectField);
1814
                }
1815 2043 sgarg
1816 2075 jones
                //don't put the duplicate docId into the vector
1817
                if (!docIdList.contains(docIdInObjectField)) {
1818
                    docIdList.add(docIdInObjectField);
1819
                }
1820
            }//while
1821
            //close the pStmt
1822
            pStmt.close();
1823
        }//try
1824
        catch (SQLException e) {
1825 5165 daigle
            logMetacat.error("DBQuery.getCurrentDocidListForDataPackage - Error in getDocidListForDataPackage: "
1826 2663 sgarg
                    + e.getMessage());
1827 2075 jones
        }//catch
1828
        finally {
1829
            try {
1830
                pStmt.close();
1831
            }//try
1832
            catch (SQLException ee) {
1833 5165 daigle
                logMetacat.error("DBQuery.getCurrentDocidListForDataPackage - SQL Error: "
1834 2663 sgarg
                                + ee.getMessage());
1835 2075 jones
            }//catch
1836
            finally {
1837
                DBConnectionPool.returnDBConnection(dbConn, serialNumber);
1838
            }//fianlly
1839
        }//finally
1840
        return docIdList;
1841
    }//getCurrentDocidListForDataPackadge()
1842 2043 sgarg
1843 2075 jones
    /**
1844
     * Get all docIds list for a data packadge
1845 2087 tao
     *
1846 2075 jones
     * @param dataPackageDocid, the string in docId field of xml_relation table
1847
     */
1848 2641 tao
    private Vector getOldVersionDocidListForDataPackage(String dataPackageDocidWithRev)
1849 940 tao
    {
1850 2043 sgarg
1851 2075 jones
        Vector docIdList = new Vector();//return value
1852
        Vector tripleList = null;
1853
        String xml = null;
1854 2043 sgarg
1855 2075 jones
        // Check the parameter
1856 2641 tao
        if (dataPackageDocidWithRev == null || dataPackageDocidWithRev.equals("")) { return docIdList; }//if
1857 2043 sgarg
1858 2075 jones
        try {
1859
            //initial a documentImpl object
1860 2641 tao
            DocumentImpl packageDocument = new DocumentImpl(dataPackageDocidWithRev);
1861 2075 jones
            //transfer to documentImpl object to string
1862
            xml = packageDocument.toString();
1863 2043 sgarg
1864 2075 jones
            //create a tripcollection object
1865
            TripleCollection tripleForPackage = new TripleCollection(
1866
                    new StringReader(xml));
1867
            //get the vetor of triples
1868
            tripleList = tripleForPackage.getCollection();
1869 2043 sgarg
1870 2075 jones
            for (int i = 0; i < tripleList.size(); i++) {
1871
                //put subject docid into docIdlist without duplicate
1872
                if (!docIdList.contains(((Triple) tripleList.elementAt(i))
1873
                        .getSubject())) {
1874
                    //put subject docid into docIdlist
1875
                    docIdList.add(((Triple) tripleList.get(i)).getSubject());
1876
                }
1877
                //put object docid into docIdlist without duplicate
1878
                if (!docIdList.contains(((Triple) tripleList.elementAt(i))
1879
                        .getObject())) {
1880
                    docIdList.add(((Triple) (tripleList.get(i))).getObject());
1881
                }
1882
            }//for
1883
        }//try
1884
        catch (Exception e) {
1885 5165 daigle
            logMetacat.error("DBQuery.getCurrentDocidListForDataPackage - General error: "
1886 2663 sgarg
                    + e.getMessage());
1887 2075 jones
        }//catch
1888 2043 sgarg
1889 2075 jones
        // return result
1890
        return docIdList;
1891
    }//getDocidListForPackageInXMLRevisions()
1892 2043 sgarg
1893 2075 jones
    /**
1894
     * Check if the docId is a data packadge id. If the id is a data packadage
1895
     * id, it should be store in the docId fields in xml_relation table. So we
1896
     * can use a query to get the entries which the docId equals the given
1897
     * value. If the result is null. The docId is not a packadge id. Otherwise,
1898
     * it is.
1899 2087 tao
     *
1900 2075 jones
     * @param docId, the id need to be checked
1901
     */
1902
    private boolean isDataPackageId(String docId)
1903 940 tao
    {
1904 2075 jones
        boolean result = false;
1905
        PreparedStatement pStmt = null;
1906
        ResultSet rs = null;
1907
        String query = "SELECT docId from xml_relation where docId = ?";
1908
        DBConnection dbConn = null;
1909
        int serialNumber = -1;
1910
        try {
1911
            dbConn = DBConnectionPool
1912
                    .getDBConnection("DBQuery.isDataPackageId");
1913
            serialNumber = dbConn.getCheckOutSerialNumber();
1914
            pStmt = dbConn.prepareStatement(query);
1915
            //bind the value to query
1916
            pStmt.setString(1, docId);
1917
            //execute the query
1918
            pStmt.execute();
1919
            rs = pStmt.getResultSet();
1920
            //process the result
1921
            if (rs.next()) //There are some records for the id in docId fields
1922
            {
1923
                result = true;//It is a data packadge id
1924
            }
1925
            pStmt.close();
1926
        }//try
1927
        catch (SQLException e) {
1928 5165 daigle
            logMetacat.error("DBQuery.isDataPackageId - SQL Error: "
1929 2663 sgarg
                    + e.getMessage());
1930 2075 jones
        } finally {
1931
            try {
1932
                pStmt.close();
1933
            }//try
1934
            catch (SQLException ee) {
1935 5165 daigle
                logMetacat.error("DBQuery.isDataPackageId - SQL Error in isDataPackageId: "
1936 2663 sgarg
                        + ee.getMessage());
1937 2075 jones
            }//catch
1938
            finally {
1939
                DBConnectionPool.returnDBConnection(dbConn, serialNumber);
1940
            }//finally
1941
        }//finally
1942
        return result;
1943
    }//isDataPackageId()
1944 2043 sgarg
1945 6035 leinfelder
    public String getOperator() {
1946
		return operator;
1947
	}
1948
1949 2075 jones
    /**
1950 6035 leinfelder
     * Specifies if and how docid overrides should be included in the general query
1951
     * @param operator null, UNION, or INTERSECT (see QueryGroup)
1952
     */
1953
	public void setOperator(String operator) {
1954
		this.operator = operator;
1955
	}
1956
1957
	/**
1958 2075 jones
     * Check if the user has the permission to export data package
1959 2087 tao
     *
1960 2075 jones
     * @param conn, the connection
1961
     * @param docId, the id need to be checked
1962
     * @param user, the name of user
1963
     * @param groups, the user's group
1964
     */
1965
    private boolean hasPermissionToExportPackage(String docId, String user,
1966
            String[] groups) throws Exception
1967 940 tao
    {
1968 2075 jones
        //DocumentImpl doc=new DocumentImpl(conn,docId);
1969
        return DocumentImpl.hasReadPermission(user, groups, docId);
1970
    }
1971 2043 sgarg
1972 2075 jones
    /**
1973
     * Get the current Rev for a docid in xml_documents table
1974 2087 tao
     *
1975 2075 jones
     * @param docId, the id need to get version numb If the return value is -5,
1976
     *            means no value in rev field for this docid
1977
     */
1978
    private int getCurrentRevFromXMLDoumentsTable(String docId)
1979
            throws SQLException
1980
    {
1981
        int rev = -5;
1982
        PreparedStatement pStmt = null;
1983
        ResultSet rs = null;
1984
        String query = "SELECT rev from xml_documents where docId = ?";
1985
        DBConnection dbConn = null;
1986
        int serialNumber = -1;
1987
        try {
1988
            dbConn = DBConnectionPool
1989
                    .getDBConnection("DBQuery.getCurrentRevFromXMLDocumentsTable");
1990
            serialNumber = dbConn.getCheckOutSerialNumber();
1991
            pStmt = dbConn.prepareStatement(query);
1992
            //bind the value to query
1993
            pStmt.setString(1, docId);
1994
            //execute the query
1995
            pStmt.execute();
1996
            rs = pStmt.getResultSet();
1997
            //process the result
1998
            if (rs.next()) //There are some records for rev
1999
            {
2000
                rev = rs.getInt(1);
2001
                ;//It is the version for given docid
2002
            } else {
2003
                rev = -5;
2004
            }
2005 2043 sgarg
2006 1292 tao
        }//try
2007 2075 jones
        catch (SQLException e) {
2008 5165 daigle
            logMetacat.error("DBQuery.getCurrentRevFromXMLDoumentsTable - SQL Error: "
2009 2663 sgarg
                            + e.getMessage());
2010 2075 jones
            throw e;
2011 1292 tao
        }//catch
2012 2075 jones
        finally {
2013
            try {
2014
                pStmt.close();
2015
            }//try
2016
            catch (SQLException ee) {
2017 2663 sgarg
                logMetacat.error(
2018 5165 daigle
                        "DBQuery.getCurrentRevFromXMLDoumentsTable - SQL Error: "
2019 2663 sgarg
                                + ee.getMessage());
2020 2075 jones
            }//catch
2021
            finally {
2022
                DBConnectionPool.returnDBConnection(dbConn, serialNumber);
2023
            }//finally
2024
        }//finally
2025
        return rev;
2026
    }//getCurrentRevFromXMLDoumentsTable
2027 2043 sgarg
2028 2075 jones
    /**
2029
     * put a doc into a zip output stream
2030 2087 tao
     *
2031 2075 jones
     * @param docImpl, docmentImpl object which will be sent to zip output
2032
     *            stream
2033
     * @param zipOut, zip output stream which the docImpl will be put
2034
     * @param packageZipEntry, the zip entry name for whole package
2035
     */
2036
    private void addDocToZipOutputStream(DocumentImpl docImpl,
2037
            ZipOutputStream zipOut, String packageZipEntry)
2038
            throws ClassNotFoundException, IOException, SQLException,
2039
            McdbException, Exception
2040
    {
2041
        byte[] byteString = null;
2042
        ZipEntry zEntry = null;
2043 2043 sgarg
2044 5760 leinfelder
        byteString = docImpl.getBytes();
2045 2075 jones
        //use docId as the zip entry's name
2046
        zEntry = new ZipEntry(packageZipEntry + "/metadata/"
2047
                + docImpl.getDocID());
2048
        zEntry.setSize(byteString.length);
2049
        zipOut.putNextEntry(zEntry);
2050
        zipOut.write(byteString, 0, byteString.length);
2051
        zipOut.closeEntry();
2052 2043 sgarg
2053 2075 jones
    }//addDocToZipOutputStream()
2054 940 tao
2055 2075 jones
    /**
2056
     * Transfer a docid vetor to a documentImpl vector. The documentImpl vetor
2057
     * only inlcudes current version. If a DocumentImple object couldn't find
2058
     * for a docid, then the String of this docid was added to vetor rather
2059
     * than DocumentImple object.
2060 2087 tao
     *
2061 2075 jones
     * @param docIdList, a vetor hold a docid list for a data package. In
2062
     *            docid, there is not version number in it.
2063
     */
2064 2043 sgarg
2065 2075 jones
    private Vector getCurrentAllDocumentImpl(Vector docIdList)
2066
            throws McdbException, Exception
2067 940 tao
    {
2068 2075 jones
        //Connection dbConn=null;
2069
        Vector documentImplList = new Vector();
2070
        int rev = 0;
2071 2043 sgarg
2072 2075 jones
        // Check the parameter
2073
        if (docIdList.isEmpty()) { return documentImplList; }//if
2074 2043 sgarg
2075 2075 jones
        //for every docid in vector
2076
        for (int i = 0; i < docIdList.size(); i++) {
2077
            try {
2078
                //get newest version for this docId
2079
                rev = getCurrentRevFromXMLDoumentsTable((String) docIdList
2080
                        .elementAt(i));
2081 940 tao
2082 2075 jones
                // There is no record for this docId in xml_documents table
2083
                if (rev == -5) {
2084
                    // Rather than put DocumentImple object, put a String
2085
                    // Object(docid)
2086
                    // into the documentImplList
2087
                    documentImplList.add((String) docIdList.elementAt(i));
2088
                    // Skip other code
2089
                    continue;
2090
                }
2091 2043 sgarg
2092 2075 jones
                String docidPlusVersion = ((String) docIdList.elementAt(i))
2093 4212 daigle
                        + PropertyService.getProperty("document.accNumSeparator") + rev;
2094 2043 sgarg
2095 2075 jones
                //create new documentImpl object
2096
                DocumentImpl documentImplObject = new DocumentImpl(
2097
                        docidPlusVersion);
2098
                //add them to vector
2099
                documentImplList.add(documentImplObject);
2100
            }//try
2101
            catch (Exception e) {
2102 5165 daigle
                logMetacat.error("DBQuery.getCurrentAllDocumentImpl - General error: "
2103 2663 sgarg
                        + e.getMessage());
2104 2075 jones
                // continue the for loop
2105
                continue;
2106
            }
2107
        }//for
2108
        return documentImplList;
2109
    }
2110 2043 sgarg
2111 2075 jones
    /**
2112
     * Transfer a docid vetor to a documentImpl vector. If a DocumentImple
2113
     * object couldn't find for a docid, then the String of this docid was
2114
     * added to vetor rather than DocumentImple object.
2115 2087 tao
     *
2116 2075 jones
     * @param docIdList, a vetor hold a docid list for a data package. In
2117
     *            docid, t here is version number in it.
2118
     */
2119
    private Vector getOldVersionAllDocumentImpl(Vector docIdList)
2120
    {
2121
        //Connection dbConn=null;
2122
        Vector documentImplList = new Vector();
2123
        String siteCode = null;
2124
        String uniqueId = null;
2125
        int rev = 0;
2126 2043 sgarg
2127 2075 jones
        // Check the parameter
2128
        if (docIdList.isEmpty()) { return documentImplList; }//if
2129 2043 sgarg
2130 2075 jones
        //for every docid in vector
2131
        for (int i = 0; i < docIdList.size(); i++) {
2132 2043 sgarg
2133 2075 jones
            String docidPlusVersion = (String) (docIdList.elementAt(i));
2134
2135
            try {
2136
                //create new documentImpl object
2137
                DocumentImpl documentImplObject = new DocumentImpl(
2138
                        docidPlusVersion);
2139
                //add them to vector
2140
                documentImplList.add(documentImplObject);
2141
            }//try
2142
            catch (McdbDocNotFoundException notFoundE) {
2143 5165 daigle
                logMetacat.error("DBQuery.getOldVersionAllDocument - Error finding doc "
2144
                		+ docidPlusVersion + " : " + notFoundE.getMessage());
2145 2075 jones
                // Rather than add a DocumentImple object into vetor, a String
2146
                // object
2147
                // - the doicd was added to the vector
2148
                documentImplList.add(docidPlusVersion);
2149
                // Continue the for loop
2150
                continue;
2151
            }//catch
2152
            catch (Exception e) {
2153 2663 sgarg
                logMetacat.error(
2154 5165 daigle
                        "DBQuery.getOldVersionAllDocument - General error: "
2155 2663 sgarg
                                + e.getMessage());
2156 2075 jones
                // Continue the for loop
2157
                continue;
2158
            }//catch
2159
2160
        }//for
2161
        return documentImplList;
2162
    }//getOldVersionAllDocumentImple
2163
2164
    /**
2165
     * put a data file into a zip output stream
2166 2087 tao
     *
2167 2075 jones
     * @param docImpl, docmentImpl object which will be sent to zip output
2168
     *            stream
2169
     * @param zipOut, the zip output stream which the docImpl will be put
2170
     * @param packageZipEntry, the zip entry name for whole package
2171
     */
2172
    private void addDataFileToZipOutputStream(DocumentImpl docImpl,
2173
            ZipOutputStream zipOut, String packageZipEntry)
2174
            throws ClassNotFoundException, IOException, SQLException,
2175
            McdbException, Exception
2176 940 tao
    {
2177 2075 jones
        byte[] byteString = null;
2178
        ZipEntry zEntry = null;
2179
        // this is data file; add file to zip
2180 4080 daigle
        String filePath = PropertyService.getProperty("application.datafilepath");
2181 2075 jones
        if (!filePath.endsWith("/")) {
2182
            filePath += "/";
2183
        }
2184
        String fileName = filePath + docImpl.getDocID();
2185
        zEntry = new ZipEntry(packageZipEntry + "/data/" + docImpl.getDocID());
2186
        zipOut.putNextEntry(zEntry);
2187
        FileInputStream fin = null;
2188
        try {
2189
            fin = new FileInputStream(fileName);
2190
            byte[] buf = new byte[4 * 1024]; // 4K buffer
2191
            int b = fin.read(buf);
2192
            while (b != -1) {
2193
                zipOut.write(buf, 0, b);
2194
                b = fin.read(buf);
2195
            }//while
2196
            zipOut.closeEntry();
2197
        }//try
2198
        catch (IOException ioe) {
2199 5165 daigle
            logMetacat.error("DBQuery.addDataFileToZipOutputStream - I/O error: "
2200 2663 sgarg
                    + ioe.getMessage());
2201 2075 jones
        }//catch
2202
    }//addDataFileToZipOutputStream()
2203 2043 sgarg
2204 2075 jones
    /**
2205
     * create a html summary for data package and put it into zip output stream
2206 2087 tao
     *
2207 2075 jones
     * @param docImplList, the documentImpl ojbects in data package
2208
     * @param zipOut, the zip output stream which the html should be put
2209
     * @param packageZipEntry, the zip entry name for whole package
2210
     */
2211
    private void addHtmlSummaryToZipOutputStream(Vector docImplList,
2212
            ZipOutputStream zipOut, String packageZipEntry) throws Exception
2213
    {
2214
        StringBuffer htmlDoc = new StringBuffer();
2215
        ZipEntry zEntry = null;
2216
        byte[] byteString = null;
2217
        InputStream source;
2218
        DBTransform xmlToHtml;
2219 2043 sgarg
2220 2075 jones
        //create a DBTransform ojbect
2221
        xmlToHtml = new DBTransform();
2222
        //head of html
2223
        htmlDoc.append("<html><head></head><body>");
2224
        for (int i = 0; i < docImplList.size(); i++) {
2225
            // If this String object, this means it is missed data file
2226
            if ((((docImplList.elementAt(i)).getClass()).toString())
2227
                    .equals("class java.lang.String")) {
2228 2043 sgarg
2229 2075 jones
                htmlDoc.append("<a href=\"");
2230
                String dataFileid = (String) docImplList.elementAt(i);
2231
                htmlDoc.append("./data/").append(dataFileid).append("\">");
2232
                htmlDoc.append("Data File: ");
2233
                htmlDoc.append(dataFileid).append("</a><br>");
2234
                htmlDoc.append("<br><hr><br>");
2235 1356 tao
2236 2075 jones
            }//if
2237
            else if ((((DocumentImpl) docImplList.elementAt(i)).getDoctype())
2238
                    .compareTo("BIN") != 0) { //this is an xml file so we can
2239
                                              // transform it.
2240
                //transform each file individually then concatenate all of the
2241
                //transformations together.
2242 1356 tao
2243 2075 jones
                //for metadata xml title
2244
                htmlDoc.append("<h2>");
2245
                htmlDoc.append(((DocumentImpl) docImplList.elementAt(i))
2246
                        .getDocID());
2247
                //htmlDoc.append(".");
2248
                //htmlDoc.append(((DocumentImpl)docImplList.elementAt(i)).getRev());
2249
                htmlDoc.append("</h2>");
2250
                //do the actual transform
2251
                StringWriter docString = new StringWriter();
2252
                xmlToHtml.transformXMLDocument(((DocumentImpl) docImplList
2253
                        .elementAt(i)).toString(), "-//NCEAS//eml-generic//EN",
2254 5025 daigle
                        "-//W3C//HTML//EN", "html", docString, null, null);
2255 2075 jones
                htmlDoc.append(docString.toString());
2256
                htmlDoc.append("<br><br><hr><br><br>");
2257
            }//if
2258
            else { //this is a data file so we should link to it in the html
2259
                htmlDoc.append("<a href=\"");
2260
                String dataFileid = ((DocumentImpl) docImplList.elementAt(i))
2261
                        .getDocID();
2262
                htmlDoc.append("./data/").append(dataFileid).append("\">");
2263
                htmlDoc.append("Data File: ");
2264
                htmlDoc.append(dataFileid).append("</a><br>");
2265
                htmlDoc.append("<br><hr><br>");
2266
            }//else
2267
        }//for
2268
        htmlDoc.append("</body></html>");
2269 5760 leinfelder
        // use standard encoding even though the different docs might have use different encodings,
2270
        // the String objects in java should be correct and able to be encoded as the same Metacat default
2271
        byteString = htmlDoc.toString().getBytes(MetaCatServlet.DEFAULT_ENCODING);
2272 2075 jones
        zEntry = new ZipEntry(packageZipEntry + "/metadata.html");
2273
        zEntry.setSize(byteString.length);
2274
        zipOut.putNextEntry(zEntry);
2275
        zipOut.write(byteString, 0, byteString.length);
2276
        zipOut.closeEntry();
2277
        //dbConn.close();
2278 1356 tao
2279 2075 jones
    }//addHtmlSummaryToZipOutputStream
2280 1356 tao
2281 2075 jones
    /**
2282
     * put a data packadge into a zip output stream
2283 2087 tao
     *
2284 2641 tao
     * @param docId, which the user want to put into zip output stream,it has version
2285 2075 jones
     * @param out, a servletoutput stream which the zip output stream will be
2286
     *            put
2287
     * @param user, the username of the user
2288
     * @param groups, the group of the user
2289
     */
2290
    public ZipOutputStream getZippedPackage(String docIdString,
2291
            ServletOutputStream out, String user, String[] groups,
2292
            String passWord) throws ClassNotFoundException, IOException,
2293
            SQLException, McdbException, NumberFormatException, Exception
2294 945 tao
    {
2295 2075 jones
        ZipOutputStream zOut = null;
2296
        String elementDocid = null;
2297
        DocumentImpl docImpls = null;
2298
        //Connection dbConn = null;
2299
        Vector docIdList = new Vector();
2300
        Vector documentImplList = new Vector();
2301
        Vector htmlDocumentImplList = new Vector();
2302
        String packageId = null;
2303
        String rootName = "package";//the package zip entry name
2304 2043 sgarg
2305 2075 jones
        String docId = null;
2306
        int version = -5;
2307
        // Docid without revision
2308 5025 daigle
        docId = DocumentUtil.getDocIdFromString(docIdString);
2309 2075 jones
        // revision number
2310 5025 daigle
        version = DocumentUtil.getVersionFromString(docIdString);
2311 2043 sgarg
2312 2075 jones
        //check if the reqused docId is a data package id
2313
        if (!isDataPackageId(docId)) {
2314 2043 sgarg
2315 2075 jones
            /*
2316
             * Exception e = new Exception("The request the doc id "
2317
             * +docIdString+ " is not a data package id");
2318
             */
2319 940 tao
2320 2075 jones
            //CB 1/6/03: if the requested docid is not a datapackage, we just
2321
            // zip
2322
            //up the single document and return the zip file.
2323
            if (!hasPermissionToExportPackage(docId, user, groups)) {
2324 2043 sgarg
2325 2075 jones
                Exception e = new Exception("User " + user
2326
                        + " does not have permission"
2327
                        + " to export the data package " + docIdString);
2328
                throw e;
2329
            }
2330 2043 sgarg
2331 2641 tao
            docImpls = new DocumentImpl(docIdString);
2332 2075 jones
            //checking if the user has the permission to read the documents
2333
            if (DocumentImpl.hasReadPermission(user, groups, docImpls
2334
                    .getDocID())) {
2335
                zOut = new ZipOutputStream(out);
2336
                //if the docImpls is metadata
2337
                if ((docImpls.getDoctype()).compareTo("BIN") != 0) {
2338
                    //add metadata into zip output stream
2339
                    addDocToZipOutputStream(docImpls, zOut, rootName);
2340
                }//if
2341
                else {
2342
                    //it is data file
2343
                    addDataFileToZipOutputStream(docImpls, zOut, rootName);
2344
                    htmlDocumentImplList.add(docImpls);
2345
                }//else
2346 1292 tao
            }//if
2347 2043 sgarg
2348 2075 jones
            zOut.finish(); //terminate the zip file
2349
            return zOut;
2350
        }
2351
        // Check the permission of user
2352
        else if (!hasPermissionToExportPackage(docId, user, groups)) {
2353
2354
            Exception e = new Exception("User " + user
2355
                    + " does not have permission"
2356
                    + " to export the data package " + docIdString);
2357
            throw e;
2358
        } else //it is a packadge id
2359 1292 tao
        {
2360 2075 jones
            //store the package id
2361
            packageId = docId;
2362
            //get current version in database
2363
            int currentVersion = getCurrentRevFromXMLDoumentsTable(packageId);
2364
            //If it is for current version (-1 means user didn't specify
2365
            // revision)
2366
            if ((version == -1) || version == currentVersion) {
2367
                //get current version number
2368
                version = currentVersion;
2369
                //get package zip entry name
2370
                //it should be docId.revsion.package
2371 4212 daigle
                rootName = packageId + PropertyService.getProperty("document.accNumSeparator")
2372
                        + version + PropertyService.getProperty("document.accNumSeparator")
2373 2075 jones
                        + "package";
2374
                //get the whole id list for data packadge
2375
                docIdList = getCurrentDocidListForDataPackage(packageId);
2376
                //get the whole documentImple object
2377
                documentImplList = getCurrentAllDocumentImpl(docIdList);
2378 2043 sgarg
2379 1292 tao
            }//if
2380 2075 jones
            else if (version > currentVersion || version < -1) {
2381
                throw new Exception("The user specified docid: " + docId + "."
2382
                        + version + " doesn't exist");
2383
            }//else if
2384
            else //for an old version
2385 1292 tao
            {
2386 2075 jones
2387
                rootName = docIdString
2388 4212 daigle
                        + PropertyService.getProperty("document.accNumSeparator") + "package";
2389 2075 jones
                //get the whole id list for data packadge
2390
                docIdList = getOldVersionDocidListForDataPackage(docIdString);
2391
2392
                //get the whole documentImple object
2393
                documentImplList = getOldVersionAllDocumentImpl(docIdList);
2394 1292 tao
            }//else
2395 940 tao
2396 2075 jones
            // Make sure documentImplist is not empty
2397
            if (documentImplList.isEmpty()) { throw new Exception(
2398
                    "Couldn't find component for data package: " + packageId); }//if
2399 2043 sgarg
2400 2075 jones
            zOut = new ZipOutputStream(out);
2401
            //put every element into zip output stream
2402
            for (int i = 0; i < documentImplList.size(); i++) {
2403
                // if the object in the vetor is String, this means we couldn't
2404
                // find
2405
                // the document locally, we need find it remote
2406
                if ((((documentImplList.elementAt(i)).getClass()).toString())
2407
                        .equals("class java.lang.String")) {
2408
                    // Get String object from vetor
2409
                    String documentId = (String) documentImplList.elementAt(i);
2410 5165 daigle
                    logMetacat.info("DBQuery.getZippedPackage - docid: " + documentId);
2411 2075 jones
                    // Get doicd without revision
2412 5025 daigle
                    String docidWithoutRevision =
2413
                    	DocumentUtil.getDocIdFromString(documentId);
2414 5165 daigle
                    logMetacat.info("DBQuery.getZippedPackage - docidWithoutRevsion: "
2415 2663 sgarg
                            + docidWithoutRevision);
2416 2075 jones
                    // Get revision
2417 5025 daigle
                    String revision =
2418
                    	DocumentUtil.getRevisionStringFromString(documentId);
2419 5165 daigle
                    logMetacat.info("DBQuery.getZippedPackage - revision from docIdentifier: "
2420 2663 sgarg
                            + revision);
2421 2075 jones
                    // Zip entry string
2422
                    String zipEntryPath = rootName + "/data/";
2423
                    // Create a RemoteDocument object
2424
                    RemoteDocument remoteDoc = new RemoteDocument(
2425
                            docidWithoutRevision, revision, user, passWord,
2426
                            zipEntryPath);
2427
                    // Here we only read data file from remote metacat
2428
                    String docType = remoteDoc.getDocType();
2429
                    if (docType != null) {
2430
                        if (docType.equals("BIN")) {
2431
                            // Put remote document to zip output
2432
                            remoteDoc.readDocumentFromRemoteServerByZip(zOut);
2433
                            // Add String object to htmlDocumentImplList
2434
                            String elementInHtmlList = remoteDoc
2435
                                    .getDocIdWithoutRevsion()
2436 4212 daigle
                                    + PropertyService.getProperty("document.accNumSeparator")
2437 2075 jones
                                    + remoteDoc.getRevision();
2438
                            htmlDocumentImplList.add(elementInHtmlList);
2439
                        }//if
2440
                    }//if
2441 1361 tao
2442 2075 jones
                }//if
2443
                else {
2444
                    //create a docmentImpls object (represent xml doc) base on
2445
                    // the docId
2446
                    docImpls = (DocumentImpl) documentImplList.elementAt(i);
2447
                    //checking if the user has the permission to read the
2448
                    // documents
2449
                    if (DocumentImpl.hasReadPermission(user, groups, docImpls
2450
                            .getDocID())) {
2451
                        //if the docImpls is metadata
2452
                        if ((docImpls.getDoctype()).compareTo("BIN") != 0) {
2453
                            //add metadata into zip output stream
2454
                            addDocToZipOutputStream(docImpls, zOut, rootName);
2455
                            //add the documentImpl into the vetor which will
2456
                            // be used in html
2457
                            htmlDocumentImplList.add(docImpls);
2458 2043 sgarg
2459 2075 jones
                        }//if
2460
                        else {
2461
                            //it is data file
2462
                            addDataFileToZipOutputStream(docImpls, zOut,
2463
                                    rootName);
2464
                            htmlDocumentImplList.add(docImpls);
2465
                        }//else
2466
                    }//if
2467
                }//else
2468
            }//for
2469 2043 sgarg
2470 2075 jones
            //add html summary file
2471
            addHtmlSummaryToZipOutputStream(htmlDocumentImplList, zOut,
2472
                    rootName);
2473
            zOut.finish(); //terminate the zip file
2474
            //dbConn.close();
2475
            return zOut;
2476
        }//else
2477
    }//getZippedPackage()
2478 2043 sgarg
2479 2075 jones
    private class ReturnFieldValue
2480 1361 tao
    {
2481 2043 sgarg
2482 2075 jones
        private String docid = null; //return field value for this docid
2483 2043 sgarg
2484 2075 jones
        private String fieldValue = null;
2485 2043 sgarg
2486 2075 jones
        private String xmlFieldValue = null; //return field value in xml
2487
                                             // format
2488 3635 leinfelder
        private String fieldType = null; //ATTRIBUTE, TEXT...
2489 2075 jones
2490
        public void setDocid(String myDocid)
2491
        {
2492
            docid = myDocid;
2493
        }
2494
2495
        public String getDocid()
2496
        {
2497
            return docid;
2498
        }
2499
2500
        public void setFieldValue(String myValue)
2501
        {
2502
            fieldValue = myValue;
2503
        }
2504
2505
        public String getFieldValue()
2506
        {
2507
            return fieldValue;
2508
        }
2509
2510
        public void setXMLFieldValue(String xml)
2511
        {
2512
            xmlFieldValue = xml;
2513
        }
2514
2515
        public String getXMLFieldValue()
2516
        {
2517
            return xmlFieldValue;
2518
        }
2519 3635 leinfelder
2520
        public void setFieldType(String myType)
2521
        {
2522
            fieldType = myType;
2523
        }
2524 2075 jones
2525 3635 leinfelder
        public String getFieldType()
2526
        {
2527
            return fieldType;
2528
        }
2529
2530 1361 tao
    }
2531 3246 berkley
2532
    /**
2533
     * a class to store one result document consisting of a docid and a document
2534
     */
2535
    private class ResultDocument
2536
    {
2537
      public String docid;
2538
      public String document;
2539
2540
      public ResultDocument(String docid, String document)
2541
      {
2542
        this.docid = docid;
2543
        this.document = document;
2544
      }
2545
    }
2546
2547
    /**
2548
     * a private class to handle a set of resultDocuments
2549
     */
2550
    private class ResultDocumentSet
2551
    {
2552
      private Vector docids;
2553
      private Vector documents;
2554
2555
      public ResultDocumentSet()
2556
      {
2557
        docids = new Vector();
2558
        documents = new Vector();
2559
      }
2560
2561
      /**
2562
       * adds a result document to the set
2563
       */
2564
      public void addResultDocument(ResultDocument rd)
2565
      {
2566
        if(rd.docid == null)
2567 3263 tao
          return;
2568 3246 berkley
        if(rd.document == null)
2569
          rd.document = "";
2570 3349 tao
2571 3263 tao
           docids.addElement(rd.docid);
2572
           documents.addElement(rd.document);
2573 3349 tao
2574 3246 berkley
      }
2575
2576
      /**
2577
       * gets an iterator of docids
2578
       */
2579
      public Iterator getDocids()
2580
      {
2581
        return docids.iterator();
2582
      }
2583
2584
      /**
2585
       * gets an iterator of documents
2586
       */
2587
      public Iterator getDocuments()
2588
      {
2589
        return documents.iterator();
2590
      }
2591
2592
      /**
2593
       * returns the size of the set
2594
       */
2595
      public int size()
2596
      {
2597
        return docids.size();
2598
      }
2599
2600
      /**
2601
       * tests to see if this set contains the given docid
2602
       */
2603 3337 tao
      private boolean containsDocid(String docid)
2604 3246 berkley
      {
2605
        for(int i=0; i<docids.size(); i++)
2606
        {
2607
          String docid0 = (String)docids.elementAt(i);
2608
          if(docid0.trim().equals(docid.trim()))
2609
          {
2610
            return true;
2611
          }
2612
        }
2613
        return false;
2614
      }
2615
2616
      /**
2617
       * removes the element with the given docid
2618
       */
2619
      public String remove(String docid)
2620
      {
2621
        for(int i=0; i<docids.size(); i++)
2622
        {
2623
          String docid0 = (String)docids.elementAt(i);
2624
          if(docid0.trim().equals(docid.trim()))
2625
          {
2626
            String returnDoc = (String)documents.elementAt(i);
2627
            documents.remove(i);
2628
            docids.remove(i);
2629
            return returnDoc;
2630
          }
2631
        }
2632
        return null;
2633
      }
2634
2635
      /**
2636
       * add a result document
2637
       */
2638
      public void put(ResultDocument rd)
2639
      {
2640
        addResultDocument(rd);
2641
      }
2642
2643
      /**
2644
       * add a result document by components
2645
       */
2646
      public void put(String docid, String document)
2647
      {
2648
        addResultDocument(new ResultDocument(docid, document));
2649
      }
2650
2651
      /**
2652
       * get the document part of the result document by docid
2653
       */
2654
      public Object get(String docid)
2655
      {
2656
        for(int i=0; i<docids.size(); i++)
2657
        {
2658
          String docid0 = (String)docids.elementAt(i);
2659
          if(docid0.trim().equals(docid.trim()))
2660
          {
2661
            return documents.elementAt(i);
2662
          }
2663
        }
2664
        return null;
2665
      }
2666
2667
      /**
2668
       * get the document part of the result document by an object
2669
       */
2670
      public Object get(Object o)
2671
      {
2672
        return get((String)o);
2673
      }
2674
2675
      /**
2676
       * get an entire result document by index number
2677
       */
2678
      public ResultDocument get(int index)
2679
      {
2680
        return new ResultDocument((String)docids.elementAt(index),
2681
          (String)documents.elementAt(index));
2682
      }
2683
2684
      /**
2685
       * return a string representation of this object
2686
       */
2687
      public String toString()
2688
      {
2689
        String s = "";
2690
        for(int i=0; i<docids.size(); i++)
2691
        {
2692
          s += (String)docids.elementAt(i) + "\n";
2693
        }
2694
        return s;
2695
      }
2696 3263 tao
      /*
2697
       * Set a new document value for a given docid
2698
       */
2699
      public void set(String docid, String document)
2700
      {
2701
    	   for(int i=0; i<docids.size(); i++)
2702
           {
2703
             String docid0 = (String)docids.elementAt(i);
2704
             if(docid0.trim().equals(docid.trim()))
2705
             {
2706
                 documents.set(i, document);
2707
             }
2708
           }
2709
2710
      }
2711 3246 berkley
    }
2712 155 jones
}