Project

General

Profile

1 5298 jones
/**
2
 *  '$RCSfile$'
3
 *  Copyright: 2000 Regents of the University of California and the
4
 *              National Center for Ecological Analysis and Synthesis
5
 *
6 5299 jones
 *   '$Author: $'
7 5298 jones
 *     '$Date: 2009-06-13 15:28:13 +0300  $'
8
 *
9
 * This program is free software; you can redistribute it and/or modify
10
 * it under the terms of the GNU General Public License as published by
11
 * the Free Software Foundation; either version 2 of the License, or
12
 * (at your option) any later version.
13
 *
14
 * This program is distributed in the hope that it will be useful,
15
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 * GNU General Public License for more details.
18
 *
19
 * You should have received a copy of the GNU General Public License
20
 * along with this program; if not, write to the Free Software
21
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22
 */
23
package edu.ucsb.nceas.metacat.dataone;
24
25 5329 jones
import java.io.ByteArrayOutputStream;
26 5319 jones
import java.io.File;
27
import java.io.FileNotFoundException;
28
import java.io.FileOutputStream;
29 5298 jones
import java.io.IOException;
30
import java.io.InputStream;
31
import java.io.OutputStream;
32 5329 jones
import java.io.PrintWriter;
33 5378 berkley
import java.io.StringBufferInputStream;
34
import java.security.MessageDigest;
35 5298 jones
import java.sql.SQLException;
36 5366 berkley
import java.util.*;
37 5364 berkley
import java.text.DateFormat;
38 5420 berkley
import java.text.SimpleDateFormat;
39 5298 jones
40
import javax.servlet.ServletContext;
41
import javax.servlet.http.HttpServletRequest;
42
import javax.servlet.http.HttpServletResponse;
43
44 5319 jones
import org.apache.commons.io.IOUtils;
45
import org.apache.log4j.Logger;
46 5298 jones
import org.dataone.service.exceptions.IdentifierNotUnique;
47
import org.dataone.service.exceptions.InsufficientResources;
48
import org.dataone.service.exceptions.InvalidRequest;
49
import org.dataone.service.exceptions.InvalidSystemMetadata;
50
import org.dataone.service.exceptions.InvalidToken;
51
import org.dataone.service.exceptions.NotAuthorized;
52
import org.dataone.service.exceptions.NotFound;
53
import org.dataone.service.exceptions.NotImplemented;
54
import org.dataone.service.exceptions.ServiceFailure;
55
import org.dataone.service.exceptions.UnsupportedType;
56
import org.dataone.service.mn.MemberNodeCrud;
57 5364 berkley
import org.dataone.service.types.*;
58 5329 jones
import org.jibx.runtime.BindingDirectory;
59
import org.jibx.runtime.IBindingFactory;
60
import org.jibx.runtime.IMarshallingContext;
61 5332 jones
import org.jibx.runtime.IUnmarshallingContext;
62 5329 jones
import org.jibx.runtime.JiBXException;
63 5298 jones
64 5547 berkley
import org.dataone.service.types.Identifier;
65
66 5298 jones
import com.gc.iotools.stream.is.InputStreamFromOutputStream;
67
68 5319 jones
import edu.ucsb.nceas.metacat.AccessionNumberException;
69 5363 berkley
import edu.ucsb.nceas.metacat.MetacatResultSet;
70
import edu.ucsb.nceas.metacat.MetacatResultSet.Document;
71 5510 berkley
import edu.ucsb.nceas.metacat.DBQuery;
72 5319 jones
import edu.ucsb.nceas.metacat.DocumentImpl;
73
import edu.ucsb.nceas.metacat.EventLog;
74 5298 jones
import edu.ucsb.nceas.metacat.IdentifierManager;
75
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
76
import edu.ucsb.nceas.metacat.McdbException;
77
import edu.ucsb.nceas.metacat.MetacatHandler;
78
import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;
79 5370 berkley
import edu.ucsb.nceas.metacat.client.rest.MetacatRestClient;
80 5319 jones
import edu.ucsb.nceas.metacat.properties.PropertyService;
81
import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;
82 5298 jones
import edu.ucsb.nceas.metacat.service.SessionService;
83 5329 jones
import edu.ucsb.nceas.metacat.util.DocumentUtil;
84 5298 jones
import edu.ucsb.nceas.metacat.util.SessionData;
85
import edu.ucsb.nceas.utilities.ParseLSIDException;
86
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
87
88
/**
89
 *
90
 * Implements DataONE MemberNode CRUD API for Metacat.
91
 *
92
 * @author Matthew Jones
93
 */
94 5394 berkley
public class CrudService implements MemberNodeCrud
95
{
96 5337 berkley
    private static CrudService crudService = null;
97 5298 jones
98
    private MetacatHandler handler;
99
    private Hashtable<String, String[]> params;
100 5392 berkley
    private Logger logMetacat = null;
101
    private Logger logCrud = null;
102 5337 berkley
103
    private String metacatUrl;
104 5420 berkley
105 5428 berkley
    private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ssZ");
106 5298 jones
107
    /**
108 5337 berkley
     * singleton accessor
109
     */
110
    public static CrudService getInstance()
111
    {
112
      if(crudService == null)
113
      {
114
        crudService = new CrudService();
115
      }
116
117
      return crudService;
118
    }
119
120
    /**
121 5298 jones
     * Initializes new instance by setting servlet context,request and response.
122
     */
123 5337 berkley
    public CrudService() {
124 5333 berkley
    //change crud service into a singleton.  dont pass servlet data structures here
125 5337 berkley
        logMetacat = Logger.getLogger(CrudService.class);
126 5392 berkley
        logCrud = Logger.getLogger("DataOneLogger");
127 5337 berkley
        try
128
        {
129
            String server = PropertyService.getProperty("server.name");
130
            String port = PropertyService.getProperty("server.httpPort");
131
            String context = PropertyService.getProperty("application.context");
132
            metacatUrl = "http://" + server + ":" + port + "/" + context;
133
            logMetacat.debug("Initializing CrudService with url " + metacatUrl);
134
        }
135
        catch(Exception e)
136
        {
137
            logMetacat.error("Could not find servlet url in CrudService: " + e.getMessage());
138
            e.printStackTrace();
139
            throw new RuntimeException("Error getting servlet url in CrudService: " + e.getMessage());
140
        }
141
142
        /*this.servletContext = servletContext;
143 5298 jones
        this.request = request;
144 5337 berkley
        this.response = response;*/
145
146
        params = new Hashtable<String, String[]>();
147 5298 jones
148 5337 berkley
        handler = new MetacatHandler(new Timer());
149
150 5298 jones
    }
151 5337 berkley
152 5298 jones
    /**
153 5337 berkley
     * return the context url CrudService is using.
154 5298 jones
     */
155 5337 berkley
    public String getContextUrl()
156
    {
157
        return metacatUrl;
158
    }
159
160
    /**
161 5394 berkley
     * Set the context url that this service uses.  It is normally not necessary
162
     * to call this method unless you are trying to connect to a server other
163
     * than the one in which this service is installed.  Otherwise, this value is
164
     * taken from the metacat.properties file (server.name, server.port, application.context).
165
     */
166
    public void setContextUrl(String url)
167
    {
168
        metacatUrl = url;
169
    }
170
171
    /**
172 5337 berkley
     * set the params for this service from an HttpServletRequest param list
173
     */
174
    public void setParamsFromRequest(HttpServletRequest request)
175
    {
176 5298 jones
        Enumeration paramlist = request.getParameterNames();
177
        while (paramlist.hasMoreElements()) {
178 5337 berkley
            String name = (String) paramlist.nextElement();
179
            String[] value = (String[])request.getParameterValues(name);
180 5298 jones
            params.put(name, value);
181
        }
182
    }
183
184 5337 berkley
    /**
185 5370 berkley
     * Authenticate against metacat and get a token.
186
     * @param username
187
     * @param password
188
     * @return
189
     * @throws ServiceFailure
190
     */
191
    public AuthToken authenticate(String username, String password)
192
      throws ServiceFailure
193
    {
194 5466 berkley
        /* TODO:
195
         * This method is not in the original D1 crud spec.  It is highly
196
         * metacat centric.  Higher level decisions need to be made on authentication
197
         * interfaces for D1 nodes.
198
         */
199 5370 berkley
        try
200
        {
201
            MetacatRestClient restClient = new MetacatRestClient(getContextUrl());
202
            String response = restClient.login(username, password);
203
            String sessionid = restClient.getSessionId();
204
            SessionService sessionService = SessionService.getInstance();
205
            sessionService.registerSession(new SessionData(sessionid, username, new String[0], password, "CrudServiceLogin"));
206
            AuthToken token = new AuthToken(sessionid);
207 5383 berkley
            EventLog.getInstance().log(metacatUrl,
208 5384 berkley
                    username, null, "authenticate");
209 5436 berkley
            logCrud.info("authenticate");
210 5370 berkley
            return token;
211
        }
212
        catch(Exception e)
213
        {
214
            throw new ServiceFailure("1000", "Error authenticating with metacat: " + e.getMessage());
215
        }
216
    }
217
218
    /**
219 5337 berkley
     * set the parameter values needed for this request
220
     */
221
    public void setParameter(String name, String[] value)
222
    {
223
        params.put(name, value);
224
    }
225
226 5377 berkley
    /**
227
     * Generate SystemMetadata for any object in the object store that does
228
     * not already have it.  SystemMetadata documents themselves, are, of course,
229
     * exempt.  This is a utility method for migration of existing object
230
     * stores to DataONE where SystemMetadata is required for all objects.  See
231
     * https://trac.dataone.org/ticket/591
232 5378 berkley
     *
233
     * @param token an authtoken with appropriate permissions to read all
234
     * documents in the object store.  To work correctly, this should probably
235
     * be an adminstrative credential.
236 5377 berkley
     */
237 5378 berkley
    public void generateMissingSystemMetadata(AuthToken token)
238 5377 berkley
    {
239 5378 berkley
        IdentifierManager im = IdentifierManager.getInstance();
240 5377 berkley
        //get the list of ids with no SM
241 5378 berkley
        List<String> l = im.getLocalIdsWithNoSystemMetadata();
242 5377 berkley
        for(int i=0; i<l.size(); i++)
243
        { //for each id, add a system metadata doc
244 5378 berkley
            String localId = l.get(i);
245 5387 berkley
            //System.out.println("Creating SystemMetadata for localId " + localId);
246 5378 berkley
            //get the document
247
            try
248
            {
249
                //generate required system metadata fields from the document
250
                SystemMetadata sm = createSystemMetadata(localId, token);
251
                //insert the systemmetadata object
252 5379 berkley
                SessionData sessionData = getSessionData(token);
253
                insertSystemMetadata(sm, sessionData);
254 5431 berkley
                String username = "public";
255
                if(sessionData != null)
256
                {
257
                    username = sessionData.getUserName();
258
                }
259 5383 berkley
                EventLog.getInstance().log(metacatUrl,
260 5384 berkley
                        username, localId, "generateMissingSystemMetadata");
261 5378 berkley
            }
262
            catch(Exception e)
263
            {
264 5379 berkley
                //e.printStackTrace();
265 5378 berkley
                System.out.println("Exception generating missing system metadata: " + e.getMessage());
266
                logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
267
            }
268 5377 berkley
        }
269 5392 berkley
        logCrud.info("generateMissingSystemMetadata");
270 5377 berkley
    }
271
272 5378 berkley
    /**
273 5379 berkley
     * create an object via the crud interface
274 5378 berkley
     */
275 5320 jones
    public Identifier create(AuthToken token, Identifier guid,
276 5298 jones
            InputStream object, SystemMetadata sysmeta) throws InvalidToken,
277
            ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType,
278
            InsufficientResources, InvalidSystemMetadata, NotImplemented {
279 5319 jones
        logMetacat.debug("Starting CrudService.create()...");
280
281 5329 jones
        // authenticate & get user info
282
        SessionData sessionData = getSessionData(token);
283 5431 berkley
        String username = "public";
284
        String[] groups = null;
285
        if(sessionData != null)
286
        {
287
            username = sessionData.getUserName();
288
            groups = sessionData.getGroupNames();
289
        }
290 5383 berkley
        String localId = null;
291 5319 jones
292 5376 berkley
        if (username == null || username.equals("public"))
293
        {
294 5466 berkley
            //TODO: many of the thrown exceptions do not use the correct error codes
295
            //check these against the docs and correct them
296 5376 berkley
            throw new NotAuthorized("1000", "User " + username + " is not authorized to create content." +
297
                    "  If you are not logged in, please do so and retry the request.");
298
        }
299
300 5319 jones
        // verify that guid == SystemMetadata.getIdentifier()
301 5320 jones
        logMetacat.debug("Comparing guid|sysmeta_guid: " + guid.getValue() + "|" + sysmeta.getIdentifier().getValue());
302 5329 jones
        if (!guid.getValue().equals(sysmeta.getIdentifier().getValue())) {
303 5356 berkley
            throw new InvalidSystemMetadata("1180",
304 5329 jones
                "GUID in method call does not match GUID in system metadata.");
305
        }
306 5319 jones
307
        logMetacat.debug("Checking if identifier exists...");
308
        // Check that the identifier does not already exist
309 5436 berkley
        IdentifierManager im = IdentifierManager.getInstance();
310 5320 jones
        if (im.identifierExists(guid.getValue())) {
311 5356 berkley
            throw new IdentifierNotUnique("1120",
312 5319 jones
                "GUID is already in use by an existing object.");
313
        }
314
315 5329 jones
        // Check if we are handling metadata or data
316
        boolean isScienceMetadata = isScienceMetadata(sysmeta);
317 5319 jones
318 5466 berkley
        /* TODO:
319
         * For EML documents, the data url field needs to be checked here
320
         * and the describes and describedBy fields of the system metadata
321
         * need to be set with the appropriate uri.  Older EML documents
322
         * maybe have ecogrid:// uri's embedded in them.  See bug 618
323
         * (https://trac.dataone.org/ticket/618) for more info on this.
324
         */
325 5329 jones
        if (isScienceMetadata) {
326 5331 jones
            // CASE METADATA:
327
            try {
328 5466 berkley
                //System.out.println("CrudService: inserting document with guid " + guid.getValue());
329 5350 berkley
                this.insertDocument(object, guid, sessionData);
330 5383 berkley
                localId = im.getLocalId(guid.getValue());
331 5331 jones
            } catch (IOException e) {
332
                String msg = "Could not create string from XML stream: " +
333
                    " " + e.getMessage();
334
                logMetacat.debug(msg);
335 5356 berkley
                throw new ServiceFailure("1190", msg);
336 5383 berkley
            } catch(Exception e) {
337
                String msg = "Unexpected error in CrudService.create: " + e.getMessage();
338
                logMetacat.debug(msg);
339
                throw new ServiceFailure("1190", msg);
340 5331 jones
            }
341 5383 berkley
342 5319 jones
343 5329 jones
        } else {
344 5331 jones
            // DEFAULT CASE: DATA (needs to be checked and completed)
345 5460 berkley
            localId = insertDataObject(object, guid, sessionData);
346 5331 jones
347
        }
348 5319 jones
349 5331 jones
        // For Metadata and Data, insert the system metadata into the object store too
350 5441 berkley
        String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);
351
        //get the document info.  add any access params for the sysmeta too
352 5466 berkley
        //System.out.println("looking for access records to add for system " +
353
        //    "metadata who's parent doc's  local id is " + localId);
354 5441 berkley
        try
355
        {
356 5444 berkley
            Hashtable<String, Object> h = im.getDocumentInfo(localId.substring(0, localId.lastIndexOf(".")));
357 5441 berkley
            Vector v = (Vector)h.get("access");
358
            for(int i=0; i<v.size(); i++)
359
            {
360
                Hashtable ah = (Hashtable)v.elementAt(i);
361
                String principal = (String)ah.get("principal_name");
362
                String permission = (String)ah.get("permission");
363
                String permissionType = (String)ah.get("permission_type");
364
                String permissionOrder = (String)ah.get("permission_order");
365 5443 berkley
                int perm = new Integer(permission).intValue();
366 5466 berkley
                //System.out.println("found access record for principal " + principal);
367
                //System.out.println("permission: " + perm + " perm_type: " + permissionType +
368
                //    " perm_order: " + permissionOrder);
369 5444 berkley
                this.setAccess(token, guid, principal, perm, permissionType, permissionOrder, true);
370 5441 berkley
            }
371
        }
372
        catch(Exception e)
373
        {
374
            logMetacat.error("Error setting permissions on System Metadata object " +
375
                    " with id " + sysMetaLocalId + ": " + e.getMessage());
376 5466 berkley
            //TODO: decide if this error should cancel the entire create or
377
            //if it should continue with just a logged error.
378 5441 berkley
        }
379
380
381 5319 jones
        logMetacat.debug("Returning from CrudService.create()");
382 5383 berkley
        EventLog.getInstance().log(metacatUrl,
383 5384 berkley
                username, localId, "create");
384 5459 berkley
        logCrud.info("create D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId +
385
                ":D1SYSMETA:"+ sysMetaLocalId + ":");
386 5319 jones
        return guid;
387 5298 jones
    }
388 5353 berkley
389
    /**
390
     * update an existing object with a new object.  Change the system metadata
391
     * to reflect the changes and update it as well.
392
     */
393
    public Identifier update(AuthToken token, Identifier guid,
394
            InputStream object, Identifier obsoletedGuid, SystemMetadata sysmeta)
395
            throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique,
396
            UnsupportedType, InsufficientResources, NotFound, InvalidSystemMetadata,
397
            NotImplemented {
398
        try
399
        {
400
            SessionData sessionData = getSessionData(token);
401
402
            //find the old systemmetadata (sm.old) document id (the one linked to obsoletedGuid)
403
            SystemMetadata sm = getSystemMetadata(token, obsoletedGuid);
404
            //change sm.old's obsoletedBy field
405
            List l = sm.getObsoletedByList();
406
            l.add(guid);
407
            sm.setObsoletedByList(l);
408
            //update sm.old
409
            updateSystemMetadata(sm, sessionData);
410
411
            //change the obsoletes field of the new systemMetadata (sm.new) to point to the id of the old one
412
            sysmeta.addObsolete(obsoletedGuid);
413
            //insert sm.new
414 5459 berkley
            String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);
415
            String localId;
416 5353 berkley
417
            boolean isScienceMetadata = isScienceMetadata(sysmeta);
418
            if(isScienceMetadata)
419
            {
420
                //update the doc
421 5459 berkley
                localId = updateDocument(object, obsoletedGuid, guid, sessionData, false);
422 5353 berkley
            }
423
            else
424
            {
425
                //update a data file, not xml
426 5459 berkley
                localId = insertDataObject(object, guid, sessionData);
427 5353 berkley
            }
428 5383 berkley
429
            IdentifierManager im = IdentifierManager.getInstance();
430 5431 berkley
            String username = "public";
431
            if(sessionData != null)
432
            {
433
                username = sessionData.getUserName();
434
            }
435 5383 berkley
            EventLog.getInstance().log(metacatUrl,
436 5384 berkley
                    username, im.getLocalId(guid.getValue()), "update");
437 5459 berkley
            logCrud.info("update D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId +
438
                    ":D1SYSMETA:"+ sysMetaLocalId + ":");
439 5353 berkley
            return guid;
440
        }
441
        catch(Exception e)
442
        {
443 5356 berkley
            throw new ServiceFailure("1030", "Error updating document in CrudService: " + e.getMessage());
444 5353 berkley
        }
445
    }
446 5359 berkley
447
    /**
448 5521 berkley
     * set access permissions on both the science metadata and system metadata
449
     */
450
    public void setAccess(AuthToken token, Identifier id, String principal, String permission,
451
            String permissionType, String permissionOrder)
452
      throws ServiceFailure
453
    {
454
        setAccess(token, id, principal, permission, permissionType, permissionOrder, true);
455
    }
456
457
    /**
458 5443 berkley
     * set access control on the doc
459
     * @param token
460
     * @param id
461
     * @param principal
462
     * @param permission
463
     */
464
    public void setAccess(AuthToken token, Identifier id, String principal, int permission,
465
      String permissionType, String permissionOrder, boolean setSystemMetadata)
466
      throws ServiceFailure
467
    {
468
        String perm = "";
469 5445 berkley
        if(permission >= 4)
470 5443 berkley
        {
471
            perm = "read";
472
        }
473 5445 berkley
        if(permission >= 6)
474 5443 berkley
        {
475
            perm = "write";
476
        }
477 5466 berkley
        //System.out.println("perm in setAccess: " + perm);
478
        //System.out.println("permission in setAccess: " + permission);
479 5443 berkley
        setAccess(token, id, principal, perm, permissionType, permissionOrder,
480
                setSystemMetadata);
481
482
    }
483
484
    /**
485 5362 berkley
     * set the permission on the document
486
     * @param token
487
     * @param principal
488
     * @param permission
489
     * @param permissionType
490
     * @param permissionOrder
491
     * @return
492
     */
493
    public void setAccess(AuthToken token, Identifier id, String principal, String permission,
494 5424 berkley
            String permissionType, String permissionOrder, boolean setSystemMetadata)
495 5362 berkley
      throws ServiceFailure
496
    {
497 5466 berkley
        /* TODO:
498
         * This is also not part of the D1 Crud spec.  This method is needed for
499
         * systems such as metacat where access to objects is controlled by
500
         * and ACL.  Higher level decisions need to be made about how this
501
         * should work within D1.
502
         */
503 5362 berkley
        try
504
        {
505 5449 berkley
            final SessionData sessionData = getSessionData(token);
506
            if(sessionData == null)
507
            {
508
                throw new ServiceFailure("1000", "User must be logged in to set access.");
509
            }
510 5362 berkley
            IdentifierManager im = IdentifierManager.getInstance();
511
            String docid = im.getLocalId(id.getValue());
512 5449 berkley
513 5362 berkley
            String permNum = "0";
514 5370 berkley
            if(permission.equals("read"))
515 5362 berkley
            {
516
                permNum = "4";
517
            }
518 5370 berkley
            else if(permission.equals("write"))
519 5362 berkley
            {
520
                permNum = "6";
521
            }
522 5424 berkley
            System.out.println("user " + sessionData.getUserName() +
523
                    " is setting access level " + permNum + " for permission " +
524
                    permissionType + " on doc with localid " + docid);
525 5362 berkley
            handler.setAccess(metacatUrl, sessionData.getUserName(), docid,
526
                    principal, permNum, permissionType, permissionOrder);
527 5424 berkley
            if(setSystemMetadata)
528
            {
529
                //set the same perms on the system metadata doc
530
                String smlocalid = im.getSystemMetadataLocalId(id.getValue());
531
                System.out.println("setting access on SM doc with localid " + smlocalid);
532
                //cs.setAccess(token, smid, principal, permission, permissionType, permissionOrder);
533
                handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid,
534
                        principal, permNum, permissionType, permissionOrder);
535
            }
536 5431 berkley
            String username = "public";
537
            if(sessionData != null)
538
            {
539
                username = sessionData.getUserName();
540
            }
541 5383 berkley
            EventLog.getInstance().log(metacatUrl,
542 5384 berkley
                    username, im.getLocalId(id.getValue()), "setAccess");
543 5392 berkley
            logCrud.info("setAccess");
544 5362 berkley
        }
545
        catch(Exception e)
546
        {
547 5370 berkley
            e.printStackTrace();
548 5362 berkley
            throw new ServiceFailure("1000", "Could not set access on the document with id " + id.getValue());
549
        }
550
    }
551
552
    /**
553 5359 berkley
     *  Retrieve the list of objects present on the MN that match the calling
554
     *  parameters. This method is required to support the process of Member
555
     *  Node synchronization. At a minimum, this method should be able to
556
     *  return a list of objects that match:
557
     *  startTime <= SystemMetadata.dateSysMetadataModified
558
     *  but is expected to also support date range (by also specifying endTime),
559
     *  and should also support slicing of the matching set of records by
560
     *  indicating the starting index of the response (where 0 is the index
561
     *  of the first item) and the count of elements to be returned.
562
     *
563 5364 berkley
     *  If startTime or endTime is null, the query is not restricted by that parameter.
564
     *
565 5359 berkley
     * @see http://mule1.dataone.org/ArchitectureDocs/mn_api_replication.html#MN_replication.listObjects
566
     * @param token
567
     * @param startTime
568
     * @param endTime
569
     * @param objectFormat
570
     * @param replicaStatus
571
     * @param start
572
     * @param count
573
     * @return ObjectList
574
     * @throws NotAuthorized
575
     * @throws InvalidRequest
576
     * @throws NotImplemented
577
     * @throws ServiceFailure
578
     * @throws InvalidToken
579
     */
580
    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime,
581
        ObjectFormat objectFormat, boolean replicaStatus, int start, int count)
582
      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken
583
    {
584
      ObjectList ol = new ObjectList();
585 5362 berkley
      final SessionData sessionData = getSessionData(token);
586 5413 berkley
      int totalAfterQuery = 0;
587 5510 berkley
588 5362 berkley
      try
589
      {
590 5510 berkley
          if (PropertyService.getProperty("database.queryCacheOn").equals("true"))
591
          {
592
              //System.out.println("the string stored into cache is "+ resultsetBuffer.toString());
593
              DBQuery.clearQueryResultCache();
594
          }
595
      }
596
      catch (PropertyNotFoundException e1)
597
      {
598
          //just don't do anything
599
      }
600
601
      try
602
      {
603 5466 berkley
          //TODO: Milliseconds need to be added to the dateFormat
604 5430 berkley
          System.out.println("=========== Listing Objects =============");
605
          System.out.println("Current server time is: " + new Date());
606
          if(startTime != null)
607
          {
608
              System.out.println("query start time is " + startTime);
609
          }
610
          if(endTime != null)
611
          {
612
              System.out.println("query end time is " + endTime);
613
          }
614 5362 berkley
          params.clear();
615 5466 berkley
          /* TODO:
616
           * These metacat queries should not be hard coded here.  They
617
           * should be accessed from within a .properties file
618
           */
619 5362 berkley
          params.put("returndoctype", new String[] {"http://dataone.org/service/types/SystemMetadata/0.1"});
620
          params.put("qformat", new String[] {"xml"});
621 5364 berkley
          params.put("returnfield", new String[] {"size", "originMemberNode",
622 5387 berkley
                  "identifier", "objectFormat", "dateSysMetadataModified", "checksum", "checksum/@algorithm"});
623 5362 berkley
          params.put("anyfield", new String[] {"%"});
624 5363 berkley
625 5411 berkley
          //System.out.println("query is: metacatUrl: " + metacatUrl + " user: " + sessionData.getUserName() +
626
          //    " sessionid: " + sessionData.getId() + " params: " + params.toString());
627 5426 berkley
          String username = "public";
628
          String[] groups = null;
629
          String sessionid = "";
630
          if(sessionData != null)
631
          {
632
              username = sessionData.getUserName();
633
              groups = sessionData.getGroupNames();
634
              sessionid = sessionData.getId();
635
          }
636 5408 berkley
637 5426 berkley
          MetacatResultSet rs = handler.query(metacatUrl, params, username,
638
                  groups, sessionid);
639 5363 berkley
          List docs = rs.getDocuments();
640 5403 berkley
641 5510 berkley
          System.out.println("query returned " + docs.size() + " documents.");
642 5413 berkley
          Vector<Document> docCopy = new Vector<Document>();
643 5408 berkley
644 5413 berkley
          //preparse the list to remove any that don't match the query params
645 5466 berkley
          /* TODO: this type of query/subquery processing is probably not scalable
646
           * to larger object stores.  This code should be revisited.  The metacat
647
           * query handler should probably be altered to handle the type of query
648
           * done here.
649
           */
650 5413 berkley
          for(int i=0; i<docs.size(); i++)
651 5369 berkley
          {
652 5363 berkley
              Document d = (Document)docs.get(i);
653 5364 berkley
              ObjectFormat returnedObjectFormat = ObjectFormat.convert(d.getField("objectFormat"));
654 5413 berkley
655
              if(returnedObjectFormat != null &&
656
                 objectFormat != null &&
657
                 !objectFormat.toString().trim().equals(returnedObjectFormat.toString().trim()))
658 5364 berkley
              { //make sure the objectFormat is the one specified
659
                  continue;
660
              }
661 5384 berkley
662
              String dateSMM = d.getField("dateSysMetadataModified");
663 5413 berkley
              if((startTime != null || endTime != null) && dateSMM == null)
664
              {  //if startTime or endTime are not null, we need a date to compare to
665 5384 berkley
                  continue;
666
              }
667 5413 berkley
668
              //date parse
669
              Date dateSysMetadataModified = null;
670
              if(dateSMM != null)
671
              {
672 5420 berkley
                  //dateSysMetadataModified = parseDate(dateSMM);
673
                  if(dateSMM.indexOf(".") != -1)
674
                  {  //strip the milliseconds
675 5466 berkley
                      //TODO: don't do this. we need milliseconds now.
676 5481 berkley
                      //TODO: explore ISO 8601 to figure out milliseconds
677 5420 berkley
                      dateSMM = dateSMM.substring(0, dateSMM.indexOf(".")) + 'Z';
678
                  }
679
                  //System.out.println("dateSMM: " + dateSMM);
680 5428 berkley
                  //dateFormat.setTimeZone(TimeZone.getTimeZone("GMT-0"));
681
                  try
682
                  {   //the format we want
683
                      dateSysMetadataModified = dateFormat.parse(dateSMM);
684
                  }
685
                  catch(java.text.ParseException pe)
686
                  {   //try another legacy format
687
                      DateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'");
688
                      dateFormat2.setTimeZone(TimeZone.getTimeZone("GMT-0"));
689
                      dateSysMetadataModified = dateFormat2.parse(dateSMM);
690
                  }
691 5413 berkley
              }
692 5428 berkley
693 5510 berkley
              /*System.out.println("====================================");
694
              System.out.println("doc number " + i);
695
              System.out.println("docid: " + d.docid);
696 5420 berkley
              System.out.println("dateSysMetadataModified: " + dateSysMetadataModified);
697
              System.out.println("startTime: " + startTime);
698
              System.out.println("endtime: " + endTime);*/
699 5427 berkley
700 5364 berkley
              int startDateComparison = 0;
701
              int endDateComparison = 0;
702
              if(startTime != null)
703
              {
704 5427 berkley
                  Calendar zTime = Calendar.getInstance(TimeZone.getTimeZone("GMT-0"));
705
                  zTime.setTime(startTime);
706
                  startTime = zTime.getTime();
707
708 5413 berkley
                  if(dateSysMetadataModified == null)
709
                  {
710
                      startDateComparison = -1;
711
                  }
712
                  else
713
                  {
714
                      startDateComparison = dateSysMetadataModified.compareTo(startTime);
715
                  }
716
                  //System.out.println("startDateCom: " + startDateComparison);
717 5364 berkley
              }
718 5413 berkley
              else
719
              {
720
                  startDateComparison = 1;
721
              }
722 5364 berkley
723
              if(endTime != null)
724
              {
725 5427 berkley
                  Calendar zTime = Calendar.getInstance(TimeZone.getTimeZone("GMT-0"));
726
                  zTime.setTime(endTime);
727
                  endTime = zTime.getTime();
728
729 5413 berkley
                  if(dateSysMetadataModified == null)
730
                  {
731
                      endDateComparison = 1;
732
                  }
733
                  else
734
                  {
735
                      endDateComparison = dateSysMetadataModified.compareTo(endTime);
736
                  }
737
                  //System.out.println("endDateCom: " + endDateComparison);
738 5364 berkley
              }
739 5413 berkley
              else
740
              {
741
                  endDateComparison = -1;
742
              }
743 5364 berkley
744 5413 berkley
745 5364 berkley
              if(startDateComparison < 0 || endDateComparison > 0)
746 5413 berkley
              {
747 5364 berkley
                  continue;
748
              }
749
750 5413 berkley
              docCopy.add((Document)docs.get(i));
751
          } //end pre-parse
752
753
          docs = docCopy;
754
          totalAfterQuery = docs.size();
755 5428 berkley
          //System.out.println("total after subquery: " + totalAfterQuery);
756 5413 berkley
757
          //make sure we don't run over the end
758
          int end = start + count;
759
          if(end > docs.size())
760
          {
761
              end = docs.size();
762
          }
763
764
          for(int i=start; i<end; i++)
765
          {
766
              //get the document from the result
767
              Document d = (Document)docs.get(i);
768
              //System.out.println("processing doc " + d.docid);
769
770
              String dateSMM = d.getField("dateSysMetadataModified");
771
              Date dateSysMetadataModified = null;
772
              if(dateSMM != null)
773
              {
774
                  try
775
                  {
776
                      dateSysMetadataModified = parseDate(dateSMM);
777
                  }
778
                  catch(Exception e)
779
                  { //if we fail to parse the date, just ignore the value
780
                      dateSysMetadataModified = null;
781
                  }
782
              }
783
              ObjectFormat returnedObjectFormat = ObjectFormat.convert(d.getField("objectFormat"));
784
785
786 5364 berkley
              ObjectInfo info = new ObjectInfo();
787
              //add the fields to the info object
788
              Checksum cs = new Checksum();
789
              cs.setValue(d.getField("checksum"));
790 5370 berkley
              String csalg = d.getField("algorithm");
791
              if(csalg == null)
792
              {
793
                  csalg = "MD5";
794
              }
795
              ChecksumAlgorithm ca = ChecksumAlgorithm.convert(csalg);
796
              cs.setAlgorithm(ca);
797 5364 berkley
              info.setChecksum(cs);
798
              info.setDateSysMetadataModified(dateSysMetadataModified);
799
              Identifier id = new Identifier();
800 5445 berkley
              id.setValue(d.getField("identifier").trim());
801 5364 berkley
              info.setIdentifier(id);
802
              info.setObjectFormat(returnedObjectFormat);
803 5413 berkley
              String size = d.getField("size");
804
              if(size != null)
805
              {
806
                  info.setSize(new Long(size.trim()).longValue());
807
              }
808 5364 berkley
              //add the ObjectInfo to the ObjectList
809 5557 berkley
              //logCrud.info("objectFormat: " + info.getObjectFormat().toString());
810
              //logCrud.info("id: " + info.getIdentifier().getValue());
811
812 5555 berkley
              if(info.getIdentifier().getValue() != null)
813 5414 berkley
              { //id can be null from tests.  should not happen in production.
814 5555 berkley
                  if((info.getObjectFormat() != null && !info.getObjectFormat().toString().trim().equals("")))
815
                  { //objectFormat needs to not be null and not be an empty string
816
                    ol.addObjectInfo(info);
817
                  }
818 5557 berkley
                  else
819
                  {
820
                      logCrud.info("Not adding object " + info.getIdentifier().toString());
821
                  }
822 5414 berkley
              }
823 5424 berkley
824 5363 berkley
          }
825 5362 berkley
      }
826
      catch(Exception e)
827
      {
828
          e.printStackTrace();
829 5557 berkley
          logCrud.error("Error creating ObjectList: " + e.getMessage() + " cause: " + e.getCause());
830 5362 berkley
          throw new ServiceFailure("1580", "Error retrieving ObjectList: " + e.getMessage());
831
      }
832 5430 berkley
      String username = "public";
833
      if(sessionData != null)
834
      {
835
          username = sessionData.getUserName();
836
      }
837 5383 berkley
      EventLog.getInstance().log(metacatUrl,
838 5384 berkley
              username, null, "read");
839 5392 berkley
      logCrud.info("listObjects");
840 5413 berkley
      ol.setCount(count);
841
      ol.setStart(start);
842
      ol.setTotal(totalAfterQuery);
843 5359 berkley
      return ol;
844
    }
845
846
    /**
847
     * Call listObjects with the default values for replicaStatus (true), start (0),
848
     * and count (1000).
849
     * @param token
850
     * @param startTime
851
     * @param endTime
852
     * @param objectFormat
853
     * @return
854
     * @throws NotAuthorized
855
     * @throws InvalidRequest
856
     * @throws NotImplemented
857
     * @throws ServiceFailure
858
     * @throws InvalidToken
859
     */
860
    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime,
861
        ObjectFormat objectFormat)
862
      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken
863
    {
864
       return listObjects(token, startTime, endTime, objectFormat, true, 0, 1000);
865
    }
866 5298 jones
867 5359 berkley
    /**
868
     * Delete a document.  NOT IMPLEMENTED
869
     */
870 5320 jones
    public Identifier delete(AuthToken token, Identifier guid)
871 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
872
            NotImplemented {
873 5392 berkley
        logCrud.info("delete");
874 5356 berkley
        throw new NotImplemented("1000", "This method not yet implemented.");
875 5298 jones
    }
876
877 5359 berkley
    /**
878
     * describe a document.  NOT IMPLEMENTED
879
     */
880 5320 jones
    public DescribeResponse describe(AuthToken token, Identifier guid)
881 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
882
            NotImplemented {
883 5392 berkley
        logCrud.info("describe");
884 5356 berkley
        throw new NotImplemented("1000", "This method not yet implemented.");
885 5298 jones
    }
886
887 5359 berkley
    /**
888
     * get a document with a specified guid.
889
     */
890 5320 jones
    public InputStream get(AuthToken token, Identifier guid)
891 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
892
            NotImplemented {
893
894
        // Retrieve the session information from the AuthToken
895
        // If the session is expired, then the user is 'public'
896 5436 berkley
        if(token == null)
897
        {
898
            token = new AuthToken("Public");
899
        }
900 5298 jones
        final SessionData sessionData = getSessionData(token);
901
902
        // Look up the localId for this global identifier
903
        IdentifierManager im = IdentifierManager.getInstance();
904
        try {
905 5320 jones
            final String localId = im.getLocalId(guid.getValue());
906 5298 jones
907
            final InputStreamFromOutputStream<String> objectStream =
908
                new InputStreamFromOutputStream<String>() {
909
910
                @Override
911
                public String produce(final OutputStream dataSink) throws Exception {
912
913
                    try {
914 5439 berkley
                        String username = "public";
915
                        String[] groups = new String[0];
916
                        if(sessionData != null)
917
                        {
918
                            username = sessionData.getUserName();
919
                            groups = sessionData.getGroupNames();
920
                        }
921
                        /*System.out.println("metacatUrl: " + metacatUrl +
922
                            " dataSink: " + dataSink + " localId: " + localId +
923
                            " username: " + username + " params: " + params.toString());
924
                        */
925 5466 berkley
                        /* TODO:
926
                         * This multithreaded approach to getting data without
927
                         * being memory bound causes problems with exception
928
                         * handling.  The only exception the produce method
929
                         * will return is an IOException, rendering all of the
930
                         * catch blocks below mute.  This should probably be changed
931
                         * to use a memory mapped solution instead so that we
932
                         * can properly pass exceptions to the client.
933
                         * see https://trac.dataone.org/ticket/706
934
                         */
935 5344 berkley
                        handler.readFromMetacat(metacatUrl, null,
936 5298 jones
                                dataSink, localId, "xml",
937 5439 berkley
                                username,
938
                                groups, true, params);
939 5298 jones
                    } catch (PropertyNotFoundException e) {
940 5382 berkley
                        e.printStackTrace();
941
                        throw new ServiceFailure("1030", "Error getting property from metacat: " + e.getMessage());
942 5298 jones
                    } catch (ClassNotFoundException e) {
943 5382 berkley
                        e.printStackTrace();
944
                        throw new ServiceFailure("1030", "Class not found error when reading from metacat: " + e.getMessage());
945 5298 jones
                    } catch (IOException e) {
946 5382 berkley
                        e.printStackTrace();
947
                        throw new ServiceFailure("1030", "IOException while reading from metacat: " + e.getMessage());
948 5298 jones
                    } catch (SQLException e) {
949 5382 berkley
                        e.printStackTrace();
950
                        throw new ServiceFailure("1030", "SQLException while reading from metacat: " + e.getMessage());
951 5298 jones
                    } catch (McdbException e) {
952 5382 berkley
                        e.printStackTrace();
953
                        throw new ServiceFailure("1030", "Metacat DB exception while reading from metacat: " + e.getMessage());
954 5298 jones
                    } catch (ParseLSIDException e) {
955 5382 berkley
                        e.printStackTrace();
956
                        throw new NotFound("1020", "LSID parsing exception while reading from metacat: " + e.getMessage());
957 5298 jones
                    } catch (InsufficientKarmaException e) {
958 5382 berkley
                        e.printStackTrace();
959
                        throw new NotAuthorized("1000", "User not authorized for get(): " + e.getMessage());
960 5298 jones
                    }
961
962
                    return "Completed";
963
                }
964
            };
965 5431 berkley
            String username = "public";
966
            if(sessionData != null)
967
            {
968
                username = sessionData.getUserName();
969
            }
970
971 5383 berkley
            EventLog.getInstance().log(metacatUrl,
972 5384 berkley
                    username, im.getLocalId(guid.getValue()), "read");
973 5459 berkley
            logCrud.info("get D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId +
974
                    ":");
975 5298 jones
            return objectStream;
976
977
        } catch (McdbDocNotFoundException e) {
978 5356 berkley
            throw new NotFound("1020", e.getMessage());
979 5298 jones
        }
980
    }
981
982 5359 berkley
    /**
983
     * get the checksum for a document.  NOT IMPLEMENTED
984
     */
985 5320 jones
    public Checksum getChecksum(AuthToken token, Identifier guid)
986 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
987
            InvalidRequest, NotImplemented {
988 5392 berkley
        logCrud.info("getChecksum");
989 5466 berkley
        //TODO: this could now easily be implemented
990 5356 berkley
        throw new NotImplemented("1000", "This method not yet implemented.");
991 5298 jones
    }
992
993 5359 berkley
    /**
994
     * get the checksum for a document.  NOT IMPLEMENTED
995
     */
996 5320 jones
    public Checksum getChecksum(AuthToken token, Identifier guid,
997 5298 jones
            String checksumAlgorithm) throws InvalidToken, ServiceFailure,
998
            NotAuthorized, NotFound, InvalidRequest, NotImplemented {
999 5392 berkley
        logCrud.info("getChecksum");
1000 5466 berkley
        //TODO: this could now easily be implemented
1001 5356 berkley
        throw new NotImplemented("1000", "This method not yet implemented.");
1002 5298 jones
    }
1003
1004 5359 berkley
    /**
1005 5384 berkley
     * get log records.
1006 5359 berkley
     */
1007 5390 berkley
    public Log getLogRecords(AuthToken token, Date fromDate, Date toDate, Event event)
1008 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest,
1009 5384 berkley
            NotImplemented
1010
    {
1011 5430 berkley
        System.out.println("=================== Getting log records ===================");
1012
        System.out.println("Current server time is: " + new Date());
1013
        if(fromDate != null)
1014
        {
1015
          System.out.println("query start time is " + fromDate);
1016
        }
1017
        if(toDate != null)
1018
        {
1019
          System.out.println("query end time is " + toDate);
1020
        }
1021 5389 berkley
        Log log = new Log();
1022
        Vector<LogEntry> logs = new Vector<LogEntry>();
1023 5384 berkley
        IdentifierManager im = IdentifierManager.getInstance();
1024
        EventLog el = EventLog.getInstance();
1025 5391 berkley
        if(fromDate == null)
1026
        {
1027 5418 berkley
            System.out.println("setting fromdate from null");
1028 5391 berkley
            fromDate = new Date(1);
1029
        }
1030
        if(toDate == null)
1031
        {
1032 5418 berkley
            System.out.println("setting todate from null");
1033 5391 berkley
            toDate = new Date();
1034
        }
1035 5418 berkley
1036
        System.out.println("fromDate: " + fromDate);
1037
        System.out.println("toDate: " + toDate);
1038
1039 5384 berkley
        String report = el.getReport(null, null, null, null,
1040
                new java.sql.Timestamp(fromDate.getTime()),
1041
                new java.sql.Timestamp(toDate.getTime()));
1042
1043 5418 berkley
        //System.out.println("report: " + report);
1044
1045 5384 berkley
        String logEntry = "<logEntry>";
1046
        String endLogEntry = "</logEntry>";
1047
        int startIndex = 0;
1048
        int foundIndex = report.indexOf(logEntry, startIndex);
1049
        while(foundIndex != -1)
1050
        {
1051
            //parse out each entry
1052
            int endEntryIndex = report.indexOf(endLogEntry, foundIndex);
1053
            String entry = report.substring(foundIndex, endEntryIndex);
1054
            //System.out.println("entry: " + entry);
1055
            startIndex = endEntryIndex + endLogEntry.length();
1056
            foundIndex = report.indexOf(logEntry, startIndex);
1057
1058
            String entryId = getLogEntryField("entryid", entry);
1059
            String ipAddress = getLogEntryField("ipAddress", entry);
1060
            String principal = getLogEntryField("principal", entry);
1061
            String docid = getLogEntryField("docid", entry);
1062 5390 berkley
            String eventS = getLogEntryField("event", entry);
1063 5384 berkley
            String dateLogged = getLogEntryField("dateLogged", entry);
1064
1065 5389 berkley
            LogEntry le = new LogEntry();
1066 5384 berkley
1067 5390 berkley
            Event e = Event.convert(eventS);
1068 5384 berkley
            if(e == null)
1069
            { //skip any events that are not Dataone Crud events
1070
                continue;
1071
            }
1072 5389 berkley
            le.setEvent(e);
1073 5384 berkley
            Identifier entryid = new Identifier();
1074
            entryid.setValue(entryId);
1075 5389 berkley
            le.setEntryId(entryid);
1076 5384 berkley
            Identifier identifier = new Identifier();
1077
            try
1078
            {
1079 5432 berkley
                //System.out.println("converting docid '" + docid + "' to a guid.");
1080 5417 berkley
                if(docid == null || docid.trim().equals("") || docid.trim().equals("null"))
1081
                {
1082
                    continue;
1083
                }
1084 5416 berkley
                docid = docid.substring(0, docid.lastIndexOf("."));
1085 5384 berkley
                identifier.setValue(im.getGUID(docid, im.getLatestRevForLocalId(docid)));
1086
            }
1087
            catch(Exception ex)
1088
            { //try to get the guid, if that doesn't work, just use the local id
1089 5419 berkley
                //throw new ServiceFailure("1030", "Error getting guid for localId " +
1090 5424 berkley
                //        docid + ": " + ex.getMessage());\
1091
1092
                //skip it if the guid can't be found
1093 5419 berkley
                continue;
1094 5384 berkley
            }
1095
1096 5389 berkley
            le.setIdentifier(identifier);
1097
            le.setIpAddress(ipAddress);
1098 5384 berkley
            Calendar c = Calendar.getInstance();
1099
            String year = dateLogged.substring(0, 4);
1100
            String month = dateLogged.substring(5, 7);
1101
            String date = dateLogged.substring(8, 10);
1102
            //System.out.println("year: " + year + " month: " + month + " day: " + date);
1103
            c.set(new Integer(year).intValue(), new Integer(month).intValue(), new Integer(date).intValue());
1104
            Date logDate = c.getTime();
1105 5389 berkley
            le.setDateLogged(logDate);
1106 5384 berkley
            NodeReference memberNode = new NodeReference();
1107
            memberNode.setValue(ipAddress);
1108 5389 berkley
            le.setMemberNode(memberNode);
1109 5384 berkley
            Principal princ = new Principal();
1110
            princ.setValue(principal);
1111 5389 berkley
            le.setPrincipal(princ);
1112
            le.setUserAgent("metacat/RESTService");
1113 5390 berkley
1114
            if(event == null)
1115
            {
1116
                logs.add(le);
1117
            }
1118
1119
            if(event != null &&
1120
               e.toString().toLowerCase().trim().equals(event.toString().toLowerCase().trim()))
1121
            {
1122
              logs.add(le);
1123
            }
1124 5384 berkley
        }
1125
1126 5389 berkley
        log.setLogEntryList(logs);
1127 5392 berkley
        logCrud.info("getLogRecords");
1128 5389 berkley
        return log;
1129 5298 jones
    }
1130 5384 berkley
1131
    /**
1132
     * parse a logEntry and get the relavent field from it
1133
     * @param fieldname
1134
     * @param entry
1135
     * @return
1136
     */
1137
    private String getLogEntryField(String fieldname, String entry)
1138
    {
1139
        String begin = "<" + fieldname + ">";
1140
        String end = "</" + fieldname + ">";
1141
        //System.out.println("looking for " + begin + " and " + end + " in entry " + entry);
1142
        String s = entry.substring(entry.indexOf(begin) + begin.length(), entry.indexOf(end));
1143
        //System.out.println("entry " + fieldname + " : " + s);
1144
        return s;
1145
    }
1146 5298 jones
1147 5359 berkley
    /**
1148
     * get the system metadata for a document with a specified guid.
1149
     */
1150 5481 berkley
public SystemMetadata getSystemMetadata(AuthToken token, Identifier guid)
1151 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
1152
            InvalidRequest, NotImplemented {
1153 5319 jones
1154 5332 jones
        logMetacat.debug("CrudService.getSystemMetadata - for guid: " + guid.getValue());
1155
1156
        // Retrieve the session information from the AuthToken
1157
        // If the session is expired, then the user is 'public'
1158
        final SessionData sessionData = getSessionData(token);
1159
1160
        try {
1161
            IdentifierManager im = IdentifierManager.getInstance();
1162 5424 berkley
            final String localId = im.getSystemMetadataLocalId(guid.getValue());
1163 5332 jones
1164
            // Read system metadata from metacat's db
1165
            final InputStreamFromOutputStream<String> objectStream =
1166
                new InputStreamFromOutputStream<String>() {
1167
1168
                @Override
1169
                public String produce(final OutputStream dataSink) throws Exception {
1170 5481 berkley
                    //TODO: change to memory mapped IO so that exceptions get
1171
                    //passed to the response correctly.
1172 5332 jones
                    try {
1173 5449 berkley
                        String username = "public";
1174
                        String[] groupnames = null;
1175
                        if(sessionData != null)
1176
                        {
1177
                            username = sessionData.getUserName();
1178
                            groupnames = sessionData.getGroupNames();
1179
                        }
1180
1181 5344 berkley
                        handler.readFromMetacat(metacatUrl, null,
1182 5332 jones
                                dataSink, localId, "xml",
1183 5449 berkley
                                username,
1184
                                groupnames, true, params);
1185 5332 jones
                    } catch (PropertyNotFoundException e) {
1186 5382 berkley
                        e.printStackTrace();
1187
                        throw new ServiceFailure("1030", "Property not found while reading system metadata from metacat: " + e.getMessage());
1188 5332 jones
                    } catch (ClassNotFoundException e) {
1189 5382 berkley
                        e.printStackTrace();
1190
                        throw new ServiceFailure("1030", "Class not found while reading system metadata from metacat: " + e.getMessage());
1191 5332 jones
                    } catch (IOException e) {
1192 5382 berkley
                        e.printStackTrace();
1193
                        throw new ServiceFailure("1030", "IOException while reading system metadata from metacat: " + e.getMessage());
1194 5332 jones
                    } catch (SQLException e) {
1195 5382 berkley
                        e.printStackTrace();
1196
                        throw new ServiceFailure("1030", "SQLException while reading system metadata from metacat: " + e.getMessage());
1197 5332 jones
                    } catch (McdbException e) {
1198 5382 berkley
                        e.printStackTrace();
1199
                        throw new ServiceFailure("1030", "Metacat DB Exception while reading system metadata from metacat: " + e.getMessage());
1200 5332 jones
                    } catch (ParseLSIDException e) {
1201 5382 berkley
                        e.printStackTrace();
1202
                        throw new NotFound("1020", "Error parsing LSID while reading system metadata from metacat: " + e.getMessage());
1203 5332 jones
                    } catch (InsufficientKarmaException e) {
1204 5382 berkley
                        e.printStackTrace();
1205
                        throw new NotAuthorized("1000", "User not authorized for get() on system metadata: " + e.getMessage());
1206 5332 jones
                    }
1207
1208
                    return "Completed";
1209
                }
1210
            };
1211
1212
            // Deserialize the xml to create a SystemMetadata object
1213
            SystemMetadata sysmeta = deserializeSystemMetadata(objectStream);
1214 5431 berkley
            String username = "public";
1215
            if(sessionData != null)
1216
            {
1217
                username = sessionData.getUserName();
1218
            }
1219 5383 berkley
            EventLog.getInstance().log(metacatUrl,
1220 5384 berkley
                    username, im.getLocalId(guid.getValue()), "read");
1221 5459 berkley
            logCrud.info("getsystemmetadata D1GUID:" + guid.getValue()  +
1222
                    ":D1SYSMETA:"+ localId + ":");
1223 5332 jones
            return sysmeta;
1224
1225
        } catch (McdbDocNotFoundException e) {
1226 5348 berkley
            //e.printStackTrace();
1227 5356 berkley
            throw new NotFound("1000", e.getMessage());
1228 5332 jones
        }
1229 5298 jones
    }
1230 5366 berkley
1231
    /**
1232
     * parse the date in the systemMetadata
1233
     * @param s
1234
     * @return
1235
     * @throws Exception
1236
     */
1237 5418 berkley
    public Date parseDate(String s)
1238 5366 berkley
      throws Exception
1239
    {
1240 5466 berkley
        /* TODO:
1241
         * This method should be replaced by a DateFormatter
1242
         */
1243 5366 berkley
        Date d = null;
1244
        int tIndex = s.indexOf("T");
1245
        int zIndex = s.indexOf("Z");
1246
        if(tIndex != -1 && zIndex != -1)
1247
        { //parse a date that looks like 2010-05-18T21:12:54.362Z
1248
            //System.out.println("original date: " + s);
1249
1250
            String date = s.substring(0, tIndex);
1251
            String year = date.substring(0, date.indexOf("-"));
1252
            String month = date.substring(date.indexOf("-") + 1, date.lastIndexOf("-"));
1253
            String day = date.substring(date.lastIndexOf("-") + 1, date.length());
1254
            /*System.out.println("date: " + "year: " + new Integer(year).intValue() +
1255
                    " month: " + new Integer(month).intValue() + " day: " +
1256
                    new Integer(day).intValue());
1257
            */
1258
            String time = s.substring(tIndex + 1, zIndex);
1259
            String hour = time.substring(0, time.indexOf(":"));
1260
            String minute = time.substring(time.indexOf(":") + 1, time.lastIndexOf(":"));
1261
            String seconds = "00";
1262
            String milliseconds = "00";
1263
            if(time.indexOf(".") != -1)
1264
            {
1265
                seconds = time.substring(time.lastIndexOf(":") + 1, time.indexOf("."));
1266
                milliseconds = time.substring(time.indexOf(".") + 1, time.length());
1267
            }
1268 5418 berkley
            else
1269
            {
1270
                seconds = time.substring(time.lastIndexOf(":") + 1, time.length());
1271
            }
1272 5366 berkley
            /*System.out.println("time: " + "hour: " + new Integer(hour).intValue() +
1273
                    " minute: " + new Integer(minute).intValue() + " seconds: " +
1274
                    new Integer(seconds).intValue() + " milli: " +
1275
                    new Integer(milliseconds).intValue());*/
1276
1277
            //d = DateFormat.getDateTimeInstance().parse(date + " " + time);
1278 5418 berkley
            Calendar c = Calendar.getInstance(/*TimeZone.getTimeZone("GMT-0")*/TimeZone.getDefault());
1279 5366 berkley
            c.set(new Integer(year).intValue(), new Integer(month).intValue() - 1,
1280
                  new Integer(day).intValue(), new Integer(hour).intValue(),
1281
                  new Integer(minute).intValue(), new Integer(seconds).intValue());
1282
            c.set(Calendar.MILLISECOND, new Integer(milliseconds).intValue());
1283
            d = new Date(c.getTimeInMillis());
1284
            //System.out.println("d: " + d);
1285
            return d;
1286
        }
1287
        else
1288
        {  //if it's not in the expected format, try the formatter
1289
            return DateFormat.getDateTimeInstance().parse(s);
1290
        }
1291
    }
1292 5298 jones
1293 5331 jones
    /*
1294
     * Look up the information on the session using the token provided in
1295
     * the AuthToken.  The Session should have all relevant user information.
1296
     * If the session has expired or is invalid, the 'public' session will
1297
     * be returned, giving the user anonymous access.
1298
     */
1299 5394 berkley
    public static SessionData getSessionData(AuthToken token) {
1300 5331 jones
        SessionData sessionData = null;
1301
        String sessionId = "PUBLIC";
1302
        if (token != null) {
1303
            sessionId = token.getToken();
1304
        }
1305
1306
        // if the session id is registered in SessionService, get the
1307
        // SessionData for it. Otherwise, use the public session.
1308 5411 berkley
        //System.out.println("sessionid: " + sessionId);
1309 5403 berkley
        if (sessionId != null &&
1310 5406 berkley
            !sessionId.toLowerCase().equals("public") &&
1311 5403 berkley
            SessionService.getInstance().isSessionRegistered(sessionId))
1312
        {
1313 5374 berkley
            sessionData = SessionService.getInstance().getRegisteredSession(sessionId);
1314 5331 jones
        } else {
1315 5374 berkley
            sessionData = SessionService.getInstance().getPublicSession();
1316 5331 jones
        }
1317 5411 berkley
1318 5331 jones
        return sessionData;
1319
    }
1320
1321
    /**
1322
     * Determine if a given object should be treated as an XML science metadata
1323
     * object.
1324
     *
1325
     * TODO: This test should be externalized in a configuration dictionary rather than being hardcoded.
1326
     *
1327
     * @param sysmeta the SystemMetadata describig the object
1328
     * @return true if the object should be treated as science metadata
1329
     */
1330
    private boolean isScienceMetadata(SystemMetadata sysmeta) {
1331 5523 berkley
        /*boolean scimeta = false;
1332 5466 berkley
        //TODO: this should be read from a .properties file instead of being hard coded
1333 5331 jones
        switch (sysmeta.getObjectFormat()) {
1334
            case EML_2_1_0: scimeta = true; break;
1335
            case EML_2_0_1: scimeta = true; break;
1336
            case EML_2_0_0: scimeta = true; break;
1337
            case FGDC_STD_001_1_1999: scimeta = true; break;
1338
            case FGDC_STD_001_1998: scimeta = true; break;
1339
            case NCML_2_2: scimeta = true; break;
1340 5521 berkley
            case DSPACE_METS_SIP_1_0: scimeta = true; break;
1341 5331 jones
        }
1342
1343 5523 berkley
        return scimeta;*/
1344
1345
        return MetadataTypeRegister.isMetadataType(sysmeta.getObjectFormat());
1346 5331 jones
    }
1347
1348 5359 berkley
    /**
1349
     * insert a data doc
1350
     * @param object
1351
     * @param guid
1352
     * @param sessionData
1353
     * @throws ServiceFailure
1354 5459 berkley
     * @returns localId of the data object inserted
1355 5359 berkley
     */
1356 5459 berkley
    private String insertDataObject(InputStream object, Identifier guid,
1357 5331 jones
            SessionData sessionData) throws ServiceFailure {
1358
1359 5431 berkley
        String username = "public";
1360
        String[] groups = null;
1361
        if(sessionData != null)
1362
        {
1363
          username = sessionData.getUserName();
1364
          groups = sessionData.getGroupNames();
1365
        }
1366 5331 jones
1367
        // generate guid/localId pair for object
1368
        logMetacat.debug("Generating a guid/localId mapping");
1369
        IdentifierManager im = IdentifierManager.getInstance();
1370
        String localId = im.generateLocalId(guid.getValue(), 1);
1371
1372
        try {
1373
            logMetacat.debug("Case DATA: starting to write to disk.");
1374
            if (DocumentImpl.getDataFileLockGrant(localId)) {
1375
1376
                // Save the data file to disk using "localId" as the name
1377
                try {
1378
                    String datafilepath = PropertyService.getProperty("application.datafilepath");
1379
1380
                    File dataDirectory = new File(datafilepath);
1381
                    dataDirectory.mkdirs();
1382
1383
                    File newFile = writeStreamToFile(dataDirectory, localId, object);
1384
1385
                    // TODO: Check that the file size matches SystemMetadata
1386
                    //                        long size = newFile.length();
1387
                    //                        if (size == 0) {
1388
                    //                            throw new IOException("Uploaded file is 0 bytes!");
1389
                    //                        }
1390
1391
                    // Register the file in the database (which generates an exception
1392
                    // if the localId is not acceptable or other untoward things happen
1393
                    try {
1394
                        logMetacat.debug("Registering document...");
1395
                        DocumentImpl.registerDocument(localId, "BIN", localId,
1396
                                username, groups);
1397
                        logMetacat.debug("Registration step completed.");
1398
                    } catch (SQLException e) {
1399
                        //newFile.delete();
1400
                        logMetacat.debug("SQLE: " + e.getMessage());
1401
                        e.printStackTrace(System.out);
1402 5356 berkley
                        throw new ServiceFailure("1190", "Registration failed: " + e.getMessage());
1403 5331 jones
                    } catch (AccessionNumberException e) {
1404
                        //newFile.delete();
1405
                        logMetacat.debug("ANE: " + e.getMessage());
1406
                        e.printStackTrace(System.out);
1407 5356 berkley
                        throw new ServiceFailure("1190", "Registration failed: " + e.getMessage());
1408 5331 jones
                    } catch (Exception e) {
1409
                        //newFile.delete();
1410
                        logMetacat.debug("Exception: " + e.getMessage());
1411
                        e.printStackTrace(System.out);
1412 5356 berkley
                        throw new ServiceFailure("1190", "Registration failed: " + e.getMessage());
1413 5331 jones
                    }
1414
1415
                    logMetacat.debug("Logging the creation event.");
1416 5344 berkley
                    EventLog.getInstance().log(metacatUrl,
1417 5384 berkley
                            username, localId, "create");
1418 5331 jones
1419
                    // Schedule replication for this data file
1420
                    logMetacat.debug("Scheduling replication.");
1421
                    ForceReplicationHandler frh = new ForceReplicationHandler(
1422 5384 berkley
                            localId, "create", false, null);
1423 5331 jones
1424
                } catch (PropertyNotFoundException e) {
1425 5356 berkley
                    throw new ServiceFailure("1190", "Could not lock file for writing:" + e.getMessage());
1426 5331 jones
                }
1427
            }
1428 5459 berkley
            return localId;
1429 5331 jones
        } catch (Exception e) {
1430
            // Could not get a lock on the document, so we can not update the file now
1431 5356 berkley
            throw new ServiceFailure("1190", "Failed to lock file: " + e.getMessage());
1432 5331 jones
        }
1433
    }
1434
1435 5359 berkley
    /**
1436
     * write a file to a stream
1437
     * @param dir
1438
     * @param fileName
1439
     * @param data
1440
     * @return
1441
     * @throws ServiceFailure
1442
     */
1443 5319 jones
    private File writeStreamToFile(File dir, String fileName, InputStream data)
1444
        throws ServiceFailure {
1445
1446
        File newFile = new File(dir, fileName);
1447
        logMetacat.debug("Filename for write is: " + newFile.getAbsolutePath());
1448
1449
        try {
1450
            if (newFile.createNewFile()) {
1451
                // write data stream to desired file
1452
                OutputStream os = new FileOutputStream(newFile);
1453
                long length = IOUtils.copyLarge(data, os);
1454
                os.flush();
1455
                os.close();
1456
            } else {
1457
                logMetacat.debug("File creation failed, or file already exists.");
1458 5356 berkley
                throw new ServiceFailure("1190", "File already exists: " + fileName);
1459 5319 jones
            }
1460
        } catch (FileNotFoundException e) {
1461
            logMetacat.debug("FNF: " + e.getMessage());
1462 5356 berkley
            throw new ServiceFailure("1190", "File not found: " + fileName + " "
1463 5319 jones
                    + e.getMessage());
1464
        } catch (IOException e) {
1465
            logMetacat.debug("IOE: " + e.getMessage());
1466 5356 berkley
            throw new ServiceFailure("1190", "File was not written: " + fileName
1467 5319 jones
                    + " " + e.getMessage());
1468
        }
1469
1470
        return newFile;
1471 5329 jones
    }
1472
1473 5350 berkley
    /**
1474 5441 berkley
     * insert a systemMetadata doc, return the localId of the sysmeta
1475 5350 berkley
     */
1476 5441 berkley
    private String insertSystemMetadata(SystemMetadata sysmeta, SessionData sessionData)
1477 5377 berkley
        throws ServiceFailure
1478
    {
1479 5329 jones
        logMetacat.debug("Starting to insert SystemMetadata...");
1480
1481
        // generate guid/localId pair for sysmeta
1482 5331 jones
        Identifier sysMetaGuid = new Identifier();
1483
        sysMetaGuid.setValue(DocumentUtil.generateDocumentId(1));
1484 5329 jones
        sysmeta.setDateSysMetadataModified(new Date());
1485 5453 berkley
        System.out.println("****inserting new system metadata with modified date " +
1486
                sysmeta.getDateSysMetadataModified());
1487 5329 jones
1488
        String xml = new String(serializeSystemMetadata(sysmeta).toByteArray());
1489 5444 berkley
        System.out.println("sysmeta: " + xml);
1490 5453 berkley
        String localId = insertDocument(xml, sysMetaGuid, sessionData, true);
1491 5420 berkley
        System.out.println("sysmeta inserted with localId " + localId);
1492 5453 berkley
        //insert the system metadata doc id into the systemmetadata table to
1493 5333 berkley
        //link it to the data or metadata document
1494 5377 berkley
        IdentifierManager.getInstance().createSystemMetadataMapping(
1495
                sysmeta.getIdentifier().getValue(), sysMetaGuid.getValue());
1496 5441 berkley
        return localId;
1497 5329 jones
    }
1498
1499 5333 berkley
    /**
1500 5350 berkley
     * update a systemMetadata doc
1501
     */
1502
    private void updateSystemMetadata(SystemMetadata sm, SessionData sessionData)
1503
      throws ServiceFailure
1504
    {
1505
        try
1506
        {
1507 5424 berkley
            String smId = IdentifierManager.getInstance().getSystemMetadataLocalId(sm.getIdentifier().getValue());
1508 5350 berkley
            sm.setDateSysMetadataModified(new Date());
1509
            String xml = new String(serializeSystemMetadata(sm).toByteArray());
1510 5456 berkley
            String localId = updateDocument(xml, sm.getIdentifier(), null, sessionData, true);
1511 5350 berkley
            IdentifierManager.getInstance().updateSystemMetadataMapping(sm.getIdentifier().getValue(), localId);
1512
        }
1513
        catch(Exception e)
1514
        {
1515 5356 berkley
            throw new ServiceFailure("1030", "Error updating system metadata: " + e.getMessage());
1516 5350 berkley
        }
1517
    }
1518
1519 5453 berkley
    private String insertDocument(String xml, Identifier guid, SessionData sessionData)
1520
        throws ServiceFailure
1521
    {
1522
        return insertDocument(xml, guid, sessionData, false);
1523
    }
1524
1525 5350 berkley
    /**
1526
     * insert a document
1527
     * NOTE: this method shouldn't be used from the update or create() methods.
1528
     * we shouldn't be putting the science metadata or data objects into memory.
1529
     */
1530 5453 berkley
    private String insertDocument(String xml, Identifier guid, SessionData sessionData,
1531
            boolean isSystemMetadata)
1532 5350 berkley
        throws ServiceFailure
1533
    {
1534 5453 berkley
        return insertOrUpdateDocument(xml, guid, sessionData, "insert", isSystemMetadata);
1535 5350 berkley
    }
1536
1537
    /**
1538
     * insert a document from a stream
1539
     */
1540
    private String insertDocument(InputStream is, Identifier guid, SessionData sessionData)
1541
      throws IOException, ServiceFailure
1542
    {
1543
        //HACK: change this eventually.  we should not be converting the stream to a string
1544
        String xml = IOUtils.toString(is);
1545
        return insertDocument(xml, guid, sessionData);
1546
    }
1547
1548
    /**
1549
     * update a document
1550
     * NOTE: this method shouldn't be used from the update or create() methods.
1551
     * we shouldn't be putting the science metadata or data objects into memory.
1552
     */
1553 5453 berkley
    private String updateDocument(String xml, Identifier obsoleteGuid,
1554
            Identifier guid, SessionData sessionData, boolean isSystemMetadata)
1555 5350 berkley
        throws ServiceFailure
1556
    {
1557 5453 berkley
        return insertOrUpdateDocument(xml, obsoleteGuid, sessionData, "update", isSystemMetadata);
1558 5350 berkley
    }
1559
1560
    /**
1561
     * update a document from a stream
1562
     */
1563 5453 berkley
    private String updateDocument(InputStream is, Identifier obsoleteGuid,
1564
            Identifier guid, SessionData sessionData, boolean isSystemMetadata)
1565 5350 berkley
      throws IOException, ServiceFailure
1566
    {
1567
        //HACK: change this eventually.  we should not be converting the stream to a string
1568
        String xml = IOUtils.toString(is);
1569 5453 berkley
        String localId = updateDocument(xml, obsoleteGuid, guid, sessionData, isSystemMetadata);
1570 5350 berkley
        IdentifierManager im = IdentifierManager.getInstance();
1571
        if(guid != null)
1572
        {
1573
          im.createMapping(guid.getValue(), localId);
1574
        }
1575
        return localId;
1576
    }
1577
1578
    /**
1579 5333 berkley
     * insert a document, return the id of the document that was inserted
1580
     */
1581 5453 berkley
    protected String insertOrUpdateDocument(String xml, Identifier guid,
1582
            SessionData sessionData, String insertOrUpdate, boolean isSystemMetadata)
1583 5329 jones
        throws ServiceFailure {
1584
        logMetacat.debug("Starting to insert xml document...");
1585
        IdentifierManager im = IdentifierManager.getInstance();
1586
1587
        // generate guid/localId pair for sysmeta
1588 5350 berkley
        String localId = null;
1589
        if(insertOrUpdate.equals("insert"))
1590
        {
1591 5453 berkley
            localId = im.generateLocalId(guid.getValue(), 1, isSystemMetadata);
1592 5350 berkley
        }
1593
        else
1594
        {
1595
            //localid should already exist in the identifier table, so just find it
1596
            try
1597
            {
1598 5456 berkley
                System.out.println("updating guid " + guid.getValue());
1599
                if(!isSystemMetadata)
1600
                {
1601
                    System.out.println("looking in identifier table for guid " + guid.getValue());
1602
                    localId = im.getLocalId(guid.getValue());
1603
                }
1604
                else
1605
                {
1606
                    System.out.println("looking in systemmetadata table for guid " + guid.getValue());
1607
                    localId = im.getSystemMetadataLocalId(guid.getValue());
1608
                }
1609
                System.out.println("localId: " + localId);
1610 5350 berkley
                //increment the revision
1611
                String docid = localId.substring(0, localId.lastIndexOf("."));
1612
                String revS = localId.substring(localId.lastIndexOf(".") + 1, localId.length());
1613
                int rev = new Integer(revS).intValue();
1614
                rev++;
1615
                docid = docid + "." + rev;
1616
                localId = docid;
1617 5456 berkley
                System.out.println("incremented localId: " + localId);
1618 5350 berkley
            }
1619
            catch(McdbDocNotFoundException e)
1620
            {
1621 5356 berkley
                throw new ServiceFailure("1030", "CrudService.insertOrUpdateDocument(): " +
1622 5350 berkley
                    "guid " + guid.getValue() + " should have been in the identifier table, but it wasn't: " + e.getMessage());
1623
            }
1624
        }
1625 5331 jones
        logMetacat.debug("Metadata guid|localId: " + guid.getValue() + "|" +
1626 5329 jones
                localId);
1627
1628
        String[] action = new String[1];
1629 5350 berkley
        action[0] = insertOrUpdate;
1630 5329 jones
        params.put("action", action);
1631
        String[] docid = new String[1];
1632
        docid[0] = localId;
1633
        params.put("docid", docid);
1634
        String[] doctext = new String[1];
1635
        doctext[0] = xml;
1636
        logMetacat.debug(doctext[0]);
1637
        params.put("doctext", doctext);
1638 5331 jones
1639
        // TODO: refactor handleInsertOrUpdateAction() to not output XML directly
1640
        // onto output stream, or alternatively, capture that and parse it to
1641
        // generate the right exceptions
1642 5337 berkley
        //ByteArrayOutputStream output = new ByteArrayOutputStream();
1643
        //PrintWriter pw = new PrintWriter(output);
1644 5449 berkley
        String username = "public";
1645
        String[] groupnames = null;
1646
        if(sessionData != null)
1647
        {
1648
            username = sessionData.getUserName();
1649
            groupnames = sessionData.getGroupNames();
1650
        }
1651 5344 berkley
        String result = handler.handleInsertOrUpdateAction(metacatUrl, null,
1652 5449 berkley
                            null, params, username, groupnames);
1653 5461 berkley
        if(result.indexOf("<error>") != -1)
1654
        {
1655
            throw new ServiceFailure("1000", "Error inserting or updating document: " + result);
1656
        }
1657 5337 berkley
        //String outputS = new String(output.toByteArray());
1658
        logMetacat.debug("CrudService.insertDocument - Metacat returned: " + result);
1659 5333 berkley
        logMetacat.debug("Finsished inserting xml document with id " + localId);
1660
        return localId;
1661 5329 jones
    }
1662
1663 5359 berkley
    /**
1664 5555 berkley
     * serialize a dataone type
1665 5550 berkley
     */
1666 5555 berkley
    private void serializeServiceType(Class type, Object object, OutputStream out)
1667
        throws JiBXException
1668 5550 berkley
    {
1669 5555 berkley
        IBindingFactory bfact = BindingDirectory.getFactory(type);
1670
        IMarshallingContext mctx = bfact.createMarshallingContext();
1671
        mctx.marshalDocument(object, "UTF-8", null, out);
1672 5550 berkley
    }
1673
1674
    /**
1675 5359 berkley
     * serialize a system metadata doc
1676
     * @param sysmeta
1677
     * @return
1678
     * @throws ServiceFailure
1679
     */
1680 5332 jones
    public static ByteArrayOutputStream serializeSystemMetadata(SystemMetadata sysmeta)
1681 5329 jones
        throws ServiceFailure {
1682
        IBindingFactory bfact;
1683
        ByteArrayOutputStream sysmetaOut = null;
1684
        try {
1685
            bfact = BindingDirectory.getFactory(SystemMetadata.class);
1686
            IMarshallingContext mctx = bfact.createMarshallingContext();
1687
            sysmetaOut = new ByteArrayOutputStream();
1688
            mctx.marshalDocument(sysmeta, "UTF-8", null, sysmetaOut);
1689
        } catch (JiBXException e) {
1690 5382 berkley
            e.printStackTrace();
1691 5356 berkley
            throw new ServiceFailure("1190", "Failed to serialize and insert SystemMetadata: " + e.getMessage());
1692 5329 jones
        }
1693
1694
        return sysmetaOut;
1695
    }
1696 5332 jones
1697 5359 berkley
    /**
1698
     * deserialize a system metadata doc
1699
     * @param xml
1700
     * @return
1701
     * @throws ServiceFailure
1702
     */
1703 5332 jones
    public static SystemMetadata deserializeSystemMetadata(InputStream xml)
1704
        throws ServiceFailure {
1705
        try {
1706
            IBindingFactory bfact = BindingDirectory.getFactory(SystemMetadata.class);
1707
            IUnmarshallingContext uctx = bfact.createUnmarshallingContext();
1708
            SystemMetadata sysmeta = (SystemMetadata) uctx.unmarshalDocument(xml, null);
1709
            return sysmeta;
1710
        } catch (JiBXException e) {
1711 5382 berkley
            e.printStackTrace();
1712
            throw new ServiceFailure("1190", "Failed to deserialize and insert SystemMetadata: " + e.getMessage());
1713 5332 jones
        }
1714
    }
1715 5378 berkley
1716
    /**
1717
     * produce an md5 checksum for item
1718
     */
1719
    private String checksum(InputStream is)
1720
      throws Exception
1721
    {
1722
        byte[] buffer = new byte[1024];
1723
        MessageDigest complete = MessageDigest.getInstance("MD5");
1724
        int numRead;
1725
1726
        do
1727
        {
1728
          numRead = is.read(buffer);
1729
          if (numRead > 0)
1730
          {
1731
            complete.update(buffer, 0, numRead);
1732
          }
1733
        } while (numRead != -1);
1734
1735
1736
        return getHex(complete.digest());
1737
    }
1738
1739
    /**
1740
     * convert a byte array to a hex string
1741
     */
1742
    private static String getHex( byte [] raw )
1743
    {
1744
        final String HEXES = "0123456789ABCDEF";
1745
        if ( raw == null ) {
1746
          return null;
1747
        }
1748
        final StringBuilder hex = new StringBuilder( 2 * raw.length );
1749
        for ( final byte b : raw ) {
1750
          hex.append(HEXES.charAt((b & 0xF0) >> 4))
1751
             .append(HEXES.charAt((b & 0x0F)));
1752
        }
1753
        return hex.toString();
1754
    }
1755
1756
    /**
1757
     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into
1758
     * a proper date object
1759
     * @param date
1760
     * @return
1761
     */
1762
    private Date parseMetacatDate(String date)
1763
    {
1764
        String year = date.substring(0, 4);
1765
        String month = date.substring(5, 7);
1766
        String day = date.substring(8, 10);
1767 5420 berkley
        Calendar c = Calendar.getInstance(TimeZone.getDefault());
1768 5378 berkley
        c.set(new Integer(year).intValue(),
1769
              new Integer(month).intValue(),
1770
              new Integer(day).intValue());
1771 5420 berkley
        System.out.println("time in parseMetacatDate: " + c.getTime());
1772 5378 berkley
        return c.getTime();
1773
    }
1774
1775
    /**
1776
     * find the size (in bytes) of a stream
1777
     * @param is
1778
     * @return
1779
     * @throws IOException
1780
     */
1781
    private long sizeOfStream(InputStream is)
1782
        throws IOException
1783
    {
1784
        long size = 0;
1785
        byte[] b = new byte[1024];
1786
        int numread = is.read(b, 0, 1024);
1787
        while(numread != -1)
1788
        {
1789
            size += numread;
1790
            numread = is.read(b, 0, 1024);
1791
        }
1792
        return size;
1793
    }
1794 5379 berkley
1795
    /**
1796
     * create system metadata with a specified id, doc and format
1797
     */
1798
    private SystemMetadata createSystemMetadata(String localId, AuthToken token)
1799
      throws Exception
1800
    {
1801
        IdentifierManager im = IdentifierManager.getInstance();
1802 5441 berkley
        Hashtable<String, Object> docInfo = im.getDocumentInfo(localId);
1803 5379 berkley
1804
        //get the document text
1805
        int rev = im.getLatestRevForLocalId(localId);
1806
        Identifier identifier = new Identifier();
1807
        identifier.setValue(im.getGUID(localId, rev));
1808
        InputStream is = this.get(token, identifier);
1809
1810
        SystemMetadata sm = new SystemMetadata();
1811
        //set the id
1812
        sm.setIdentifier(identifier);
1813
1814
        //set the object format
1815 5441 berkley
        String doctype = (String)docInfo.get("doctype");
1816
        ObjectFormat format = ObjectFormat.convert((String)docInfo.get("doctype"));
1817 5379 berkley
        if(format == null)
1818
        {
1819
            if(doctype.trim().equals("BIN"))
1820
            {
1821 5521 berkley
                format = ObjectFormat.OCTET_STREAM;
1822 5379 berkley
            }
1823
            else
1824
            {
1825
                format = ObjectFormat.convert("text/plain");
1826
            }
1827
        }
1828
        sm.setObjectFormat(format);
1829
1830
        //create the checksum
1831
        String checksumS = checksum(is);
1832
        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");
1833
        Checksum checksum = new Checksum();
1834
        checksum.setValue(checksumS);
1835
        checksum.setAlgorithm(ca);
1836
        sm.setChecksum(checksum);
1837
1838
        //set the size
1839
        is = this.get(token, identifier);
1840
        sm.setSize(sizeOfStream(is));
1841
1842
        //submitter
1843
        Principal p = new Principal();
1844 5441 berkley
        p.setValue((String)docInfo.get("user_owner"));
1845 5379 berkley
        sm.setSubmitter(p);
1846
        sm.setRightsHolder(p);
1847
        try
1848
        {
1849 5441 berkley
            Date dateCreated = parseMetacatDate((String)docInfo.get("date_created"));
1850 5379 berkley
            sm.setDateUploaded(dateCreated);
1851 5441 berkley
            Date dateUpdated = parseMetacatDate((String)docInfo.get("date_updated"));
1852 5379 berkley
            sm.setDateSysMetadataModified(dateUpdated);
1853
        }
1854
        catch(Exception e)
1855
        {
1856 5428 berkley
            System.out.println("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
1857 5379 berkley
            Date dateCreated = new Date();
1858
            sm.setDateUploaded(dateCreated);
1859
            Date dateUpdated = new Date();
1860
            sm.setDateSysMetadataModified(dateUpdated);
1861
        }
1862
        NodeReference nr = new NodeReference();
1863 5466 berkley
        //TODO: this should be set to be something more meaningful once the registry is up
1864 5379 berkley
        nr.setValue("metacat");
1865
        sm.setOriginMemberNode(nr);
1866
        sm.setAuthoritativeMemberNode(nr);
1867
        return sm;
1868
    }
1869 5298 jones
}