Project

General

Profile

1 5298 jones
/**
2
 *  '$RCSfile$'
3
 *  Copyright: 2000 Regents of the University of California and the
4
 *              National Center for Ecological Analysis and Synthesis
5
 *
6 5299 jones
 *   '$Author: $'
7 5298 jones
 *     '$Date: 2009-06-13 15:28:13 +0300  $'
8
 *
9
 * This program is free software; you can redistribute it and/or modify
10
 * it under the terms of the GNU General Public License as published by
11
 * the Free Software Foundation; either version 2 of the License, or
12
 * (at your option) any later version.
13
 *
14
 * This program is distributed in the hope that it will be useful,
15
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 * GNU General Public License for more details.
18
 *
19
 * You should have received a copy of the GNU General Public License
20
 * along with this program; if not, write to the Free Software
21
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22
 */
23
package edu.ucsb.nceas.metacat.dataone;
24
25 5329 jones
import java.io.ByteArrayOutputStream;
26 5319 jones
import java.io.File;
27 5679 berkley
import java.io.FileInputStream;
28 5319 jones
import java.io.FileNotFoundException;
29
import java.io.FileOutputStream;
30 5298 jones
import java.io.IOException;
31
import java.io.InputStream;
32
import java.io.OutputStream;
33 5329 jones
import java.io.PrintWriter;
34 5378 berkley
import java.io.StringBufferInputStream;
35
import java.security.MessageDigest;
36 5298 jones
import java.sql.SQLException;
37 5366 berkley
import java.util.*;
38 5364 berkley
import java.text.DateFormat;
39 5420 berkley
import java.text.SimpleDateFormat;
40 5298 jones
41
import javax.servlet.ServletContext;
42
import javax.servlet.http.HttpServletRequest;
43
import javax.servlet.http.HttpServletResponse;
44
45 5319 jones
import org.apache.commons.io.IOUtils;
46
import org.apache.log4j.Logger;
47 5298 jones
import org.dataone.service.exceptions.IdentifierNotUnique;
48
import org.dataone.service.exceptions.InsufficientResources;
49
import org.dataone.service.exceptions.InvalidRequest;
50
import org.dataone.service.exceptions.InvalidSystemMetadata;
51
import org.dataone.service.exceptions.InvalidToken;
52
import org.dataone.service.exceptions.NotAuthorized;
53
import org.dataone.service.exceptions.NotFound;
54
import org.dataone.service.exceptions.NotImplemented;
55
import org.dataone.service.exceptions.ServiceFailure;
56
import org.dataone.service.exceptions.UnsupportedType;
57
import org.dataone.service.mn.MemberNodeCrud;
58 5364 berkley
import org.dataone.service.types.*;
59 5329 jones
import org.jibx.runtime.BindingDirectory;
60
import org.jibx.runtime.IBindingFactory;
61
import org.jibx.runtime.IMarshallingContext;
62 5332 jones
import org.jibx.runtime.IUnmarshallingContext;
63 5329 jones
import org.jibx.runtime.JiBXException;
64 5298 jones
65 5547 berkley
import org.dataone.service.types.Identifier;
66 5851 berkley
import org.dataone.service.types.util.ServiceTypeUtil;
67 5547 berkley
68 5298 jones
import com.gc.iotools.stream.is.InputStreamFromOutputStream;
69
70 5798 berkley
import edu.ucsb.nceas.metacat.AccessionNumber;
71 5319 jones
import edu.ucsb.nceas.metacat.AccessionNumberException;
72 5363 berkley
import edu.ucsb.nceas.metacat.MetacatResultSet;
73
import edu.ucsb.nceas.metacat.MetacatResultSet.Document;
74 5510 berkley
import edu.ucsb.nceas.metacat.DBQuery;
75 5319 jones
import edu.ucsb.nceas.metacat.DocumentImpl;
76
import edu.ucsb.nceas.metacat.EventLog;
77 5298 jones
import edu.ucsb.nceas.metacat.IdentifierManager;
78
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
79
import edu.ucsb.nceas.metacat.McdbException;
80
import edu.ucsb.nceas.metacat.MetacatHandler;
81
import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;
82 5370 berkley
import edu.ucsb.nceas.metacat.client.rest.MetacatRestClient;
83 5319 jones
import edu.ucsb.nceas.metacat.properties.PropertyService;
84
import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;
85 5298 jones
import edu.ucsb.nceas.metacat.service.SessionService;
86 5329 jones
import edu.ucsb.nceas.metacat.util.DocumentUtil;
87 5298 jones
import edu.ucsb.nceas.metacat.util.SessionData;
88
import edu.ucsb.nceas.utilities.ParseLSIDException;
89
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
90
91
/**
92
 *
93
 * Implements DataONE MemberNode CRUD API for Metacat.
94
 *
95
 * @author Matthew Jones
96
 */
97 5394 berkley
public class CrudService implements MemberNodeCrud
98
{
99 5337 berkley
    private static CrudService crudService = null;
100 5298 jones
101
    private MetacatHandler handler;
102
    private Hashtable<String, String[]> params;
103 5392 berkley
    private Logger logMetacat = null;
104
    private Logger logCrud = null;
105 5337 berkley
106
    private String metacatUrl;
107 5420 berkley
108 5652 berkley
    private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
109 5298 jones
110
    /**
111 5337 berkley
     * singleton accessor
112
     */
113 5630 berkley
    public static CrudService getInstance()
114 5337 berkley
    {
115
      if(crudService == null)
116
      {
117
        crudService = new CrudService();
118
      }
119
120
      return crudService;
121
    }
122
123
    /**
124 5770 berkley
     * Constructor, private for singleton access
125 5298 jones
     */
126 5763 berkley
    private CrudService() {
127 5337 berkley
        logMetacat = Logger.getLogger(CrudService.class);
128 5392 berkley
        logCrud = Logger.getLogger("DataOneLogger");
129 5337 berkley
        try
130
        {
131
            String server = PropertyService.getProperty("server.name");
132
            String port = PropertyService.getProperty("server.httpPort");
133
            String context = PropertyService.getProperty("application.context");
134 5746 berkley
            metacatUrl = "http://" + server + ":" + port + "/" + context + "/d1";
135 5337 berkley
            logMetacat.debug("Initializing CrudService with url " + metacatUrl);
136
        }
137
        catch(Exception e)
138
        {
139
            logMetacat.error("Could not find servlet url in CrudService: " + e.getMessage());
140
            e.printStackTrace();
141
            throw new RuntimeException("Error getting servlet url in CrudService: " + e.getMessage());
142
        }
143
144
        params = new Hashtable<String, String[]>();
145
        handler = new MetacatHandler(new Timer());
146 5298 jones
    }
147 5337 berkley
148 5298 jones
    /**
149 5337 berkley
     * return the context url CrudService is using.
150 5298 jones
     */
151 5337 berkley
    public String getContextUrl()
152
    {
153
        return metacatUrl;
154
    }
155
156
    /**
157 5394 berkley
     * Set the context url that this service uses.  It is normally not necessary
158
     * to call this method unless you are trying to connect to a server other
159
     * than the one in which this service is installed.  Otherwise, this value is
160
     * taken from the metacat.properties file (server.name, server.port, application.context).
161
     */
162
    public void setContextUrl(String url)
163
    {
164
        metacatUrl = url;
165
    }
166
167
    /**
168 5337 berkley
     * set the params for this service from an HttpServletRequest param list
169
     */
170
    public void setParamsFromRequest(HttpServletRequest request)
171
    {
172 5298 jones
        Enumeration paramlist = request.getParameterNames();
173
        while (paramlist.hasMoreElements()) {
174 5337 berkley
            String name = (String) paramlist.nextElement();
175
            String[] value = (String[])request.getParameterValues(name);
176 5298 jones
            params.put(name, value);
177
        }
178
    }
179
180 5337 berkley
    /**
181 5370 berkley
     * Authenticate against metacat and get a token.
182
     * @param username
183
     * @param password
184
     * @return
185
     * @throws ServiceFailure
186
     */
187
    public AuthToken authenticate(String username, String password)
188
      throws ServiceFailure
189
    {
190 5466 berkley
        /* TODO:
191
         * This method is not in the original D1 crud spec.  It is highly
192
         * metacat centric.  Higher level decisions need to be made on authentication
193
         * interfaces for D1 nodes.
194
         */
195 5370 berkley
        try
196
        {
197
            MetacatRestClient restClient = new MetacatRestClient(getContextUrl());
198
            String response = restClient.login(username, password);
199
            String sessionid = restClient.getSessionId();
200
            SessionService sessionService = SessionService.getInstance();
201
            sessionService.registerSession(new SessionData(sessionid, username, new String[0], password, "CrudServiceLogin"));
202
            AuthToken token = new AuthToken(sessionid);
203 5383 berkley
            EventLog.getInstance().log(metacatUrl,
204 5384 berkley
                    username, null, "authenticate");
205 5436 berkley
            logCrud.info("authenticate");
206 5370 berkley
            return token;
207
        }
208
        catch(Exception e)
209
        {
210 5624 berkley
            throw new ServiceFailure("1620", "Error authenticating with metacat: " + e.getMessage());
211 5370 berkley
        }
212
    }
213
214
    /**
215 5337 berkley
     * set the parameter values needed for this request
216
     */
217
    public void setParameter(String name, String[] value)
218
    {
219
        params.put(name, value);
220
    }
221
222 5377 berkley
    /**
223
     * Generate SystemMetadata for any object in the object store that does
224
     * not already have it.  SystemMetadata documents themselves, are, of course,
225
     * exempt.  This is a utility method for migration of existing object
226
     * stores to DataONE where SystemMetadata is required for all objects.  See
227
     * https://trac.dataone.org/ticket/591
228 5378 berkley
     *
229
     * @param token an authtoken with appropriate permissions to read all
230
     * documents in the object store.  To work correctly, this should probably
231
     * be an adminstrative credential.
232 5377 berkley
     */
233 5378 berkley
    public void generateMissingSystemMetadata(AuthToken token)
234 5377 berkley
    {
235 5378 berkley
        IdentifierManager im = IdentifierManager.getInstance();
236 5377 berkley
        //get the list of ids with no SM
237 5378 berkley
        List<String> l = im.getLocalIdsWithNoSystemMetadata();
238 5377 berkley
        for(int i=0; i<l.size(); i++)
239
        { //for each id, add a system metadata doc
240 5378 berkley
            String localId = l.get(i);
241 5796 berkley
            System.out.println("Creating SystemMetadata for localId " + localId);
242 5378 berkley
            //get the document
243
            try
244
            {
245
                //generate required system metadata fields from the document
246
                SystemMetadata sm = createSystemMetadata(localId, token);
247
                //insert the systemmetadata object
248 5379 berkley
                SessionData sessionData = getSessionData(token);
249 5798 berkley
                String smlocalid = insertSystemMetadata(sm, sessionData);
250
                System.out.println("setting access on SM doc with localid " + smlocalid);
251
                handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid,
252
                        "public", "4", "allow", "allowFirst");
253
254 5431 berkley
                String username = "public";
255
                if(sessionData != null)
256
                {
257
                    username = sessionData.getUserName();
258
                }
259 5383 berkley
                EventLog.getInstance().log(metacatUrl,
260 5384 berkley
                        username, localId, "generateMissingSystemMetadata");
261 5378 berkley
            }
262
            catch(Exception e)
263
            {
264 5798 berkley
                e.printStackTrace();
265 5378 berkley
                System.out.println("Exception generating missing system metadata: " + e.getMessage());
266
                logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
267
            }
268 5377 berkley
        }
269 5392 berkley
        logCrud.info("generateMissingSystemMetadata");
270 5377 berkley
    }
271
272 5378 berkley
    /**
273 5379 berkley
     * create an object via the crud interface
274 5378 berkley
     */
275 5320 jones
    public Identifier create(AuthToken token, Identifier guid,
276 5298 jones
            InputStream object, SystemMetadata sysmeta) throws InvalidToken,
277
            ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType,
278
            InsufficientResources, InvalidSystemMetadata, NotImplemented {
279 5319 jones
        logMetacat.debug("Starting CrudService.create()...");
280
281 5329 jones
        // authenticate & get user info
282
        SessionData sessionData = getSessionData(token);
283 5431 berkley
        String username = "public";
284
        String[] groups = null;
285
        if(sessionData != null)
286
        {
287
            username = sessionData.getUserName();
288
            groups = sessionData.getGroupNames();
289
        }
290 5383 berkley
        String localId = null;
291 5319 jones
292 5376 berkley
        if (username == null || username.equals("public"))
293
        {
294 5466 berkley
            //TODO: many of the thrown exceptions do not use the correct error codes
295
            //check these against the docs and correct them
296 5624 berkley
            throw new NotAuthorized("1100", "User " + username + " is not authorized to create content." +
297 5376 berkley
                    "  If you are not logged in, please do so and retry the request.");
298
        }
299
300 5319 jones
        // verify that guid == SystemMetadata.getIdentifier()
301 5320 jones
        logMetacat.debug("Comparing guid|sysmeta_guid: " + guid.getValue() + "|" + sysmeta.getIdentifier().getValue());
302 5329 jones
        if (!guid.getValue().equals(sysmeta.getIdentifier().getValue())) {
303 5356 berkley
            throw new InvalidSystemMetadata("1180",
304 5750 berkley
                "GUID in method call (" + guid.getValue() + ") does not match GUID in system metadata (" +
305
                sysmeta.getIdentifier().getValue() + ").");
306 5329 jones
        }
307 5319 jones
308
        logMetacat.debug("Checking if identifier exists...");
309
        // Check that the identifier does not already exist
310 5436 berkley
        IdentifierManager im = IdentifierManager.getInstance();
311 5320 jones
        if (im.identifierExists(guid.getValue())) {
312 5356 berkley
            throw new IdentifierNotUnique("1120",
313 5319 jones
                "GUID is already in use by an existing object.");
314
        }
315
316 5329 jones
        // Check if we are handling metadata or data
317
        boolean isScienceMetadata = isScienceMetadata(sysmeta);
318 5319 jones
319 5329 jones
        if (isScienceMetadata) {
320 5331 jones
            // CASE METADATA:
321
            try {
322 5466 berkley
                //System.out.println("CrudService: inserting document with guid " + guid.getValue());
323 5350 berkley
                this.insertDocument(object, guid, sessionData);
324 5383 berkley
                localId = im.getLocalId(guid.getValue());
325 5331 jones
            } catch (IOException e) {
326
                String msg = "Could not create string from XML stream: " +
327
                    " " + e.getMessage();
328
                logMetacat.debug(msg);
329 5356 berkley
                throw new ServiceFailure("1190", msg);
330 5383 berkley
            } catch(Exception e) {
331
                String msg = "Unexpected error in CrudService.create: " + e.getMessage();
332
                logMetacat.debug(msg);
333
                throw new ServiceFailure("1190", msg);
334 5331 jones
            }
335 5383 berkley
336 5319 jones
337 5329 jones
        } else {
338 5331 jones
            // DEFAULT CASE: DATA (needs to be checked and completed)
339 5460 berkley
            localId = insertDataObject(object, guid, sessionData);
340 5331 jones
341
        }
342 5319 jones
343 5331 jones
        // For Metadata and Data, insert the system metadata into the object store too
344 5441 berkley
        String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);
345
        //get the document info.  add any access params for the sysmeta too
346 5466 berkley
        //System.out.println("looking for access records to add for system " +
347
        //    "metadata who's parent doc's  local id is " + localId);
348 5441 berkley
        try
349
        {
350 5444 berkley
            Hashtable<String, Object> h = im.getDocumentInfo(localId.substring(0, localId.lastIndexOf(".")));
351 5441 berkley
            Vector v = (Vector)h.get("access");
352
            for(int i=0; i<v.size(); i++)
353
            {
354
                Hashtable ah = (Hashtable)v.elementAt(i);
355
                String principal = (String)ah.get("principal_name");
356
                String permission = (String)ah.get("permission");
357
                String permissionType = (String)ah.get("permission_type");
358
                String permissionOrder = (String)ah.get("permission_order");
359 5443 berkley
                int perm = new Integer(permission).intValue();
360 5466 berkley
                //System.out.println("found access record for principal " + principal);
361
                //System.out.println("permission: " + perm + " perm_type: " + permissionType +
362
                //    " perm_order: " + permissionOrder);
363 5444 berkley
                this.setAccess(token, guid, principal, perm, permissionType, permissionOrder, true);
364 5441 berkley
            }
365
        }
366
        catch(Exception e)
367
        {
368
            logMetacat.error("Error setting permissions on System Metadata object " +
369
                    " with id " + sysMetaLocalId + ": " + e.getMessage());
370 5466 berkley
            //TODO: decide if this error should cancel the entire create or
371
            //if it should continue with just a logged error.
372 5441 berkley
        }
373
374
375 5319 jones
        logMetacat.debug("Returning from CrudService.create()");
376 5383 berkley
        EventLog.getInstance().log(metacatUrl,
377 5384 berkley
                username, localId, "create");
378 5459 berkley
        logCrud.info("create D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId +
379 5582 berkley
                ":D1SYSMETADATA:"+ sysMetaLocalId + ":");
380 5319 jones
        return guid;
381 5298 jones
    }
382 5353 berkley
383
    /**
384
     * update an existing object with a new object.  Change the system metadata
385
     * to reflect the changes and update it as well.
386
     */
387
    public Identifier update(AuthToken token, Identifier guid,
388
            InputStream object, Identifier obsoletedGuid, SystemMetadata sysmeta)
389
            throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique,
390
            UnsupportedType, InsufficientResources, NotFound, InvalidSystemMetadata,
391
            NotImplemented {
392
        try
393
        {
394
            SessionData sessionData = getSessionData(token);
395
396
            //find the old systemmetadata (sm.old) document id (the one linked to obsoletedGuid)
397
            SystemMetadata sm = getSystemMetadata(token, obsoletedGuid);
398
            //change sm.old's obsoletedBy field
399
            List l = sm.getObsoletedByList();
400
            l.add(guid);
401
            sm.setObsoletedByList(l);
402
            //update sm.old
403
            updateSystemMetadata(sm, sessionData);
404
405
            //change the obsoletes field of the new systemMetadata (sm.new) to point to the id of the old one
406
            sysmeta.addObsolete(obsoletedGuid);
407
            //insert sm.new
408 5459 berkley
            String sysMetaLocalId = insertSystemMetadata(sysmeta, sessionData);
409
            String localId;
410 5353 berkley
411
            boolean isScienceMetadata = isScienceMetadata(sysmeta);
412
            if(isScienceMetadata)
413
            {
414
                //update the doc
415 5459 berkley
                localId = updateDocument(object, obsoletedGuid, guid, sessionData, false);
416 5353 berkley
            }
417
            else
418
            {
419
                //update a data file, not xml
420 5459 berkley
                localId = insertDataObject(object, guid, sessionData);
421 5353 berkley
            }
422 5383 berkley
423
            IdentifierManager im = IdentifierManager.getInstance();
424 5431 berkley
            String username = "public";
425
            if(sessionData != null)
426
            {
427
                username = sessionData.getUserName();
428
            }
429 5383 berkley
            EventLog.getInstance().log(metacatUrl,
430 5384 berkley
                    username, im.getLocalId(guid.getValue()), "update");
431 5459 berkley
            logCrud.info("update D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId +
432 5582 berkley
                    ":D1SYSMETADATA:"+ sysMetaLocalId + ":");
433 5353 berkley
            return guid;
434
        }
435
        catch(Exception e)
436
        {
437 5624 berkley
            throw new ServiceFailure("1310", "Error updating document in CrudService: " + e.getMessage());
438 5353 berkley
        }
439
    }
440 5359 berkley
441
    /**
442 5521 berkley
     * set access permissions on both the science metadata and system metadata
443
     */
444
    public void setAccess(AuthToken token, Identifier id, String principal, String permission,
445
            String permissionType, String permissionOrder)
446
      throws ServiceFailure
447
    {
448
        setAccess(token, id, principal, permission, permissionType, permissionOrder, true);
449
    }
450
451
    /**
452 5443 berkley
     * set access control on the doc
453
     * @param token
454
     * @param id
455
     * @param principal
456
     * @param permission
457
     */
458
    public void setAccess(AuthToken token, Identifier id, String principal, int permission,
459
      String permissionType, String permissionOrder, boolean setSystemMetadata)
460
      throws ServiceFailure
461
    {
462
        String perm = "";
463 5445 berkley
        if(permission >= 4)
464 5443 berkley
        {
465
            perm = "read";
466
        }
467 5445 berkley
        if(permission >= 6)
468 5443 berkley
        {
469
            perm = "write";
470
        }
471 5466 berkley
        //System.out.println("perm in setAccess: " + perm);
472
        //System.out.println("permission in setAccess: " + permission);
473 5443 berkley
        setAccess(token, id, principal, perm, permissionType, permissionOrder,
474
                setSystemMetadata);
475
476
    }
477
478
    /**
479 5362 berkley
     * set the permission on the document
480
     * @param token
481
     * @param principal
482
     * @param permission
483
     * @param permissionType
484
     * @param permissionOrder
485
     * @return
486
     */
487
    public void setAccess(AuthToken token, Identifier id, String principal, String permission,
488 5424 berkley
            String permissionType, String permissionOrder, boolean setSystemMetadata)
489 5362 berkley
      throws ServiceFailure
490
    {
491 5466 berkley
        /* TODO:
492
         * This is also not part of the D1 Crud spec.  This method is needed for
493
         * systems such as metacat where access to objects is controlled by
494
         * and ACL.  Higher level decisions need to be made about how this
495
         * should work within D1.
496
         */
497 5362 berkley
        try
498
        {
499 5449 berkley
            final SessionData sessionData = getSessionData(token);
500
            if(sessionData == null)
501
            {
502
                throw new ServiceFailure("1000", "User must be logged in to set access.");
503
            }
504 5362 berkley
            IdentifierManager im = IdentifierManager.getInstance();
505
            String docid = im.getLocalId(id.getValue());
506 5449 berkley
507 5362 berkley
            String permNum = "0";
508 5370 berkley
            if(permission.equals("read"))
509 5362 berkley
            {
510
                permNum = "4";
511
            }
512 5370 berkley
            else if(permission.equals("write"))
513 5362 berkley
            {
514
                permNum = "6";
515
            }
516 5424 berkley
            System.out.println("user " + sessionData.getUserName() +
517
                    " is setting access level " + permNum + " for permission " +
518
                    permissionType + " on doc with localid " + docid);
519 5362 berkley
            handler.setAccess(metacatUrl, sessionData.getUserName(), docid,
520
                    principal, permNum, permissionType, permissionOrder);
521 5424 berkley
            if(setSystemMetadata)
522
            {
523
                //set the same perms on the system metadata doc
524
                String smlocalid = im.getSystemMetadataLocalId(id.getValue());
525
                System.out.println("setting access on SM doc with localid " + smlocalid);
526
                //cs.setAccess(token, smid, principal, permission, permissionType, permissionOrder);
527
                handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid,
528
                        principal, permNum, permissionType, permissionOrder);
529
            }
530 5431 berkley
            String username = "public";
531
            if(sessionData != null)
532
            {
533
                username = sessionData.getUserName();
534
            }
535 5383 berkley
            EventLog.getInstance().log(metacatUrl,
536 5384 berkley
                    username, im.getLocalId(id.getValue()), "setAccess");
537 5392 berkley
            logCrud.info("setAccess");
538 5362 berkley
        }
539
        catch(Exception e)
540
        {
541 5370 berkley
            e.printStackTrace();
542 5362 berkley
            throw new ServiceFailure("1000", "Could not set access on the document with id " + id.getValue());
543
        }
544
    }
545
546
    /**
547 5359 berkley
     *  Retrieve the list of objects present on the MN that match the calling
548
     *  parameters. This method is required to support the process of Member
549
     *  Node synchronization. At a minimum, this method should be able to
550
     *  return a list of objects that match:
551
     *  startTime <= SystemMetadata.dateSysMetadataModified
552
     *  but is expected to also support date range (by also specifying endTime),
553
     *  and should also support slicing of the matching set of records by
554
     *  indicating the starting index of the response (where 0 is the index
555
     *  of the first item) and the count of elements to be returned.
556
     *
557 5364 berkley
     *  If startTime or endTime is null, the query is not restricted by that parameter.
558
     *
559 5359 berkley
     * @see http://mule1.dataone.org/ArchitectureDocs/mn_api_replication.html#MN_replication.listObjects
560
     * @param token
561
     * @param startTime
562
     * @param endTime
563
     * @param objectFormat
564
     * @param replicaStatus
565
     * @param start
566
     * @param count
567
     * @return ObjectList
568
     * @throws NotAuthorized
569
     * @throws InvalidRequest
570
     * @throws NotImplemented
571
     * @throws ServiceFailure
572
     * @throws InvalidToken
573
     */
574
    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime,
575
        ObjectFormat objectFormat, boolean replicaStatus, int start, int count)
576
      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken
577
    {
578
      ObjectList ol = new ObjectList();
579 5362 berkley
      final SessionData sessionData = getSessionData(token);
580 5413 berkley
      int totalAfterQuery = 0;
581 5510 berkley
582 5362 berkley
      try
583
      {
584 5510 berkley
          if (PropertyService.getProperty("database.queryCacheOn").equals("true"))
585
          {
586
              //System.out.println("the string stored into cache is "+ resultsetBuffer.toString());
587
              DBQuery.clearQueryResultCache();
588
          }
589
      }
590
      catch (PropertyNotFoundException e1)
591
      {
592
          //just don't do anything
593
      }
594
595
      try
596
      {
597 5466 berkley
          //TODO: Milliseconds need to be added to the dateFormat
598 5430 berkley
          System.out.println("=========== Listing Objects =============");
599
          System.out.println("Current server time is: " + new Date());
600
          if(startTime != null)
601
          {
602
              System.out.println("query start time is " + startTime);
603
          }
604
          if(endTime != null)
605
          {
606
              System.out.println("query end time is " + endTime);
607
          }
608 5362 berkley
          params.clear();
609 5622 berkley
          params.put("qformat", new String[] {PropertyService.getProperty("crudService.listObjects.QFormat")});
610 5870 berkley
          params.put("action", new String[] {"squery"});
611
          params.put("query", new String[] {createListObjectsPathQueryDocument()});
612 5363 berkley
613 5630 berkley
          /*System.out.println("query is: metacatUrl: " + metacatUrl + " user: " + sessionData.getUserName() +
614
                  " sessionid: " + sessionData.getId() + " params: ");
615
          String url = metacatUrl + "/metacat?action=query&sessionid=" + sessionData.getId();
616 5622 berkley
          Enumeration keys = params.keys();
617
          while(keys.hasMoreElements())
618
          {
619
              String key = (String)keys.nextElement();
620
              String[] parr = params.get(key);
621
              for(int i=0; i<parr.length; i++)
622
              {
623
                  System.out.println("param " + key + ": " + parr[i]);
624 5630 berkley
                  url += "&" + key + "=" + parr[i] ;
625 5622 berkley
              }
626
          }
627 5630 berkley
          System.out.println("query url: " + url);
628
          */
629 5426 berkley
          String username = "public";
630
          String[] groups = null;
631
          String sessionid = "";
632
          if(sessionData != null)
633
          {
634
              username = sessionData.getUserName();
635
              groups = sessionData.getGroupNames();
636
              sessionid = sessionData.getId();
637
          }
638 5408 berkley
639 5426 berkley
          MetacatResultSet rs = handler.query(metacatUrl, params, username,
640
                  groups, sessionid);
641 5363 berkley
          List docs = rs.getDocuments();
642 5403 berkley
643 5510 berkley
          System.out.println("query returned " + docs.size() + " documents.");
644 5413 berkley
          Vector<Document> docCopy = new Vector<Document>();
645 5408 berkley
646 5413 berkley
          //preparse the list to remove any that don't match the query params
647 5466 berkley
          /* TODO: this type of query/subquery processing is probably not scalable
648
           * to larger object stores.  This code should be revisited.  The metacat
649
           * query handler should probably be altered to handle the type of query
650
           * done here.
651
           */
652 5413 berkley
          for(int i=0; i<docs.size(); i++)
653 5369 berkley
          {
654 5363 berkley
              Document d = (Document)docs.get(i);
655 5630 berkley
656 5364 berkley
              ObjectFormat returnedObjectFormat = ObjectFormat.convert(d.getField("objectFormat"));
657 5413 berkley
658
              if(returnedObjectFormat != null &&
659
                 objectFormat != null &&
660
                 !objectFormat.toString().trim().equals(returnedObjectFormat.toString().trim()))
661 5364 berkley
              { //make sure the objectFormat is the one specified
662
                  continue;
663
              }
664 5384 berkley
665
              String dateSMM = d.getField("dateSysMetadataModified");
666 5413 berkley
              if((startTime != null || endTime != null) && dateSMM == null)
667
              {  //if startTime or endTime are not null, we need a date to compare to
668 5384 berkley
                  continue;
669
              }
670 5413 berkley
671
              //date parse
672
              Date dateSysMetadataModified = null;
673
              if(dateSMM != null)
674
              {
675 5643 berkley
676
                  /*
677 5420 berkley
                  if(dateSMM.indexOf(".") != -1)
678
                  {  //strip the milliseconds
679 5466 berkley
                      //TODO: don't do this. we need milliseconds now.
680 5481 berkley
                      //TODO: explore ISO 8601 to figure out milliseconds
681 5420 berkley
                      dateSMM = dateSMM.substring(0, dateSMM.indexOf(".")) + 'Z';
682
                  }
683 5643 berkley
                  */
684 5420 berkley
                  //System.out.println("dateSMM: " + dateSMM);
685 5428 berkley
                  //dateFormat.setTimeZone(TimeZone.getTimeZone("GMT-0"));
686
                  try
687
                  {   //the format we want
688
                      dateSysMetadataModified = dateFormat.parse(dateSMM);
689
                  }
690
                  catch(java.text.ParseException pe)
691
                  {   //try another legacy format
692 5681 berkley
                      try
693
                      {
694
                          DateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss.S'Z'");
695
                          dateFormat2.setTimeZone(TimeZone.getTimeZone("GMT-0"));
696
                          dateSysMetadataModified = dateFormat2.parse(dateSMM);
697
                      }
698
                      catch(java.text.ParseException pe2)
699
                      {
700
                          //try another legacy format
701
                          DateFormat dateFormat3 = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'");
702
                          dateFormat3.setTimeZone(TimeZone.getTimeZone("GMT-0"));
703
                          dateSysMetadataModified = dateFormat3.parse(dateSMM);
704
                      }
705
706 5428 berkley
                  }
707 5413 berkley
              }
708 5428 berkley
709 5510 berkley
              /*System.out.println("====================================");
710
              System.out.println("doc number " + i);
711
              System.out.println("docid: " + d.docid);
712 5630 berkley
              System.out.println("guid: " + d.getField("identifier").trim());
713
              System.out.println("dateSMM: " + dateSMM);
714 5420 berkley
              System.out.println("dateSysMetadataModified: " + dateSysMetadataModified);
715
              System.out.println("startTime: " + startTime);
716
              System.out.println("endtime: " + endTime);*/
717 5427 berkley
718 5364 berkley
              int startDateComparison = 0;
719
              int endDateComparison = 0;
720
              if(startTime != null)
721
              {
722 5427 berkley
                  Calendar zTime = Calendar.getInstance(TimeZone.getTimeZone("GMT-0"));
723
                  zTime.setTime(startTime);
724
                  startTime = zTime.getTime();
725
726 5413 berkley
                  if(dateSysMetadataModified == null)
727
                  {
728
                      startDateComparison = -1;
729
                  }
730
                  else
731
                  {
732
                      startDateComparison = dateSysMetadataModified.compareTo(startTime);
733
                  }
734
                  //System.out.println("startDateCom: " + startDateComparison);
735 5364 berkley
              }
736 5413 berkley
              else
737
              {
738
                  startDateComparison = 1;
739
              }
740 5364 berkley
741
              if(endTime != null)
742
              {
743 5427 berkley
                  Calendar zTime = Calendar.getInstance(TimeZone.getTimeZone("GMT-0"));
744
                  zTime.setTime(endTime);
745
                  endTime = zTime.getTime();
746
747 5413 berkley
                  if(dateSysMetadataModified == null)
748
                  {
749
                      endDateComparison = 1;
750
                  }
751
                  else
752
                  {
753
                      endDateComparison = dateSysMetadataModified.compareTo(endTime);
754
                  }
755
                  //System.out.println("endDateCom: " + endDateComparison);
756 5364 berkley
              }
757 5413 berkley
              else
758
              {
759
                  endDateComparison = -1;
760
              }
761 5364 berkley
762 5413 berkley
763 5364 berkley
              if(startDateComparison < 0 || endDateComparison > 0)
764 5413 berkley
              {
765 5364 berkley
                  continue;
766
              }
767
768 5413 berkley
              docCopy.add((Document)docs.get(i));
769
          } //end pre-parse
770
771
          docs = docCopy;
772
          totalAfterQuery = docs.size();
773 5428 berkley
          //System.out.println("total after subquery: " + totalAfterQuery);
774 5413 berkley
775
          //make sure we don't run over the end
776
          int end = start + count;
777
          if(end > docs.size())
778
          {
779
              end = docs.size();
780
          }
781
782
          for(int i=start; i<end; i++)
783
          {
784
              //get the document from the result
785
              Document d = (Document)docs.get(i);
786
              //System.out.println("processing doc " + d.docid);
787
788
              String dateSMM = d.getField("dateSysMetadataModified");
789 5630 berkley
              //System.out.println("dateSMM: " + dateSMM);
790
              //System.out.println("parsed date: " + parseDate(dateSMM));
791 5413 berkley
              Date dateSysMetadataModified = null;
792
              if(dateSMM != null)
793
              {
794
                  try
795
                  {
796
                      dateSysMetadataModified = parseDate(dateSMM);
797
                  }
798
                  catch(Exception e)
799
                  { //if we fail to parse the date, just ignore the value
800
                      dateSysMetadataModified = null;
801
                  }
802
              }
803
              ObjectFormat returnedObjectFormat = ObjectFormat.convert(d.getField("objectFormat"));
804
805
806 5364 berkley
              ObjectInfo info = new ObjectInfo();
807
              //add the fields to the info object
808
              Checksum cs = new Checksum();
809
              cs.setValue(d.getField("checksum"));
810 5370 berkley
              String csalg = d.getField("algorithm");
811
              if(csalg == null)
812
              {
813
                  csalg = "MD5";
814
              }
815
              ChecksumAlgorithm ca = ChecksumAlgorithm.convert(csalg);
816
              cs.setAlgorithm(ca);
817 5364 berkley
              info.setChecksum(cs);
818
              info.setDateSysMetadataModified(dateSysMetadataModified);
819
              Identifier id = new Identifier();
820 5445 berkley
              id.setValue(d.getField("identifier").trim());
821 5364 berkley
              info.setIdentifier(id);
822
              info.setObjectFormat(returnedObjectFormat);
823 5413 berkley
              String size = d.getField("size");
824
              if(size != null)
825
              {
826
                  info.setSize(new Long(size.trim()).longValue());
827
              }
828 5364 berkley
              //add the ObjectInfo to the ObjectList
829 5557 berkley
              //logCrud.info("objectFormat: " + info.getObjectFormat().toString());
830
              //logCrud.info("id: " + info.getIdentifier().getValue());
831
832 5555 berkley
              if(info.getIdentifier().getValue() != null)
833 5414 berkley
              { //id can be null from tests.  should not happen in production.
834 5555 berkley
                  if((info.getObjectFormat() != null && !info.getObjectFormat().toString().trim().equals("")))
835
                  { //objectFormat needs to not be null and not be an empty string
836
                    ol.addObjectInfo(info);
837
                  }
838 5557 berkley
                  else
839
                  {
840 5559 berkley
                      logCrud.info("Not adding object with null objectFormat" + info.getIdentifier().getValue().toString());
841 5557 berkley
                  }
842 5414 berkley
              }
843 5424 berkley
844 5363 berkley
          }
845 5362 berkley
      }
846
      catch(Exception e)
847
      {
848
          e.printStackTrace();
849 5557 berkley
          logCrud.error("Error creating ObjectList: " + e.getMessage() + " cause: " + e.getCause());
850 5362 berkley
          throw new ServiceFailure("1580", "Error retrieving ObjectList: " + e.getMessage());
851
      }
852 5430 berkley
      String username = "public";
853
      if(sessionData != null)
854
      {
855
          username = sessionData.getUserName();
856
      }
857 5383 berkley
      EventLog.getInstance().log(metacatUrl,
858 5384 berkley
              username, null, "read");
859 5392 berkley
      logCrud.info("listObjects");
860 5577 berkley
      if(totalAfterQuery < count)
861
      {
862
          count = totalAfterQuery;
863
      }
864 5413 berkley
      ol.setCount(count);
865
      ol.setStart(start);
866
      ol.setTotal(totalAfterQuery);
867 5359 berkley
      return ol;
868
    }
869
870
    /**
871
     * Call listObjects with the default values for replicaStatus (true), start (0),
872
     * and count (1000).
873
     * @param token
874
     * @param startTime
875
     * @param endTime
876
     * @param objectFormat
877
     * @return
878
     * @throws NotAuthorized
879
     * @throws InvalidRequest
880
     * @throws NotImplemented
881
     * @throws ServiceFailure
882
     * @throws InvalidToken
883
     */
884
    public ObjectList listObjects(AuthToken token, Date startTime, Date endTime,
885
        ObjectFormat objectFormat)
886
      throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken
887
    {
888
       return listObjects(token, startTime, endTime, objectFormat, true, 0, 1000);
889
    }
890 5298 jones
891 5359 berkley
    /**
892 5654 berkley
     * Delete a document.
893 5359 berkley
     */
894 5320 jones
    public Identifier delete(AuthToken token, Identifier guid)
895 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
896 5654 berkley
            NotImplemented, InvalidRequest {
897 5392 berkley
        logCrud.info("delete");
898 5654 berkley
899
        if(token == null || token.getToken().equals("publid"))
900
        {
901
            throw new NotAuthorized("1320", "You must be logged in to delete records.");
902
        }
903
904
        if(guid == null || guid.getValue().trim().equals(""))
905
        {
906
            throw new InvalidRequest("1322", "No GUID specified in CrudService.delete()");
907
        }
908
        final SessionData sessionData = getSessionData(token);
909
        IdentifierManager manager = IdentifierManager.getInstance();
910
911
        String docid;
912
        try
913
        {
914
            docid = manager.getLocalId(guid.getValue());
915
        }
916
        catch(McdbDocNotFoundException mnfe)
917
        {
918
            throw new InvalidRequest("1322", "GUID " + guid + " not found.");
919
        }
920
921
        try
922
        {
923
            DocumentImpl.delete(docid, sessionData.getUserName(), sessionData.getGroupNames(), null);
924
        }
925
        catch(Exception e)
926
        {
927
            throw new ServiceFailure("1350", "Could not delete document: " + e.getMessage());
928
        }
929
930
        return guid;
931 5298 jones
    }
932
933 5359 berkley
    /**
934 5648 berkley
     * describe a document.
935 5359 berkley
     */
936 5320 jones
    public DescribeResponse describe(AuthToken token, Identifier guid)
937 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
938 5648 berkley
            NotImplemented, InvalidRequest {
939 5392 berkley
        logCrud.info("describe");
940 5648 berkley
941
        if(token == null)
942
        {
943
            throw new InvalidToken("1370", "Authentication token is null");
944
        }
945
946
        if(guid == null || guid.getValue().trim().equals(""))
947
        {
948
            throw new InvalidRequest("1362", "Guid is null.  A valid guid is required.");
949
        }
950
951
        SystemMetadata sm = getSystemMetadata(token, guid);
952
        DescribeResponse dr = new DescribeResponse(sm.getObjectFormat(),
953
                sm.getSize(), sm.getDateSysMetadataModified(), sm.getChecksum());
954
        return dr;
955 5298 jones
    }
956
957 5359 berkley
    /**
958
     * get a document with a specified guid.
959
     */
960 5320 jones
    public InputStream get(AuthToken token, Identifier guid)
961 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
962
            NotImplemented {
963
964
        // Retrieve the session information from the AuthToken
965
        // If the session is expired, then the user is 'public'
966 5436 berkley
        if(token == null)
967
        {
968
            token = new AuthToken("Public");
969
        }
970 5298 jones
        final SessionData sessionData = getSessionData(token);
971
972
        // Look up the localId for this global identifier
973
        IdentifierManager im = IdentifierManager.getInstance();
974 5683 berkley
975 5679 berkley
        try
976
        {
977 5320 jones
            final String localId = im.getLocalId(guid.getValue());
978 5683 berkley
            InputStream objectStream;
979 5679 berkley
            try
980
            {
981
                String username = "public";
982
                String[] groups = new String[0];
983
                if(sessionData != null)
984
                {
985
                    username = sessionData.getUserName();
986
                    groups = sessionData.getGroupNames();
987
                }
988 5298 jones
989 5683 berkley
                objectStream = readFromMetacat(localId, username, groups);
990 5679 berkley
991
            } catch (PropertyNotFoundException e) {
992
                e.printStackTrace();
993
                throw new ServiceFailure("1030", "Error getting property from metacat: " + e.getMessage());
994
            } catch (ClassNotFoundException e) {
995
                e.printStackTrace();
996
                throw new ServiceFailure("1030", "Class not found error when reading from metacat: " + e.getMessage());
997
            } catch (IOException e) {
998
                e.printStackTrace();
999
                throw new ServiceFailure("1030", "IOException while reading from metacat: " + e.getMessage());
1000
            } catch (SQLException e) {
1001
                e.printStackTrace();
1002
                throw new ServiceFailure("1030", "SQLException while reading from metacat: " + e.getMessage());
1003
            } catch (McdbException e) {
1004
                e.printStackTrace();
1005
                throw new ServiceFailure("1030", "Metacat DB exception while reading from metacat: " + e.getMessage());
1006
            } catch (ParseLSIDException e) {
1007
                e.printStackTrace();
1008
                throw new NotFound("1020", "LSID parsing exception while reading from metacat: " + e.getMessage());
1009
            } catch (InsufficientKarmaException e) {
1010
                e.printStackTrace();
1011
                throw new NotAuthorized("1000", "User not authorized for get(): " + e.getMessage());
1012
            }
1013
1014
1015 5431 berkley
            String username = "public";
1016
            if(sessionData != null)
1017
            {
1018
                username = sessionData.getUserName();
1019
            }
1020
1021 5383 berkley
            EventLog.getInstance().log(metacatUrl,
1022 5384 berkley
                    username, im.getLocalId(guid.getValue()), "read");
1023 5459 berkley
            logCrud.info("get D1GUID:" + guid.getValue() + ":D1SCIMETADATA:" + localId +
1024
                    ":");
1025 5680 berkley
1026 5298 jones
            return objectStream;
1027 5679 berkley
        }
1028
        catch (McdbDocNotFoundException e)
1029
        {
1030 5356 berkley
            throw new NotFound("1020", e.getMessage());
1031 5621 berkley
        }
1032 5298 jones
    }
1033
1034 5359 berkley
    /**
1035 5621 berkley
     * get the checksum for a document.  defaults to MD5.
1036 5359 berkley
     */
1037 5320 jones
    public Checksum getChecksum(AuthToken token, Identifier guid)
1038 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
1039 5621 berkley
            InvalidRequest, NotImplemented
1040
    {
1041 5392 berkley
        logCrud.info("getChecksum");
1042 5621 berkley
        return getChecksum(token, guid, "MD5");
1043 5298 jones
    }
1044
1045 5359 berkley
    /**
1046 5621 berkley
     * get the checksum for a document with the given algorithm
1047 5359 berkley
     */
1048 5320 jones
    public Checksum getChecksum(AuthToken token, Identifier guid,
1049 5298 jones
            String checksumAlgorithm) throws InvalidToken, ServiceFailure,
1050 5621 berkley
            NotAuthorized, NotFound, InvalidRequest, NotImplemented
1051
    {
1052 5392 berkley
        logCrud.info("getChecksum");
1053 5670 berkley
        SystemMetadata sm = getSystemMetadata(token, guid);
1054
        Checksum cs = sm.getChecksum();
1055
        if(cs.getAlgorithm().toString().equals(checksumAlgorithm))
1056 5644 berkley
        {
1057 5670 berkley
            return cs;
1058 5644 berkley
        }
1059 5670 berkley
        else
1060 5621 berkley
        {
1061 5670 berkley
            if(checksumAlgorithm == null)
1062
            {
1063
                checksumAlgorithm = "MD5";
1064
            }
1065
            InputStream docStream = get(token, guid);
1066
            String checksum;
1067
            try
1068
            {
1069
                checksum = checksum(docStream, checksumAlgorithm);
1070
            }
1071
            catch(Exception e)
1072
            {
1073
                throw new ServiceFailure("1410", "Error getting checksum: " + e.getMessage());
1074
            }
1075
            Checksum c = new Checksum();
1076
            c.setAlgorithm(ChecksumAlgorithm.convert(checksumAlgorithm));
1077
            c.setValue(checksum);
1078
            return c;
1079 5621 berkley
        }
1080 5298 jones
    }
1081
1082 5359 berkley
    /**
1083 5384 berkley
     * get log records.
1084 5359 berkley
     */
1085 5390 berkley
    public Log getLogRecords(AuthToken token, Date fromDate, Date toDate, Event event)
1086 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest,
1087 5384 berkley
            NotImplemented
1088
    {
1089 5630 berkley
        /*System.out.println("=================== Getting log records ===================");
1090 5430 berkley
        System.out.println("Current server time is: " + new Date());
1091
        if(fromDate != null)
1092
        {
1093
          System.out.println("query start time is " + fromDate);
1094
        }
1095
        if(toDate != null)
1096
        {
1097
          System.out.println("query end time is " + toDate);
1098 5630 berkley
        }*/
1099 5389 berkley
        Log log = new Log();
1100
        Vector<LogEntry> logs = new Vector<LogEntry>();
1101 5384 berkley
        IdentifierManager im = IdentifierManager.getInstance();
1102
        EventLog el = EventLog.getInstance();
1103 5391 berkley
        if(fromDate == null)
1104
        {
1105 5630 berkley
            //System.out.println("setting fromdate from null");
1106 5391 berkley
            fromDate = new Date(1);
1107
        }
1108
        if(toDate == null)
1109
        {
1110 5630 berkley
            //System.out.println("setting todate from null");
1111 5391 berkley
            toDate = new Date();
1112
        }
1113 5418 berkley
1114 5630 berkley
        //System.out.println("fromDate: " + fromDate);
1115
        //System.out.println("toDate: " + toDate);
1116 5418 berkley
1117 5384 berkley
        String report = el.getReport(null, null, null, null,
1118
                new java.sql.Timestamp(fromDate.getTime()),
1119 5693 leinfelder
                new java.sql.Timestamp(toDate.getTime()), false);
1120 5384 berkley
1121 5418 berkley
        //System.out.println("report: " + report);
1122
1123 5384 berkley
        String logEntry = "<logEntry>";
1124
        String endLogEntry = "</logEntry>";
1125
        int startIndex = 0;
1126
        int foundIndex = report.indexOf(logEntry, startIndex);
1127
        while(foundIndex != -1)
1128
        {
1129
            //parse out each entry
1130
            int endEntryIndex = report.indexOf(endLogEntry, foundIndex);
1131
            String entry = report.substring(foundIndex, endEntryIndex);
1132
            //System.out.println("entry: " + entry);
1133
            startIndex = endEntryIndex + endLogEntry.length();
1134
            foundIndex = report.indexOf(logEntry, startIndex);
1135
1136
            String entryId = getLogEntryField("entryid", entry);
1137
            String ipAddress = getLogEntryField("ipAddress", entry);
1138
            String principal = getLogEntryField("principal", entry);
1139
            String docid = getLogEntryField("docid", entry);
1140 5390 berkley
            String eventS = getLogEntryField("event", entry);
1141 5384 berkley
            String dateLogged = getLogEntryField("dateLogged", entry);
1142
1143 5389 berkley
            LogEntry le = new LogEntry();
1144 5384 berkley
1145 5390 berkley
            Event e = Event.convert(eventS);
1146 5384 berkley
            if(e == null)
1147
            { //skip any events that are not Dataone Crud events
1148
                continue;
1149
            }
1150 5389 berkley
            le.setEvent(e);
1151 5384 berkley
            Identifier entryid = new Identifier();
1152
            entryid.setValue(entryId);
1153 5389 berkley
            le.setEntryId(entryid);
1154 5384 berkley
            Identifier identifier = new Identifier();
1155
            try
1156
            {
1157 5432 berkley
                //System.out.println("converting docid '" + docid + "' to a guid.");
1158 5417 berkley
                if(docid == null || docid.trim().equals("") || docid.trim().equals("null"))
1159
                {
1160
                    continue;
1161
                }
1162 5416 berkley
                docid = docid.substring(0, docid.lastIndexOf("."));
1163 5384 berkley
                identifier.setValue(im.getGUID(docid, im.getLatestRevForLocalId(docid)));
1164
            }
1165
            catch(Exception ex)
1166
            { //try to get the guid, if that doesn't work, just use the local id
1167 5419 berkley
                //throw new ServiceFailure("1030", "Error getting guid for localId " +
1168 5424 berkley
                //        docid + ": " + ex.getMessage());\
1169
1170
                //skip it if the guid can't be found
1171 5419 berkley
                continue;
1172 5384 berkley
            }
1173
1174 5389 berkley
            le.setIdentifier(identifier);
1175
            le.setIpAddress(ipAddress);
1176 5384 berkley
            Calendar c = Calendar.getInstance();
1177
            String year = dateLogged.substring(0, 4);
1178
            String month = dateLogged.substring(5, 7);
1179
            String date = dateLogged.substring(8, 10);
1180
            //System.out.println("year: " + year + " month: " + month + " day: " + date);
1181
            c.set(new Integer(year).intValue(), new Integer(month).intValue(), new Integer(date).intValue());
1182
            Date logDate = c.getTime();
1183 5389 berkley
            le.setDateLogged(logDate);
1184 5384 berkley
            NodeReference memberNode = new NodeReference();
1185
            memberNode.setValue(ipAddress);
1186 5389 berkley
            le.setMemberNode(memberNode);
1187 5384 berkley
            Principal princ = new Principal();
1188
            princ.setValue(principal);
1189 5389 berkley
            le.setPrincipal(princ);
1190
            le.setUserAgent("metacat/RESTService");
1191 5390 berkley
1192
            if(event == null)
1193
            {
1194
                logs.add(le);
1195
            }
1196
1197
            if(event != null &&
1198
               e.toString().toLowerCase().trim().equals(event.toString().toLowerCase().trim()))
1199
            {
1200
              logs.add(le);
1201
            }
1202 5384 berkley
        }
1203
1204 5389 berkley
        log.setLogEntryList(logs);
1205 5392 berkley
        logCrud.info("getLogRecords");
1206 5389 berkley
        return log;
1207 5298 jones
    }
1208 5384 berkley
1209
    /**
1210
     * parse a logEntry and get the relavent field from it
1211
     * @param fieldname
1212
     * @param entry
1213
     * @return
1214
     */
1215
    private String getLogEntryField(String fieldname, String entry)
1216
    {
1217
        String begin = "<" + fieldname + ">";
1218
        String end = "</" + fieldname + ">";
1219
        //System.out.println("looking for " + begin + " and " + end + " in entry " + entry);
1220
        String s = entry.substring(entry.indexOf(begin) + begin.length(), entry.indexOf(end));
1221
        //System.out.println("entry " + fieldname + " : " + s);
1222
        return s;
1223
    }
1224 5298 jones
1225 5359 berkley
    /**
1226
     * get the system metadata for a document with a specified guid.
1227
     */
1228 5481 berkley
public SystemMetadata getSystemMetadata(AuthToken token, Identifier guid)
1229 5298 jones
            throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
1230
            InvalidRequest, NotImplemented {
1231 5319 jones
1232 5332 jones
        logMetacat.debug("CrudService.getSystemMetadata - for guid: " + guid.getValue());
1233
1234
        // Retrieve the session information from the AuthToken
1235
        // If the session is expired, then the user is 'public'
1236
        final SessionData sessionData = getSessionData(token);
1237
1238
        try {
1239
            IdentifierManager im = IdentifierManager.getInstance();
1240 5424 berkley
            final String localId = im.getSystemMetadataLocalId(guid.getValue());
1241 5683 berkley
            InputStream objectStream;
1242 5332 jones
1243 5683 berkley
            try {
1244
                String username = "public";
1245
                String[] groupnames = null;
1246
                if(sessionData != null)
1247
                {
1248
                    username = sessionData.getUserName();
1249
                    groupnames = sessionData.getGroupNames();
1250
                }
1251 5332 jones
1252 5683 berkley
                objectStream = readFromMetacat(localId, username, groupnames);
1253
1254
            } catch (PropertyNotFoundException e) {
1255
                e.printStackTrace();
1256
                throw new ServiceFailure("1090", "Property not found while reading system metadata from metacat: " + e.getMessage());
1257
            } catch (ClassNotFoundException e) {
1258
                e.printStackTrace();
1259
                throw new ServiceFailure("1090", "Class not found while reading system metadata from metacat: " + e.getMessage());
1260
            } catch (IOException e) {
1261
                e.printStackTrace();
1262
                throw new ServiceFailure("1090", "IOException while reading system metadata from metacat: " + e.getMessage());
1263
            } catch (SQLException e) {
1264
                e.printStackTrace();
1265
                throw new ServiceFailure("1090", "SQLException while reading system metadata from metacat: " + e.getMessage());
1266
            } catch (McdbException e) {
1267
                e.printStackTrace();
1268
                throw new ServiceFailure("1090", "Metacat DB Exception while reading system metadata from metacat: " + e.getMessage());
1269
            } catch (ParseLSIDException e) {
1270
                e.printStackTrace();
1271
                throw new NotFound("1060", "Error parsing LSID while reading system metadata from metacat: " + e.getMessage());
1272
            } catch (InsufficientKarmaException e) {
1273
                e.printStackTrace();
1274
                throw new NotAuthorized("1040", "User not authorized for get() on system metadata: " + e.getMessage());
1275
            }
1276 5449 berkley
1277 5332 jones
            // Deserialize the xml to create a SystemMetadata object
1278
            SystemMetadata sysmeta = deserializeSystemMetadata(objectStream);
1279 5431 berkley
            String username = "public";
1280
            if(sessionData != null)
1281
            {
1282
                username = sessionData.getUserName();
1283
            }
1284 5383 berkley
            EventLog.getInstance().log(metacatUrl,
1285 5384 berkley
                    username, im.getLocalId(guid.getValue()), "read");
1286 5459 berkley
            logCrud.info("getsystemmetadata D1GUID:" + guid.getValue()  +
1287 5582 berkley
                    ":D1SYSMETADATA:"+ localId + ":");
1288 5332 jones
            return sysmeta;
1289
1290
        } catch (McdbDocNotFoundException e) {
1291 5348 berkley
            //e.printStackTrace();
1292 5624 berkley
            throw new NotFound("1040", e.getMessage());
1293 5332 jones
        }
1294 5298 jones
    }
1295 5366 berkley
1296
    /**
1297
     * parse the date in the systemMetadata
1298
     * @param s
1299
     * @return
1300
     * @throws Exception
1301
     */
1302 5418 berkley
    public Date parseDate(String s)
1303 5366 berkley
      throws Exception
1304
    {
1305 5466 berkley
        /* TODO:
1306
         * This method should be replaced by a DateFormatter
1307
         */
1308 5366 berkley
        Date d = null;
1309
        int tIndex = s.indexOf("T");
1310
        int zIndex = s.indexOf("Z");
1311
        if(tIndex != -1 && zIndex != -1)
1312
        { //parse a date that looks like 2010-05-18T21:12:54.362Z
1313
            //System.out.println("original date: " + s);
1314
1315
            String date = s.substring(0, tIndex);
1316
            String year = date.substring(0, date.indexOf("-"));
1317
            String month = date.substring(date.indexOf("-") + 1, date.lastIndexOf("-"));
1318
            String day = date.substring(date.lastIndexOf("-") + 1, date.length());
1319
            /*System.out.println("date: " + "year: " + new Integer(year).intValue() +
1320
                    " month: " + new Integer(month).intValue() + " day: " +
1321
                    new Integer(day).intValue());
1322
            */
1323
            String time = s.substring(tIndex + 1, zIndex);
1324
            String hour = time.substring(0, time.indexOf(":"));
1325
            String minute = time.substring(time.indexOf(":") + 1, time.lastIndexOf(":"));
1326
            String seconds = "00";
1327
            String milliseconds = "00";
1328
            if(time.indexOf(".") != -1)
1329
            {
1330
                seconds = time.substring(time.lastIndexOf(":") + 1, time.indexOf("."));
1331
                milliseconds = time.substring(time.indexOf(".") + 1, time.length());
1332
            }
1333 5418 berkley
            else
1334
            {
1335
                seconds = time.substring(time.lastIndexOf(":") + 1, time.length());
1336
            }
1337 5366 berkley
            /*System.out.println("time: " + "hour: " + new Integer(hour).intValue() +
1338
                    " minute: " + new Integer(minute).intValue() + " seconds: " +
1339
                    new Integer(seconds).intValue() + " milli: " +
1340
                    new Integer(milliseconds).intValue());*/
1341
1342
            //d = DateFormat.getDateTimeInstance().parse(date + " " + time);
1343 5630 berkley
            Calendar c = Calendar.getInstance(TimeZone.getTimeZone("GMT-0")/*TimeZone.getDefault()*/);
1344 5366 berkley
            c.set(new Integer(year).intValue(), new Integer(month).intValue() - 1,
1345
                  new Integer(day).intValue(), new Integer(hour).intValue(),
1346
                  new Integer(minute).intValue(), new Integer(seconds).intValue());
1347
            c.set(Calendar.MILLISECOND, new Integer(milliseconds).intValue());
1348
            d = new Date(c.getTimeInMillis());
1349
            //System.out.println("d: " + d);
1350
            return d;
1351
        }
1352
        else
1353
        {  //if it's not in the expected format, try the formatter
1354
            return DateFormat.getDateTimeInstance().parse(s);
1355
        }
1356
    }
1357 5298 jones
1358 5331 jones
    /*
1359
     * Look up the information on the session using the token provided in
1360
     * the AuthToken.  The Session should have all relevant user information.
1361
     * If the session has expired or is invalid, the 'public' session will
1362
     * be returned, giving the user anonymous access.
1363
     */
1364 5394 berkley
    public static SessionData getSessionData(AuthToken token) {
1365 5331 jones
        SessionData sessionData = null;
1366
        String sessionId = "PUBLIC";
1367
        if (token != null) {
1368
            sessionId = token.getToken();
1369
        }
1370
1371
        // if the session id is registered in SessionService, get the
1372
        // SessionData for it. Otherwise, use the public session.
1373 5411 berkley
        //System.out.println("sessionid: " + sessionId);
1374 5403 berkley
        if (sessionId != null &&
1375 5406 berkley
            !sessionId.toLowerCase().equals("public") &&
1376 5403 berkley
            SessionService.getInstance().isSessionRegistered(sessionId))
1377
        {
1378 5374 berkley
            sessionData = SessionService.getInstance().getRegisteredSession(sessionId);
1379 5331 jones
        } else {
1380 5374 berkley
            sessionData = SessionService.getInstance().getPublicSession();
1381 5331 jones
        }
1382 5411 berkley
1383 5331 jones
        return sessionData;
1384
    }
1385
1386
    /**
1387
     * Determine if a given object should be treated as an XML science metadata
1388
     * object.
1389
     *
1390
     * TODO: This test should be externalized in a configuration dictionary rather than being hardcoded.
1391
     *
1392
     * @param sysmeta the SystemMetadata describig the object
1393
     * @return true if the object should be treated as science metadata
1394
     */
1395
    private boolean isScienceMetadata(SystemMetadata sysmeta) {
1396 5523 berkley
        /*boolean scimeta = false;
1397 5466 berkley
        //TODO: this should be read from a .properties file instead of being hard coded
1398 5331 jones
        switch (sysmeta.getObjectFormat()) {
1399
            case EML_2_1_0: scimeta = true; break;
1400
            case EML_2_0_1: scimeta = true; break;
1401
            case EML_2_0_0: scimeta = true; break;
1402
            case FGDC_STD_001_1_1999: scimeta = true; break;
1403
            case FGDC_STD_001_1998: scimeta = true; break;
1404
            case NCML_2_2: scimeta = true; break;
1405 5521 berkley
            case DSPACE_METS_SIP_1_0: scimeta = true; break;
1406 5331 jones
        }
1407
1408 5523 berkley
        return scimeta;*/
1409
1410
        return MetadataTypeRegister.isMetadataType(sysmeta.getObjectFormat());
1411 5331 jones
    }
1412
1413 5359 berkley
    /**
1414
     * insert a data doc
1415
     * @param object
1416
     * @param guid
1417
     * @param sessionData
1418
     * @throws ServiceFailure
1419 5459 berkley
     * @returns localId of the data object inserted
1420 5359 berkley
     */
1421 5459 berkley
    private String insertDataObject(InputStream object, Identifier guid,
1422 5331 jones
            SessionData sessionData) throws ServiceFailure {
1423
1424 5431 berkley
        String username = "public";
1425
        String[] groups = null;
1426
        if(sessionData != null)
1427
        {
1428
          username = sessionData.getUserName();
1429
          groups = sessionData.getGroupNames();
1430
        }
1431 5331 jones
1432
        // generate guid/localId pair for object
1433
        logMetacat.debug("Generating a guid/localId mapping");
1434
        IdentifierManager im = IdentifierManager.getInstance();
1435
        String localId = im.generateLocalId(guid.getValue(), 1);
1436
1437
        try {
1438
            logMetacat.debug("Case DATA: starting to write to disk.");
1439
            if (DocumentImpl.getDataFileLockGrant(localId)) {
1440
1441
                // Save the data file to disk using "localId" as the name
1442
                try {
1443
                    String datafilepath = PropertyService.getProperty("application.datafilepath");
1444
1445
                    File dataDirectory = new File(datafilepath);
1446
                    dataDirectory.mkdirs();
1447
1448
                    File newFile = writeStreamToFile(dataDirectory, localId, object);
1449
1450
                    // TODO: Check that the file size matches SystemMetadata
1451
                    //                        long size = newFile.length();
1452
                    //                        if (size == 0) {
1453
                    //                            throw new IOException("Uploaded file is 0 bytes!");
1454
                    //                        }
1455
1456
                    // Register the file in the database (which generates an exception
1457
                    // if the localId is not acceptable or other untoward things happen
1458
                    try {
1459
                        logMetacat.debug("Registering document...");
1460
                        DocumentImpl.registerDocument(localId, "BIN", localId,
1461
                                username, groups);
1462
                        logMetacat.debug("Registration step completed.");
1463
                    } catch (SQLException e) {
1464
                        //newFile.delete();
1465
                        logMetacat.debug("SQLE: " + e.getMessage());
1466
                        e.printStackTrace(System.out);
1467 5356 berkley
                        throw new ServiceFailure("1190", "Registration failed: " + e.getMessage());
1468 5331 jones
                    } catch (AccessionNumberException e) {
1469
                        //newFile.delete();
1470
                        logMetacat.debug("ANE: " + e.getMessage());
1471
                        e.printStackTrace(System.out);
1472 5356 berkley
                        throw new ServiceFailure("1190", "Registration failed: " + e.getMessage());
1473 5331 jones
                    } catch (Exception e) {
1474
                        //newFile.delete();
1475
                        logMetacat.debug("Exception: " + e.getMessage());
1476
                        e.printStackTrace(System.out);
1477 5356 berkley
                        throw new ServiceFailure("1190", "Registration failed: " + e.getMessage());
1478 5331 jones
                    }
1479
1480
                    logMetacat.debug("Logging the creation event.");
1481 5344 berkley
                    EventLog.getInstance().log(metacatUrl,
1482 5384 berkley
                            username, localId, "create");
1483 5331 jones
1484
                    // Schedule replication for this data file
1485
                    logMetacat.debug("Scheduling replication.");
1486
                    ForceReplicationHandler frh = new ForceReplicationHandler(
1487 5384 berkley
                            localId, "create", false, null);
1488 5331 jones
1489
                } catch (PropertyNotFoundException e) {
1490 5356 berkley
                    throw new ServiceFailure("1190", "Could not lock file for writing:" + e.getMessage());
1491 5331 jones
                }
1492
            }
1493 5459 berkley
            return localId;
1494 5331 jones
        } catch (Exception e) {
1495
            // Could not get a lock on the document, so we can not update the file now
1496 5356 berkley
            throw new ServiceFailure("1190", "Failed to lock file: " + e.getMessage());
1497 5331 jones
        }
1498
    }
1499
1500 5359 berkley
    /**
1501
     * write a file to a stream
1502
     * @param dir
1503
     * @param fileName
1504
     * @param data
1505
     * @return
1506
     * @throws ServiceFailure
1507
     */
1508 5319 jones
    private File writeStreamToFile(File dir, String fileName, InputStream data)
1509
        throws ServiceFailure {
1510
1511
        File newFile = new File(dir, fileName);
1512
        logMetacat.debug("Filename for write is: " + newFile.getAbsolutePath());
1513
1514
        try {
1515
            if (newFile.createNewFile()) {
1516
                // write data stream to desired file
1517
                OutputStream os = new FileOutputStream(newFile);
1518
                long length = IOUtils.copyLarge(data, os);
1519
                os.flush();
1520
                os.close();
1521
            } else {
1522
                logMetacat.debug("File creation failed, or file already exists.");
1523 5356 berkley
                throw new ServiceFailure("1190", "File already exists: " + fileName);
1524 5319 jones
            }
1525
        } catch (FileNotFoundException e) {
1526
            logMetacat.debug("FNF: " + e.getMessage());
1527 5356 berkley
            throw new ServiceFailure("1190", "File not found: " + fileName + " "
1528 5319 jones
                    + e.getMessage());
1529
        } catch (IOException e) {
1530
            logMetacat.debug("IOE: " + e.getMessage());
1531 5356 berkley
            throw new ServiceFailure("1190", "File was not written: " + fileName
1532 5319 jones
                    + " " + e.getMessage());
1533
        }
1534
1535
        return newFile;
1536 5329 jones
    }
1537
1538 5350 berkley
    /**
1539 5441 berkley
     * insert a systemMetadata doc, return the localId of the sysmeta
1540 5350 berkley
     */
1541 5441 berkley
    private String insertSystemMetadata(SystemMetadata sysmeta, SessionData sessionData)
1542 5377 berkley
        throws ServiceFailure
1543
    {
1544 5329 jones
        logMetacat.debug("Starting to insert SystemMetadata...");
1545
1546
        // generate guid/localId pair for sysmeta
1547 5331 jones
        Identifier sysMetaGuid = new Identifier();
1548
        sysMetaGuid.setValue(DocumentUtil.generateDocumentId(1));
1549 5329 jones
        sysmeta.setDateSysMetadataModified(new Date());
1550 5453 berkley
        System.out.println("****inserting new system metadata with modified date " +
1551
                sysmeta.getDateSysMetadataModified());
1552 5329 jones
1553
        String xml = new String(serializeSystemMetadata(sysmeta).toByteArray());
1554 5444 berkley
        System.out.println("sysmeta: " + xml);
1555 5453 berkley
        String localId = insertDocument(xml, sysMetaGuid, sessionData, true);
1556 5420 berkley
        System.out.println("sysmeta inserted with localId " + localId);
1557 5453 berkley
        //insert the system metadata doc id into the systemmetadata table to
1558 5333 berkley
        //link it to the data or metadata document
1559 5377 berkley
        IdentifierManager.getInstance().createSystemMetadataMapping(
1560
                sysmeta.getIdentifier().getValue(), sysMetaGuid.getValue());
1561 5441 berkley
        return localId;
1562 5329 jones
    }
1563
1564 5333 berkley
    /**
1565 5350 berkley
     * update a systemMetadata doc
1566
     */
1567
    private void updateSystemMetadata(SystemMetadata sm, SessionData sessionData)
1568
      throws ServiceFailure
1569
    {
1570
        try
1571
        {
1572 5424 berkley
            String smId = IdentifierManager.getInstance().getSystemMetadataLocalId(sm.getIdentifier().getValue());
1573 5630 berkley
            System.out.println("setting date modified to " + new Date());
1574 5350 berkley
            sm.setDateSysMetadataModified(new Date());
1575
            String xml = new String(serializeSystemMetadata(sm).toByteArray());
1576 5456 berkley
            String localId = updateDocument(xml, sm.getIdentifier(), null, sessionData, true);
1577 5350 berkley
            IdentifierManager.getInstance().updateSystemMetadataMapping(sm.getIdentifier().getValue(), localId);
1578
        }
1579
        catch(Exception e)
1580
        {
1581 5356 berkley
            throw new ServiceFailure("1030", "Error updating system metadata: " + e.getMessage());
1582 5350 berkley
        }
1583
    }
1584
1585 5453 berkley
    private String insertDocument(String xml, Identifier guid, SessionData sessionData)
1586
        throws ServiceFailure
1587
    {
1588
        return insertDocument(xml, guid, sessionData, false);
1589
    }
1590
1591 5350 berkley
    /**
1592
     * insert a document
1593
     * NOTE: this method shouldn't be used from the update or create() methods.
1594
     * we shouldn't be putting the science metadata or data objects into memory.
1595
     */
1596 5453 berkley
    private String insertDocument(String xml, Identifier guid, SessionData sessionData,
1597
            boolean isSystemMetadata)
1598 5350 berkley
        throws ServiceFailure
1599
    {
1600 5453 berkley
        return insertOrUpdateDocument(xml, guid, sessionData, "insert", isSystemMetadata);
1601 5350 berkley
    }
1602
1603
    /**
1604
     * insert a document from a stream
1605
     */
1606
    private String insertDocument(InputStream is, Identifier guid, SessionData sessionData)
1607
      throws IOException, ServiceFailure
1608
    {
1609
        //HACK: change this eventually.  we should not be converting the stream to a string
1610
        String xml = IOUtils.toString(is);
1611
        return insertDocument(xml, guid, sessionData);
1612
    }
1613
1614
    /**
1615
     * update a document
1616
     * NOTE: this method shouldn't be used from the update or create() methods.
1617
     * we shouldn't be putting the science metadata or data objects into memory.
1618
     */
1619 5453 berkley
    private String updateDocument(String xml, Identifier obsoleteGuid,
1620
            Identifier guid, SessionData sessionData, boolean isSystemMetadata)
1621 5350 berkley
        throws ServiceFailure
1622
    {
1623 5453 berkley
        return insertOrUpdateDocument(xml, obsoleteGuid, sessionData, "update", isSystemMetadata);
1624 5350 berkley
    }
1625
1626
    /**
1627
     * update a document from a stream
1628
     */
1629 5453 berkley
    private String updateDocument(InputStream is, Identifier obsoleteGuid,
1630
            Identifier guid, SessionData sessionData, boolean isSystemMetadata)
1631 5350 berkley
      throws IOException, ServiceFailure
1632
    {
1633
        //HACK: change this eventually.  we should not be converting the stream to a string
1634
        String xml = IOUtils.toString(is);
1635 5453 berkley
        String localId = updateDocument(xml, obsoleteGuid, guid, sessionData, isSystemMetadata);
1636 5350 berkley
        IdentifierManager im = IdentifierManager.getInstance();
1637
        if(guid != null)
1638
        {
1639
          im.createMapping(guid.getValue(), localId);
1640
        }
1641
        return localId;
1642
    }
1643
1644
    /**
1645 5333 berkley
     * insert a document, return the id of the document that was inserted
1646
     */
1647 5453 berkley
    protected String insertOrUpdateDocument(String xml, Identifier guid,
1648
            SessionData sessionData, String insertOrUpdate, boolean isSystemMetadata)
1649 5329 jones
        throws ServiceFailure {
1650
        logMetacat.debug("Starting to insert xml document...");
1651
        IdentifierManager im = IdentifierManager.getInstance();
1652
1653
        // generate guid/localId pair for sysmeta
1654 5350 berkley
        String localId = null;
1655
        if(insertOrUpdate.equals("insert"))
1656
        {
1657 5453 berkley
            localId = im.generateLocalId(guid.getValue(), 1, isSystemMetadata);
1658 5350 berkley
        }
1659
        else
1660
        {
1661
            //localid should already exist in the identifier table, so just find it
1662
            try
1663
            {
1664 5456 berkley
                System.out.println("updating guid " + guid.getValue());
1665
                if(!isSystemMetadata)
1666
                {
1667
                    System.out.println("looking in identifier table for guid " + guid.getValue());
1668
                    localId = im.getLocalId(guid.getValue());
1669
                }
1670
                else
1671
                {
1672
                    System.out.println("looking in systemmetadata table for guid " + guid.getValue());
1673
                    localId = im.getSystemMetadataLocalId(guid.getValue());
1674
                }
1675
                System.out.println("localId: " + localId);
1676 5350 berkley
                //increment the revision
1677
                String docid = localId.substring(0, localId.lastIndexOf("."));
1678
                String revS = localId.substring(localId.lastIndexOf(".") + 1, localId.length());
1679
                int rev = new Integer(revS).intValue();
1680
                rev++;
1681
                docid = docid + "." + rev;
1682
                localId = docid;
1683 5456 berkley
                System.out.println("incremented localId: " + localId);
1684 5350 berkley
            }
1685
            catch(McdbDocNotFoundException e)
1686
            {
1687 5356 berkley
                throw new ServiceFailure("1030", "CrudService.insertOrUpdateDocument(): " +
1688 5350 berkley
                    "guid " + guid.getValue() + " should have been in the identifier table, but it wasn't: " + e.getMessage());
1689
            }
1690
        }
1691 5331 jones
        logMetacat.debug("Metadata guid|localId: " + guid.getValue() + "|" +
1692 5329 jones
                localId);
1693
1694
        String[] action = new String[1];
1695 5350 berkley
        action[0] = insertOrUpdate;
1696 5329 jones
        params.put("action", action);
1697
        String[] docid = new String[1];
1698
        docid[0] = localId;
1699
        params.put("docid", docid);
1700
        String[] doctext = new String[1];
1701
        doctext[0] = xml;
1702
        logMetacat.debug(doctext[0]);
1703
        params.put("doctext", doctext);
1704 5331 jones
1705
        // TODO: refactor handleInsertOrUpdateAction() to not output XML directly
1706
        // onto output stream, or alternatively, capture that and parse it to
1707
        // generate the right exceptions
1708 5337 berkley
        //ByteArrayOutputStream output = new ByteArrayOutputStream();
1709
        //PrintWriter pw = new PrintWriter(output);
1710 5449 berkley
        String username = "public";
1711
        String[] groupnames = null;
1712
        if(sessionData != null)
1713
        {
1714
            username = sessionData.getUserName();
1715
            groupnames = sessionData.getGroupNames();
1716
        }
1717 5344 berkley
        String result = handler.handleInsertOrUpdateAction(metacatUrl, null,
1718 5449 berkley
                            null, params, username, groupnames);
1719 5461 berkley
        if(result.indexOf("<error>") != -1)
1720
        {
1721
            throw new ServiceFailure("1000", "Error inserting or updating document: " + result);
1722
        }
1723 5337 berkley
        //String outputS = new String(output.toByteArray());
1724
        logMetacat.debug("CrudService.insertDocument - Metacat returned: " + result);
1725 5333 berkley
        logMetacat.debug("Finsished inserting xml document with id " + localId);
1726
        return localId;
1727 5329 jones
    }
1728
1729 5359 berkley
    /**
1730 5555 berkley
     * serialize a dataone type
1731 5550 berkley
     */
1732 5555 berkley
    private void serializeServiceType(Class type, Object object, OutputStream out)
1733
        throws JiBXException
1734 5550 berkley
    {
1735 5555 berkley
        IBindingFactory bfact = BindingDirectory.getFactory(type);
1736
        IMarshallingContext mctx = bfact.createMarshallingContext();
1737
        mctx.marshalDocument(object, "UTF-8", null, out);
1738 5550 berkley
    }
1739
1740
    /**
1741 5359 berkley
     * serialize a system metadata doc
1742
     * @param sysmeta
1743
     * @return
1744
     * @throws ServiceFailure
1745
     */
1746 5332 jones
    public static ByteArrayOutputStream serializeSystemMetadata(SystemMetadata sysmeta)
1747 5329 jones
        throws ServiceFailure {
1748
        IBindingFactory bfact;
1749
        ByteArrayOutputStream sysmetaOut = null;
1750
        try {
1751
            bfact = BindingDirectory.getFactory(SystemMetadata.class);
1752
            IMarshallingContext mctx = bfact.createMarshallingContext();
1753
            sysmetaOut = new ByteArrayOutputStream();
1754
            mctx.marshalDocument(sysmeta, "UTF-8", null, sysmetaOut);
1755
        } catch (JiBXException e) {
1756 5382 berkley
            e.printStackTrace();
1757 5356 berkley
            throw new ServiceFailure("1190", "Failed to serialize and insert SystemMetadata: " + e.getMessage());
1758 5329 jones
        }
1759
1760
        return sysmetaOut;
1761
    }
1762 5332 jones
1763 5359 berkley
    /**
1764
     * deserialize a system metadata doc
1765
     * @param xml
1766
     * @return
1767
     * @throws ServiceFailure
1768
     */
1769 5332 jones
    public static SystemMetadata deserializeSystemMetadata(InputStream xml)
1770
        throws ServiceFailure {
1771
        try {
1772
            IBindingFactory bfact = BindingDirectory.getFactory(SystemMetadata.class);
1773
            IUnmarshallingContext uctx = bfact.createUnmarshallingContext();
1774
            SystemMetadata sysmeta = (SystemMetadata) uctx.unmarshalDocument(xml, null);
1775
            return sysmeta;
1776
        } catch (JiBXException e) {
1777 5382 berkley
            e.printStackTrace();
1778
            throw new ServiceFailure("1190", "Failed to deserialize and insert SystemMetadata: " + e.getMessage());
1779 5332 jones
        }
1780
    }
1781 5378 berkley
1782
    /**
1783 5683 berkley
     * read a document from metacat and return the InputStream
1784
     *
1785
     * @param localId
1786
     * @param username
1787
     * @param groups
1788
     * @return
1789
     * @throws InsufficientKarmaException
1790
     * @throws ParseLSIDException
1791
     * @throws PropertyNotFoundException
1792
     * @throws McdbException
1793
     * @throws SQLException
1794
     * @throws ClassNotFoundException
1795
     * @throws IOException
1796
     */
1797
    private InputStream readFromMetacat(String localId, String username, String[] groups)
1798
        throws InsufficientKarmaException, ParseLSIDException,
1799
        PropertyNotFoundException, McdbException, SQLException,
1800
        ClassNotFoundException, IOException
1801
    {
1802
        File tmpDir;
1803
        try
1804
        {
1805
            tmpDir = new File(PropertyService.getProperty("application.tempDir"));
1806
        }
1807
        catch(PropertyNotFoundException pnfe)
1808
        {
1809
            logMetacat.error("ResourceHandler.writeMMPPartstoFiles: " +
1810
                    "application.tmpDir not found.  Using /tmp instead.");
1811
            tmpDir = new File("/tmp");
1812
        }
1813
        Date d = new Date();
1814
        final File outputFile = new File(tmpDir, "metacat.output." + d.getTime());
1815
        FileOutputStream dataSink = new FileOutputStream(outputFile);
1816
1817
        handler.readFromMetacat(metacatUrl, null,
1818
                dataSink, localId, "xml",
1819
                username,
1820
                groups, true, params);
1821
1822
        //set a timer to clean up the temp files
1823
        Timer t = new Timer();
1824
        TimerTask tt = new TimerTask() {
1825
            @Override
1826
            public void run()
1827
            {
1828
                outputFile.delete();
1829
            }
1830
        };
1831
        t.schedule(tt, 20000); //schedule after 20 secs
1832
1833
        InputStream objectStream = new FileInputStream(outputFile);
1834
        return objectStream;
1835
    }
1836
1837
    /**
1838 5621 berkley
     * return an MD5 checksum for the stream
1839
     * @param is
1840
     * @return
1841 5378 berkley
     */
1842 5838 berkley
    public static String checksum(InputStream is)
1843 5621 berkley
        throws Exception
1844
    {
1845
        return checksum(is, "MD5");
1846
    }
1847
1848
    /**
1849
     * produce a checksum for item using the given algorithm
1850
     */
1851 5838 berkley
    public static String checksum(InputStream is, String algorithm)
1852 5378 berkley
      throws Exception
1853
    {
1854 5851 berkley
        return ServiceTypeUtil.checksum(is, ChecksumAlgorithm.convert(algorithm)).getValue();
1855 5378 berkley
    }
1856
1857
    /**
1858
     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into
1859
     * a proper date object
1860
     * @param date
1861
     * @return
1862
     */
1863
    private Date parseMetacatDate(String date)
1864
    {
1865
        String year = date.substring(0, 4);
1866
        String month = date.substring(5, 7);
1867
        String day = date.substring(8, 10);
1868 5420 berkley
        Calendar c = Calendar.getInstance(TimeZone.getDefault());
1869 5378 berkley
        c.set(new Integer(year).intValue(),
1870
              new Integer(month).intValue(),
1871
              new Integer(day).intValue());
1872 5420 berkley
        System.out.println("time in parseMetacatDate: " + c.getTime());
1873 5378 berkley
        return c.getTime();
1874
    }
1875
1876
    /**
1877
     * find the size (in bytes) of a stream
1878
     * @param is
1879
     * @return
1880
     * @throws IOException
1881
     */
1882
    private long sizeOfStream(InputStream is)
1883
        throws IOException
1884
    {
1885
        long size = 0;
1886
        byte[] b = new byte[1024];
1887
        int numread = is.read(b, 0, 1024);
1888
        while(numread != -1)
1889
        {
1890
            size += numread;
1891
            numread = is.read(b, 0, 1024);
1892
        }
1893
        return size;
1894
    }
1895 5379 berkley
1896
    /**
1897
     * create system metadata with a specified id, doc and format
1898
     */
1899
    private SystemMetadata createSystemMetadata(String localId, AuthToken token)
1900
      throws Exception
1901
    {
1902
        IdentifierManager im = IdentifierManager.getInstance();
1903 5441 berkley
        Hashtable<String, Object> docInfo = im.getDocumentInfo(localId);
1904 5379 berkley
1905
        //get the document text
1906
        int rev = im.getLatestRevForLocalId(localId);
1907
        Identifier identifier = new Identifier();
1908 5798 berkley
        try
1909
        {
1910
            identifier.setValue(im.getGUID(localId, rev));
1911
        }
1912
        catch(McdbDocNotFoundException mcdbe)
1913
        { //we're creating a new SM doc for a doc that is not in the identifier table
1914
          //so we need to add it to
1915
            System.out.println("No guid in the identifier table.  adding it for " + localId);
1916
            im.createMapping(localId, localId);
1917
            System.out.println("mapping created for " + localId);
1918
            AccessionNumber accNum = new AccessionNumber(localId, "NONE");
1919
            identifier.setValue(im.getGUID(accNum.getDocid(), rev));
1920
        }
1921
1922
        System.out.println("creating system metadata for guid " + identifier.getValue());
1923 5379 berkley
        InputStream is = this.get(token, identifier);
1924
1925
        SystemMetadata sm = new SystemMetadata();
1926
        //set the id
1927
        sm.setIdentifier(identifier);
1928
1929
        //set the object format
1930 5441 berkley
        String doctype = (String)docInfo.get("doctype");
1931
        ObjectFormat format = ObjectFormat.convert((String)docInfo.get("doctype"));
1932 5379 berkley
        if(format == null)
1933
        {
1934
            if(doctype.trim().equals("BIN"))
1935
            {
1936 5521 berkley
                format = ObjectFormat.OCTET_STREAM;
1937 5379 berkley
            }
1938
            else
1939
            {
1940
                format = ObjectFormat.convert("text/plain");
1941
            }
1942
        }
1943
        sm.setObjectFormat(format);
1944
1945
        //create the checksum
1946
        String checksumS = checksum(is);
1947
        ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");
1948
        Checksum checksum = new Checksum();
1949
        checksum.setValue(checksumS);
1950
        checksum.setAlgorithm(ca);
1951
        sm.setChecksum(checksum);
1952
1953
        //set the size
1954
        is = this.get(token, identifier);
1955
        sm.setSize(sizeOfStream(is));
1956
1957
        //submitter
1958
        Principal p = new Principal();
1959 5441 berkley
        p.setValue((String)docInfo.get("user_owner"));
1960 5379 berkley
        sm.setSubmitter(p);
1961
        sm.setRightsHolder(p);
1962
        try
1963
        {
1964 5441 berkley
            Date dateCreated = parseMetacatDate((String)docInfo.get("date_created"));
1965 5379 berkley
            sm.setDateUploaded(dateCreated);
1966 5441 berkley
            Date dateUpdated = parseMetacatDate((String)docInfo.get("date_updated"));
1967 5379 berkley
            sm.setDateSysMetadataModified(dateUpdated);
1968
        }
1969
        catch(Exception e)
1970
        {
1971 5428 berkley
            System.out.println("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
1972 5379 berkley
            Date dateCreated = new Date();
1973
            sm.setDateUploaded(dateCreated);
1974
            Date dateUpdated = new Date();
1975
            sm.setDateSysMetadataModified(dateUpdated);
1976
        }
1977
        NodeReference nr = new NodeReference();
1978 5466 berkley
        //TODO: this should be set to be something more meaningful once the registry is up
1979 5379 berkley
        nr.setValue("metacat");
1980
        sm.setOriginMemberNode(nr);
1981
        sm.setAuthoritativeMemberNode(nr);
1982
        return sm;
1983
    }
1984 5870 berkley
1985
    /**
1986
     * create the listObjects pathQuery document
1987
     */
1988
    private String createListObjectsPathQueryDocument()
1989
        throws PropertyNotFoundException
1990
    {
1991
        String s = "<pathquery>";
1992
        s += "<returndoctype>" + PropertyService.getProperty("crudService.listObjects.ReturnDoctype") + "</returndoctype>";
1993
        s += "<returnfield>" + PropertyService.getProperty("crudService.listObjects.ReturnField.1") + "</returnfield>";
1994
        s += "<returnfield>" + PropertyService.getProperty("crudService.listObjects.ReturnField.2") + "</returnfield>";
1995
        s += "<returnfield>" + PropertyService.getProperty("crudService.listObjects.ReturnField.3") + "</returnfield>";
1996
        s += "<returnfield>" + PropertyService.getProperty("crudService.listObjects.ReturnField.4") + "</returnfield>";
1997
        s += "<returnfield>" + PropertyService.getProperty("crudService.listObjects.ReturnField.5") + "</returnfield>";
1998
        s += "<returnfield>" + PropertyService.getProperty("crudService.listObjects.ReturnField.6") + "</returnfield>";
1999
        s += "<returnfield>" + PropertyService.getProperty("crudService.listObjects.ReturnField.7") + "</returnfield>";
2000
        s += "<querygroup operator=\"UNION\"><queryterm casesensitive=\"false\" searchmode=\"contains\">";
2001
        s += "<value>%</value>";
2002
        s += "<pathexpr>" + PropertyService.getProperty("crudService.listObjects.ReturnField.3") + "</pathexpr>";
2003
        s += "</queryterm></querygroup></pathquery>";
2004
2005
        return s;
2006
    }
2007 5298 jones
}