Project

General

Profile

1 6174 cjones
/**
2
 *  '$RCSfile$'
3
 *  Copyright: 2000-2011 Regents of the University of California and the
4
 *              National Center for Ecological Analysis and Synthesis
5
 *
6
 *   '$Author$'
7
 *     '$Date$'
8
 *
9
 * This program is free software; you can redistribute it and/or modify
10
 * it under the terms of the GNU General Public License as published by
11
 * the Free Software Foundation; either version 2 of the License, or
12
 * (at your option) any later version.
13
 *
14
 * This program is distributed in the hope that it will be useful,
15
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 * GNU General Public License for more details.
18
 *
19
 * You should have received a copy of the GNU General Public License
20
 * along with this program; if not, write to the Free Software
21
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22
 */
23
24
package edu.ucsb.nceas.metacat.dataone;
25
26 6226 cjones
import java.io.File;
27
import java.io.FileNotFoundException;
28
import java.io.FileOutputStream;
29
import java.io.IOException;
30 6174 cjones
import java.io.InputStream;
31 6241 cjones
import java.io.OutputStream;
32 6226 cjones
import java.sql.SQLException;
33 6225 leinfelder
import java.util.ArrayList;
34 6186 leinfelder
import java.util.Calendar;
35 6174 cjones
import java.util.Date;
36 6226 cjones
import java.util.Hashtable;
37 6194 leinfelder
import java.util.List;
38 6226 cjones
import java.util.Timer;
39 6186 leinfelder
import java.util.Vector;
40 6174 cjones
41 6542 leinfelder
import javax.servlet.http.HttpServletRequest;
42 6226 cjones
43 6542 leinfelder
44 6241 cjones
import org.apache.commons.io.IOUtils;
45 6186 leinfelder
import org.apache.log4j.Logger;
46 6865 cjones
import org.dataone.client.CNode;
47
import org.dataone.client.D1Client;
48 6241 cjones
import org.dataone.client.ObjectFormatCache;
49 6371 leinfelder
import org.dataone.service.util.Constants;
50 6944 leinfelder
import org.dataone.service.util.DateTimeMarshaller;
51 7132 cjones
import org.dataone.service.exceptions.BaseException;
52 6241 cjones
import org.dataone.service.exceptions.IdentifierNotUnique;
53
import org.dataone.service.exceptions.InsufficientResources;
54 6174 cjones
import org.dataone.service.exceptions.InvalidRequest;
55 6241 cjones
import org.dataone.service.exceptions.InvalidSystemMetadata;
56 6174 cjones
import org.dataone.service.exceptions.InvalidToken;
57
import org.dataone.service.exceptions.NotAuthorized;
58
import org.dataone.service.exceptions.NotFound;
59
import org.dataone.service.exceptions.NotImplemented;
60
import org.dataone.service.exceptions.ServiceFailure;
61 6241 cjones
import org.dataone.service.exceptions.UnsupportedType;
62 6366 leinfelder
import org.dataone.service.types.v1.AccessRule;
63 6803 leinfelder
import org.dataone.service.types.v1.DescribeResponse;
64 6366 leinfelder
import org.dataone.service.types.v1.Event;
65
import org.dataone.service.types.v1.Identifier;
66
import org.dataone.service.types.v1.Group;
67
import org.dataone.service.types.v1.Log;
68
import org.dataone.service.types.v1.LogEntry;
69 6865 cjones
import org.dataone.service.types.v1.Node;
70 6366 leinfelder
import org.dataone.service.types.v1.NodeReference;
71 6865 cjones
import org.dataone.service.types.v1.NodeType;
72 6366 leinfelder
import org.dataone.service.types.v1.ObjectFormat;
73
import org.dataone.service.types.v1.Permission;
74
import org.dataone.service.types.v1.Person;
75 7132 cjones
import org.dataone.service.types.v1.Replica;
76 6366 leinfelder
import org.dataone.service.types.v1.Session;
77
import org.dataone.service.types.v1.Subject;
78 6532 leinfelder
import org.dataone.service.types.v1.SubjectInfo;
79 6366 leinfelder
import org.dataone.service.types.v1.SystemMetadata;
80 6445 leinfelder
import org.dataone.service.types.v1.util.ChecksumUtil;
81 6174 cjones
82 6241 cjones
import edu.ucsb.nceas.metacat.AccessionNumberException;
83
import edu.ucsb.nceas.metacat.DocumentImpl;
84 6186 leinfelder
import edu.ucsb.nceas.metacat.EventLog;
85
import edu.ucsb.nceas.metacat.IdentifierManager;
86 6194 leinfelder
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
87 6226 cjones
import edu.ucsb.nceas.metacat.MetacatHandler;
88 7077 leinfelder
import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;
89 6803 leinfelder
import edu.ucsb.nceas.metacat.database.DBConnection;
90
import edu.ucsb.nceas.metacat.database.DBConnectionPool;
91 6447 leinfelder
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;
92 6226 cjones
import edu.ucsb.nceas.metacat.properties.PropertyService;
93 6241 cjones
import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;
94 7041 leinfelder
import edu.ucsb.nceas.metacat.util.DocumentUtil;
95 6226 cjones
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
96 6186 leinfelder
97 6174 cjones
public abstract class D1NodeService {
98 6227 cjones
99 6226 cjones
  private static Logger logMetacat = Logger.getLogger(D1NodeService.class);
100 6186 leinfelder
101 6542 leinfelder
  /** For logging the operations */
102
  protected HttpServletRequest request;
103 6241 cjones
104 6226 cjones
  /* reference to the metacat handler */
105 6389 leinfelder
  protected MetacatHandler handler;
106 6174 cjones
107 6226 cjones
  /* parameters set in the incoming request */
108
  private Hashtable<String, String[]> params;
109
110 6241 cjones
  /**
111
   * Constructor - used to set the metacatUrl from a subclass extending D1NodeService
112
   *
113
   * @param metacatUrl - the URL of the metacat service, including the ending /d1
114
   */
115 6542 leinfelder
  public D1NodeService(HttpServletRequest request) {
116
		this.request = request;
117
	}
118 6241 cjones
119
  /**
120 6803 leinfelder
   * This method provides a lighter weight mechanism than
121
   * getSystemMetadata() for a client to determine basic
122
   * properties of the referenced object.
123
   *
124
   * @param session - the Session object containing the credentials for the Subject
125
   * @param pid - the identifier of the object to be described
126
   *
127
   * @return describeResponse - A set of values providing a basic description
128
   *                            of the object.
129
   *
130
   * @throws InvalidToken
131
   * @throws ServiceFailure
132
   * @throws NotAuthorized
133
   * @throws NotFound
134
   * @throws NotImplemented
135
   * @throws InvalidRequest
136
   */
137
  public DescribeResponse describe(Session session, Identifier pid)
138
      throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
139
140
    // get system metadata and construct the describe response
141
      SystemMetadata sysmeta = getSystemMetadata(session, pid);
142
      DescribeResponse describeResponse =
143
      	new DescribeResponse(sysmeta.getFormatId(), sysmeta.getSize(),
144
      			sysmeta.getDateSysMetadataModified(),
145
      			sysmeta.getChecksum(), sysmeta.getSerialVersion());
146
147
      return describeResponse;
148
149
  }
150
151
  /**
152 7077 leinfelder
   * Deletes an object from the Member Node, where the object is either a
153
   * data object or a science metadata object.
154
   *
155
   * @param session - the Session object containing the credentials for the Subject
156
   * @param pid - The object identifier to be deleted
157
   *
158
   * @return pid - the identifier of the object used for the deletion
159
   *
160
   * @throws InvalidToken
161
   * @throws ServiceFailure
162
   * @throws NotAuthorized
163
   * @throws NotFound
164
   * @throws NotImplemented
165
   * @throws InvalidRequest
166
   */
167
  public Identifier delete(Session session, Identifier pid)
168
      throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
169
170
      String localId = null;
171
      boolean allowed = false;
172
      String username = Constants.SUBJECT_PUBLIC;
173
      String[] groupnames = null;
174
      if (session == null) {
175
      	throw new InvalidToken("1330", "No session has been provided");
176
      } else {
177
          username = session.getSubject().getValue();
178
          if (session.getSubjectInfo() != null) {
179
              List<Group> groupList = session.getSubjectInfo().getGroupList();
180
              if (groupList != null) {
181
                  groupnames = new String[groupList.size()];
182
                  for (int i = 0; i > groupList.size(); i++) {
183
                      groupnames[i] = groupList.get(i).getGroupName();
184
                  }
185
              }
186
          }
187
      }
188
189
      // do we have a valid pid?
190
      if (pid == null || pid.getValue().trim().equals("")) {
191
          throw new ServiceFailure("1350", "The provided identifier was invalid.");
192
      }
193
194
      // check for the existing identifier
195
      try {
196
          localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
197
      } catch (McdbDocNotFoundException e) {
198
          throw new NotFound("1340", "The object with the provided " + "identifier was not found.");
199
      }
200
201
      // does the subject have DELETE (a D1 CHANGE_PERMISSION level) priveleges on the pid?
202
      try {
203
			allowed = isAuthorized(session, pid, Permission.CHANGE_PERMISSION);
204
		} catch (InvalidRequest e) {
205
          throw new ServiceFailure("1350", e.getDescription());
206
		}
207
208
209
      if (allowed) {
210
          try {
211
              // delete the document
212
              DocumentImpl.delete(localId, username, groupnames, null);
213
              EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), username, localId, Event.DELETE.xmlValue());
214
215
              // archive it
216
              SystemMetadata sysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
217
              sysMeta.setArchived(true);
218
              HazelcastService.getInstance().getSystemMetadataMap().put(pid, sysMeta);
219
220
              // remove the system metadata for it
221
              //HazelcastService.getInstance().getSystemMetadataMap().remove(pid);
222
223
          } catch (McdbDocNotFoundException e) {
224
              throw new NotFound("1340", "The provided identifier was invalid.");
225
226
          } catch (SQLException e) {
227
              throw new ServiceFailure("1350", "There was a problem deleting the object." + "The error message was: " + e.getMessage());
228
229
          } catch (InsufficientKarmaException e) {
230
              throw new NotAuthorized("1320", "The provided identity does not have " + "permission to DELETE objects on the Member Node.");
231
232
          } catch (Exception e) { // for some reason DocumentImpl throws a general Exception
233
              throw new ServiceFailure("1350", "There was a problem deleting the object." + "The error message was: " + e.getMessage());
234
          }
235
236
      } else {
237
          throw new NotAuthorized("1320", "The provided identity does not have " + "permission to DELETE objects on the Member Node.");
238
      }
239
240
      return pid;
241
  }
242
243
  /**
244 6803 leinfelder
   * Low level, "are you alive" operation. A valid ping response is
245
   * indicated by a HTTP status of 200.
246
   *
247
   * @return true if the service is alive
248
   *
249
   * @throws NotImplemented
250
   * @throws ServiceFailure
251
   * @throws InsufficientResources
252
   */
253
  public Date ping()
254
      throws NotImplemented, ServiceFailure, InsufficientResources {
255
256
      // test if we can get a database connection
257
      int serialNumber = -1;
258
      DBConnection dbConn = null;
259
      try {
260
          dbConn = DBConnectionPool.getDBConnection("MNodeService.ping");
261
          serialNumber = dbConn.getCheckOutSerialNumber();
262
      } catch (SQLException e) {
263
      	ServiceFailure sf = new ServiceFailure("", e.getMessage());
264
      	sf.initCause(e);
265
          throw sf;
266
      } finally {
267
          // Return the database connection
268
          DBConnectionPool.returnDBConnection(dbConn, serialNumber);
269
      }
270
271
      return Calendar.getInstance().getTime();
272
  }
273
274
  /**
275 6241 cjones
   * Adds a new object to the Node, where the object is either a data
276
   * object or a science metadata object. This method is called by clients
277
   * to create new data objects on Member Nodes or internally for Coordinating
278
   * Nodes
279
   *
280
   * @param session - the Session object containing the credentials for the Subject
281
   * @param pid - The object identifier to be created
282
   * @param object - the object bytes
283
   * @param sysmeta - the system metadata that describes the object
284
   *
285
   * @return pid - the object identifier created
286
   *
287
   * @throws InvalidToken
288
   * @throws ServiceFailure
289
   * @throws NotAuthorized
290
   * @throws IdentifierNotUnique
291
   * @throws UnsupportedType
292
   * @throws InsufficientResources
293
   * @throws InvalidSystemMetadata
294
   * @throws NotImplemented
295
   * @throws InvalidRequest
296
   */
297
  public Identifier create(Session session, Identifier pid, InputStream object,
298
    SystemMetadata sysmeta)
299
    throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique,
300
    UnsupportedType, InsufficientResources, InvalidSystemMetadata,
301
    NotImplemented, InvalidRequest {
302 6174 cjones
303 6241 cjones
    Identifier resultPid = null;
304
    String localId = null;
305
    boolean allowed = false;
306
307 6530 leinfelder
    // check for null session
308
    if (session == null) {
309
    	throw new InvalidToken("4894", "Session is required to WRITE to the Node.");
310
    }
311
    Subject subject = session.getSubject();
312
313 6688 leinfelder
    Subject publicSubject = new Subject();
314
    publicSubject.setValue(Constants.SUBJECT_PUBLIC);
315
	// be sure the user is authenticated for create()
316 6518 leinfelder
    if (subject == null || subject.getValue() == null ||
317 6688 leinfelder
        subject.equals(publicSubject) ) {
318 6241 cjones
      throw new NotAuthorized("1100", "The provided identity does not have " +
319 6518 leinfelder
        "permission to WRITE to the Node.");
320 6241 cjones
321
    }
322
323
    // verify that pid == SystemMetadata.getIdentifier()
324
    logMetacat.debug("Comparing pid|sysmeta_pid: " +
325
      pid.getValue() + "|" + sysmeta.getIdentifier().getValue());
326
    if (!pid.getValue().equals(sysmeta.getIdentifier().getValue())) {
327
        throw new InvalidSystemMetadata("1180",
328
            "The supplied system metadata is invalid. " +
329
            "The identifier " + pid.getValue() + " does not match identifier" +
330
            "in the system metadata identified by " +
331
            sysmeta.getIdentifier().getValue() + ".");
332
333
    }
334
335 7122 leinfelder
    logMetacat.debug("Checking if identifier exists: " + pid.getValue());
336 6241 cjones
    // Check that the identifier does not already exist
337 6278 leinfelder
    if (IdentifierManager.getInstance().identifierExists(pid.getValue())) {
338
	    	throw new IdentifierNotUnique("1120",
339
			          "The requested identifier " + pid.getValue() +
340
			          " is already used by another object and" +
341
			          "therefore can not be used for this object. Clients should choose" +
342
			          "a new identifier that is unique and retry the operation or " +
343 6518 leinfelder
			          "use CN.reserveIdentifier() to reserve one.");
344 6337 leinfelder
345 6241 cjones
    }
346 6518 leinfelder
347 6596 leinfelder
    // TODO: this probably needs to be refined more
348 6241 cjones
    try {
349
      allowed = isAuthorized(session, pid, Permission.WRITE);
350
351
    } catch (NotFound e) {
352
      // The identifier doesn't exist, writing should be fine.
353
      allowed = true;
354
    }
355
356 6445 leinfelder
    // verify checksum, only if we can reset the inputstream
357
    if (object.markSupported()) {
358 7122 leinfelder
        logMetacat.debug("Checking checksum for: " + pid.getValue());
359 6445 leinfelder
	    String checksumAlgorithm = sysmeta.getChecksum().getAlgorithm();
360
	    String checksumValue = sysmeta.getChecksum().getValue();
361
	    try {
362
			String computedChecksumValue = ChecksumUtil.checksum(object, checksumAlgorithm).getValue();
363
			// it's very important that we don't consume the stream
364
			object.reset();
365
			if (!computedChecksumValue.equals(checksumValue)) {
366 7122 leinfelder
			    logMetacat.error("Checksum for " + pid.getValue() + " does not match system metadata, computed = " + computedChecksumValue );
367 6445 leinfelder
				throw new InvalidSystemMetadata("4896", "Checksum given does not match that of the object");
368
			}
369
		} catch (Exception e) {
370
			String msg = "Error verifying checksum values";
371
	      	logMetacat.error(msg, e);
372
	        throw new ServiceFailure("1190", msg + ": " + e.getMessage());
373
		}
374
    } else {
375
    	logMetacat.warn("mark is not supported on the object's input stream - cannot verify checksum without consuming stream");
376
    }
377
378 6241 cjones
    // we have the go ahead
379
    if ( allowed ) {
380
381 7122 leinfelder
        logMetacat.debug("Allowed to insert: " + pid.getValue());
382
383 6241 cjones
      // Science metadata (XML) or science data object?
384
      // TODO: there are cases where certain object formats are science metadata
385
      // but are not XML (netCDF ...).  Handle this.
386
      if ( isScienceMetadata(sysmeta) ) {
387
388
        // CASE METADATA:
389
      	String objectAsXML = "";
390
        try {
391
	        objectAsXML = IOUtils.toString(object, "UTF-8");
392
	        localId = insertOrUpdateDocument(objectAsXML, pid, session, "insert");
393
	        //localId = im.getLocalId(pid.getValue());
394
395
        } catch (IOException e) {
396
        	String msg = "The Node is unable to create the object. " +
397
          "There was a problem converting the object to XML";
398
        	logMetacat.info(msg);
399
          throw new ServiceFailure("1190", msg + ": " + e.getMessage());
400
401
        }
402
403 6302 leinfelder
      } else {
404
405
	      // DEFAULT CASE: DATA (needs to be checked and completed)
406
	      localId = insertDataObject(object, pid, session);
407
      }
408
409 6241 cjones
    }
410 6313 leinfelder
411 7122 leinfelder
    logMetacat.debug("Done inserting new object: " + pid.getValue());
412
413 6313 leinfelder
    // save the sysmeta
414 6468 leinfelder
    try {
415 6572 cjones
      // lock and unlock of the pid happens in the subclass
416 6468 leinfelder
    	HazelcastService.getInstance().getSystemMetadataMap().put(sysmeta.getIdentifier(), sysmeta);
417 6572 cjones
418 6468 leinfelder
    } catch (Exception e) {
419 6495 leinfelder
    	logMetacat.error("Problem creating system metadata: " + pid.getValue(), e);
420 6468 leinfelder
        throw new ServiceFailure("1190", e.getMessage());
421
	}
422 6302 leinfelder
423 6241 cjones
    // setting the resulting identifier failed
424 6302 leinfelder
    if (localId == null ) {
425
      throw new ServiceFailure("1190", "The Node is unable to create the object. ");
426 6241 cjones
    }
427 6313 leinfelder
428 6302 leinfelder
    resultPid = pid;
429
430 7122 leinfelder
    logMetacat.debug("create() complete for object: " + pid.getValue());
431
432 6241 cjones
    return resultPid;
433
  }
434
435 6227 cjones
  /**
436
   * Return the log records associated with a given event between the start and
437
   * end dates listed given a particular Subject listed in the Session
438
   *
439
   * @param session - the Session object containing the credentials for the Subject
440
   * @param fromDate - the start date of the desired log records
441
   * @param toDate - the end date of the desired log records
442
   * @param event - restrict log records of a specific event type
443
   * @param start - zero based offset from the first record in the
444
   *                set of matching log records. Used to assist with
445
   *                paging the response.
446
   * @param count - maximum number of log records to return in the response.
447
   *                Used to assist with paging the response.
448
   *
449
   * @return the desired log records
450
   *
451
   * @throws InvalidToken
452
   * @throws ServiceFailure
453
   * @throws NotAuthorized
454
   * @throws InvalidRequest
455
   * @throws NotImplemented
456
   */
457
  public Log getLogRecords(Session session, Date fromDate, Date toDate,
458 7101 leinfelder
      Event event, String pidFilter, Integer start, Integer count) throws InvalidToken, ServiceFailure,
459 6227 cjones
      NotAuthorized, InvalidRequest, NotImplemented {
460 6174 cjones
461 6186 leinfelder
462 6227 cjones
    Log log = new Log();
463 6321 leinfelder
    List<LogEntry> logs = new Vector<LogEntry>();
464 6227 cjones
    IdentifierManager im = IdentifierManager.getInstance();
465
    EventLog el = EventLog.getInstance();
466 6353 cjones
    if ( fromDate == null ) {
467 6227 cjones
      logMetacat.debug("setting fromdate from null");
468
      fromDate = new Date(1);
469
    }
470 6353 cjones
    if ( toDate == null ) {
471 6227 cjones
      logMetacat.debug("setting todate from null");
472
      toDate = new Date();
473
    }
474 6186 leinfelder
475 6353 cjones
    if ( start == null ) {
476
    	start = 0;
477
478
    }
479
480
    if ( count == null ) {
481
    	count = 1000;
482
483
    }
484
485 6227 cjones
    logMetacat.debug("fromDate: " + fromDate);
486
    logMetacat.debug("toDate: " + toDate);
487 6186 leinfelder
488 6227 cjones
    String report = el.getReport(null, null, null, null,
489
        new java.sql.Timestamp(fromDate.getTime()),
490
        new java.sql.Timestamp(toDate.getTime()), false);
491 6186 leinfelder
492 6227 cjones
    logMetacat.debug("report: " + report);
493 6186 leinfelder
494 6227 cjones
    String logEntry = "<logEntry>";
495
    String endLogEntry = "</logEntry>";
496
    int startIndex = 0;
497
    int foundIndex = report.indexOf(logEntry, startIndex);
498
    while (foundIndex != -1) {
499
      // parse out each entry
500
      int endEntryIndex = report.indexOf(endLogEntry, foundIndex);
501
      String entry = report.substring(foundIndex, endEntryIndex);
502
      logMetacat.debug("entry: " + entry);
503
      startIndex = endEntryIndex + endLogEntry.length();
504
      foundIndex = report.indexOf(logEntry, startIndex);
505 6186 leinfelder
506 6227 cjones
      String entryId = getLogEntryField("entryid", entry);
507
      String ipAddress = getLogEntryField("ipAddress", entry);
508
      String principal = getLogEntryField("principal", entry);
509 6542 leinfelder
      String userAgent = getLogEntryField("userAgent", entry);
510 6227 cjones
      String docid = getLogEntryField("docid", entry);
511
      String eventS = getLogEntryField("event", entry);
512
      String dateLogged = getLogEntryField("dateLogged", entry);
513 6186 leinfelder
514 6227 cjones
      LogEntry le = new LogEntry();
515 6186 leinfelder
516 7049 leinfelder
      // handle Metacat -> DataONE event conversion
517
      if (eventS.equalsIgnoreCase("insert")) {
518 7058 leinfelder
    	  eventS = Event.CREATE.xmlValue();
519 7049 leinfelder
      }
520 6227 cjones
      Event e = Event.convert(eventS);
521
      if (e == null) { // skip any events that are not Dataone Crud events
522 7049 leinfelder
          logMetacat.warn("skipping unknown event type: '" + eventS + "'");
523
          continue;
524 6227 cjones
      }
525
      le.setEvent(e);
526 6754 leinfelder
      le.setEntryId(entryId);
527 6227 cjones
      Identifier identifier = new Identifier();
528
      try {
529 6241 cjones
        logMetacat.debug("converting docid '" + docid + "' to a pid.");
530 6227 cjones
        if (docid == null || docid.trim().equals("") || docid.trim().equals("null")) {
531
          continue;
532
        }
533 7041 leinfelder
        String docidNoRev = DocumentUtil.getSmartDocId(docid);
534
        //docid = docid.substring(0, docid.lastIndexOf("."));
535
        int rev = DocumentUtil.getRevisionFromAccessionNumber(docid);
536
        //int rev = im.getLatestRevForLocalId(docid);
537
        String guid = im.getGUID(docidNoRev, rev);
538
        identifier.setValue(guid);
539 6227 cjones
      } catch (Exception ex) {
540 6241 cjones
        // try to get the pid, if that doesn't
541 6227 cjones
        // work, just use the local id
542
        // throw new ServiceFailure("1030",
543 6241 cjones
        // "Error getting pid for localId " +
544 6227 cjones
        // docid + ": " + ex.getMessage());\
545 6186 leinfelder
546 7049 leinfelder
          logMetacat.warn("could not find pid for docid '" + docid + "'");
547
548 6241 cjones
        // skip it if the pid can't be found
549 6227 cjones
        continue;
550
      }
551 6186 leinfelder
552 7042 leinfelder
      // skip if we are not allowed to read the document in question
553
      // https://redmine.dataone.org/issues/2444
554
      boolean allowed = false;
555
      try {
556
    	  allowed = isAuthorized(session, identifier, Permission.READ);
557
      } catch (NotAuthorized ignore) {}
558 7044 leinfelder
      catch (NotFound nf) {
559
    	  logMetacat.warn("Could not check authorization for pid: " + identifier.getValue(), nf);
560
      }
561 7042 leinfelder
      if (!allowed) {
562
    	  logMetacat.debug(Permission.READ + " not allowed on document: " + identifier.getValue());
563
    	  continue;
564
      }
565
566 6227 cjones
      le.setIdentifier(identifier);
567
      le.setIpAddress(ipAddress);
568 6944 leinfelder
      Date logDate = DateTimeMarshaller.deserializeDateToUTC(dateLogged);
569 6227 cjones
      le.setDateLogged(logDate);
570
      NodeReference memberNode = new NodeReference();
571 7030 cjones
      String nodeId = "localhost";
572
      try {
573
          nodeId = PropertyService.getProperty("dataone.nodeId");
574
      } catch (PropertyNotFoundException e1) {
575
          // TODO Auto-generated catch block
576
          e1.printStackTrace();
577
      }
578
      memberNode.setValue(nodeId);
579 6754 leinfelder
      le.setNodeIdentifier(memberNode);
580 6227 cjones
      Subject princ = new Subject();
581
      princ.setValue(principal);
582
      le.setSubject(princ);
583 6542 leinfelder
      le.setUserAgent(userAgent);
584 6186 leinfelder
585 6321 leinfelder
      // event filtering?
586 6227 cjones
      if (event == null) {
587 6321 leinfelder
    	  logs.add(le);
588
      } else if (le.getEvent().equals(event)) {
589
    	  logs.add(le);
590 6227 cjones
      }
591
    }
592 6321 leinfelder
593
    // d1 paging
594
    int total = logs.size();
595
    if (start != null && count != null) {
596
    	int toIndex = start + count;
597 7047 leinfelder
    	// do not exceed total
598
    	toIndex = Math.min(toIndex, total);
599
    	// do not start greater than total
600
    	start = Math.min(start, total);
601
    	// sub set of the list
602
    	logs = new ArrayList<LogEntry>(logs.subList(start, toIndex));
603 6321 leinfelder
    }
604 6186 leinfelder
605 6227 cjones
    log.setLogEntryList(logs);
606 6321 leinfelder
    log.setStart(start);
607
    log.setCount(logs.size());
608
    log.setTotal(total);
609 6227 cjones
    logMetacat.info("getLogRecords");
610
    return log;
611
  }
612 6241 cjones
613 6227 cjones
  /**
614
   * Return the object identified by the given object identifier
615
   *
616
   * @param session - the Session object containing the credentials for the Subject
617
   * @param pid - the object identifier for the given object
618
   *
619
   * TODO: The D1 Authorization API doesn't provide information on which
620
   * authentication system the Subject belongs to, and so it's not possible to
621
   * discern which Person or Group is a valid KNB LDAP DN.  Fix this.
622
   *
623
   * @return inputStream - the input stream of the given object
624
   *
625
   * @throws InvalidToken
626
   * @throws ServiceFailure
627
   * @throws NotAuthorized
628
   * @throws InvalidRequest
629
   * @throws NotImplemented
630
   */
631
  public InputStream get(Session session, Identifier pid)
632
    throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
633 6609 cjones
    NotImplemented {
634 6226 cjones
635 6227 cjones
    InputStream inputStream = null; // bytes to be returned
636 6226 cjones
    handler = new MetacatHandler(new Timer());
637 6227 cjones
    boolean allowed = false;
638
    String localId; // the metacat docid for the pid
639 6226 cjones
640 6227 cjones
    // get the local docid from Metacat
641
    try {
642
      localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
643
644 6226 cjones
    } catch (McdbDocNotFoundException e) {
645
      throw new NotFound("1020", "The object specified by " +
646 6227 cjones
                         pid.getValue() +
647 6283 leinfelder
                         " does not exist at this node.");
648 6226 cjones
    }
649
650
    // check for authorization
651 7029 leinfelder
    try {
652
		allowed = isAuthorized(session, pid, Permission.READ);
653
	} catch (InvalidRequest e) {
654
		throw new ServiceFailure("1030", e.getDescription());
655
	}
656 6227 cjones
657
    // if the person is authorized, perform the read
658 6323 leinfelder
    if (allowed) {
659 6226 cjones
      try {
660 6323 leinfelder
        inputStream = handler.read(localId);
661
      } catch (Exception e) {
662 6226 cjones
        throw new ServiceFailure("1020", "The object specified by " +
663
            pid.getValue() +
664 6323 leinfelder
            "could not be returned due to error: " +
665 6226 cjones
            e.getMessage());
666
      }
667 6227 cjones
    }
668 6226 cjones
669 6227 cjones
    // if we fail to set the input stream
670
    if ( inputStream == null ) {
671 6226 cjones
      throw new NotFound("1020", "The object specified by " +
672
                         pid.getValue() +
673
                         "does not exist at this node.");
674 6227 cjones
    }
675
676 6389 leinfelder
	// log the read event
677 6532 leinfelder
    String principal = Constants.SUBJECT_PUBLIC;
678 6421 leinfelder
    if (session != null && session.getSubject() != null) {
679 6389 leinfelder
    	principal = session.getSubject().getValue();
680
    }
681 6542 leinfelder
    EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), principal, localId, "read");
682 6389 leinfelder
683 6227 cjones
    return inputStream;
684
  }
685 6174 cjones
686 6227 cjones
  /**
687
   * Return the system metadata for a given object
688
   *
689
   * @param session - the Session object containing the credentials for the Subject
690
   * @param pid - the object identifier for the given object
691
   *
692
   * @return inputStream - the input stream of the given system metadata object
693
   *
694
   * @throws InvalidToken
695
   * @throws ServiceFailure
696
   * @throws NotAuthorized
697
   * @throws NotFound
698
   * @throws InvalidRequest
699
   * @throws NotImplemented
700
   */
701 7132 cjones
    public SystemMetadata getSystemMetadata(Session session, Identifier pid)
702
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
703
        NotImplemented {
704
705
        boolean isAuthorized = false;
706
        SystemMetadata systemMetadata = null;
707
        List<Replica> replicaList = null;
708
        NodeReference replicaNodeRef = null;
709
        List<Node> nodeListBySubject = null;
710
        Subject subject = null;
711 6572 cjones
712 7132 cjones
        if (session != null ) {
713
            subject = session.getSubject();
714
        }
715 6572 cjones
716 7145 leinfelder
        // check normal authorization
717
        BaseException originalAuthorizationException = null;
718
        if (!isAuthorized) {
719
            try {
720
                isAuthorized = isAuthorized(session, pid, Permission.READ);
721 7132 cjones
722 7145 leinfelder
            } catch (InvalidRequest e) {
723
                throw new ServiceFailure("1090", e.getDescription());
724
            } catch (NotAuthorized nae) {
725
            	// catch this for later
726
            	originalAuthorizationException = nae;
727
			}
728
        }
729 7132 cjones
730 7145 leinfelder
        // get the system metadata first because we need the replica list for auth
731
        systemMetadata = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
732
733
        // check the replica information to expand access to MNs that might need it
734
        if (!isAuthorized) {
735
736
	        try {
737
738
	            // if MNs are listed as replicas, allow access
739
	            if ( systemMetadata != null ) {
740
	                replicaList = systemMetadata.getReplicaList();
741
	                // only check if there are in fact replicas listed
742
	                if ( replicaList != null ) {
743
744
	                    if ( subject != null ) {
745
	                        // get the list of nodes with a matching node subject
746
	                        try {
747
	                            nodeListBySubject = listNodesBySubject(session.getSubject());
748
749
	                        } catch (BaseException e) {
750
	                            // Unexpected error contacting the CN via D1Client
751
	                            String msg = "Caught an unexpected error while trying "
752
	                                    + "to potentially authorize system metadata access "
753
	                                    + "based on the session subject. The error was "
754
	                                    + e.getMessage();
755
	                            logMetacat.error(msg);
756
	                            if (logMetacat.isDebugEnabled()) {
757
	                                e.printStackTrace();
758
759
	                            }
760
	                            // isAuthorized is still false
761
	                        }
762
763
	                    }
764
	                    if (nodeListBySubject != null) {
765
	                        // compare node ids to replica node ids
766
	                        outer: for (Replica replica : replicaList) {
767
	                            replicaNodeRef = replica.getReplicaMemberNode();
768
769
	                            for (Node node : nodeListBySubject) {
770
	                                if (node.getIdentifier().equals(replicaNodeRef)) {
771
	                                    // node id via session subject matches a replica node
772
	                                    isAuthorized = true;
773
	                                    break outer;
774
	                                }
775
	                            }
776
	                        }
777
	                    }
778
	                }
779
	            }
780
781
	            // if we still aren't authorized, then we are done
782
	            if (!isAuthorized) {
783
	                throw new NotAuthorized("1400", Permission.READ
784
	                        + " not allowed on " + pid.getValue());
785
	            }
786 7132 cjones
787 7145 leinfelder
	        } catch (RuntimeException e) {
788
	        	e.printStackTrace();
789
	            // convert hazelcast RuntimeException to ServiceFailure
790
	            throw new ServiceFailure("1090", "Unexpected error getting system metadata for: " +
791
	                pid.getValue());
792
	        }
793
794 7132 cjones
        }
795 7145 leinfelder
796 7132 cjones
        // It wasn't in the map
797
        if ( systemMetadata == null ) {
798
            throw new NotFound("1420", "No record found for: " + pid.getValue());
799
        }
800
801
        return systemMetadata;
802
    }
803 6572 cjones
804 6227 cjones
  /**
805 6865 cjones
   * Test if the user identified by the provided token has administrative authorization
806
   *
807
   * @param session - the Session object containing the credentials for the Subject
808
   *
809 7142 leinfelder
   * @return true if the user is admin
810 6865 cjones
   *
811
   * @throws ServiceFailure
812
   * @throws InvalidToken
813
   * @throws NotFound
814
   * @throws NotAuthorized
815
   * @throws NotImplemented
816
   */
817 7142 leinfelder
  public boolean isAdminAuthorized(Session session)
818 7079 leinfelder
      throws ServiceFailure, InvalidToken, NotAuthorized,
819 6865 cjones
      NotImplemented {
820
821
      boolean allowed = false;
822 7069 leinfelder
823
      // must have a session in order to check admin
824
      if (session == null) {
825 7142 leinfelder
         logMetacat.debug("In isAdminAuthorized(), session is null ");
826
         return false;
827 7069 leinfelder
      }
828
829 7072 cjones
      logMetacat.debug("In isAdminAuthorized(), checking CN or MN authorization for " +
830 7142 leinfelder
           session.getSubject().getValue());
831 7072 cjones
832
      // are we allowed to do this? only CNs are allowed
833 6865 cjones
      CNode cn = D1Client.getCN();
834
      List<Node> nodes = cn.listNodes().getNodeList();
835
836
      if ( nodes == null ) {
837
          throw new ServiceFailure("4852", "Couldn't get node list.");
838
839
      }
840
841
      // find the node in the node list
842
      for ( Node node : nodes ) {
843
844
          NodeReference nodeReference = node.getIdentifier();
845
          logMetacat.debug("In isAdminAuthorized(), Node reference is: " + nodeReference.getValue());
846
847
          Subject subject = session.getSubject();
848
849
          if (node.getType() == NodeType.CN) {
850
              List<Subject> nodeSubjects = node.getSubjectList();
851
852
              // check if the session subject is in the node subject list
853
              for (Subject nodeSubject : nodeSubjects) {
854 7072 cjones
                  logMetacat.debug("In isAdminAuthorized(), comparing subjects: " +
855
                      nodeSubject.getValue() + " and " + subject.getValue());
856 6865 cjones
                  if ( nodeSubject.equals(subject) ) {
857
                      allowed = true; // subject of session == target node subject
858
                      break;
859
860
                  }
861 6874 cjones
              }
862 6865 cjones
          }
863
      }
864
865
866
      return allowed;
867
  }
868
869
  /**
870 6227 cjones
   * Test if the user identified by the provided token has authorization
871 6865 cjones
   * for the operation on the specified object.
872 6227 cjones
   *
873
   * @param session - the Session object containing the credentials for the Subject
874
   * @param pid - The identifer of the resource for which access is being checked
875
   * @param operation - The type of operation which is being requested for the given pid
876
   *
877
   * @return true if the operation is allowed
878
   *
879
   * @throws ServiceFailure
880
   * @throws InvalidToken
881
   * @throws NotFound
882
   * @throws NotAuthorized
883
   * @throws NotImplemented
884
   * @throws InvalidRequest
885
   */
886
  public boolean isAuthorized(Session session, Identifier pid, Permission permission)
887
    throws ServiceFailure, InvalidToken, NotFound, NotAuthorized,
888 7029 leinfelder
    NotImplemented, InvalidRequest {
889 6174 cjones
890 6227 cjones
    boolean allowed = false;
891
892 7029 leinfelder
    if (permission == null) {
893
    	throw new InvalidRequest("1761", "Permission was not provided or is invalid");
894
    }
895
896 6816 leinfelder
    // permissions are hierarchical
897 6830 leinfelder
    List<Permission> expandedPermissions = null;
898 6816 leinfelder
899 7067 cjones
    // always allow CN access
900 7142 leinfelder
    if ( isAdminAuthorized(session) ) {
901 7067 cjones
        allowed = true;
902
        return allowed;
903
904
    }
905
906 6816 leinfelder
    // for the "Verified" symbolic user
907
    Subject verifiedSubject = new Subject();
908
	verifiedSubject.setValue(Constants.SUBJECT_VERIFIED_USER);
909
910 7000 leinfelder
    // get the subject[s] from the session
911
	List<Subject> subjects = new ArrayList<Subject>();
912
	if (session != null) {
913
		// primary subject
914
		Subject subject = session.getSubject();
915
		if (subject != null) {
916
			subjects.add(subject);
917
		}
918
		// details about the subject
919
		SubjectInfo subjectInfo = session.getSubjectInfo();
920
		if (subjectInfo != null) {
921
			// find subjectInfo for the primary subject
922
			List<Person> personList = subjectInfo.getPersonList();
923 7060 leinfelder
			List<Group> groupList = subjectInfo.getGroupList();
924 7000 leinfelder
			if (personList != null) {
925
				for (Person p : personList) {
926 7039 cjones
					  // for every person listed (isVerified is transitive)
927 7038 leinfelder
						logMetacat.debug("checking person");
928
						logMetacat.debug("p.getVerified(): " + p.getVerified());
929 7037 leinfelder
						if (p.getVerified() != null && p.getVerified()) {
930 7000 leinfelder
							// add the verified symbolic user
931
							if (!subjects.contains(verifiedSubject)) {
932
								subjects.add(verifiedSubject);
933
							}
934
						}
935
						// add the equivalent identities
936 7061 leinfelder
						List<Subject> equivList = p.getEquivalentIdentityList();
937 7000 leinfelder
						if (equivList != null) {
938 7061 leinfelder
							for (Subject equivSubject : equivList) {
939
								subjects.add(equivSubject);
940
								// find that entry
941
								for (Person equivPerson: personList) {
942
									if (equivSubject.equals(equivPerson.getSubject())) {
943
										// transitive group membership
944
										if (equivPerson.getIsMemberOfList() != null) {
945
											for (Subject equivGroup: equivPerson.getIsMemberOfList()) {
946
												subjects.add(equivGroup);
947
											}
948
										}
949
										// TODO: is verified transitive?
950
										if (equivPerson.getVerified() != null && equivPerson.getVerified()) {
951
											// add the verified symbolic user
952
											if (!subjects.contains(verifiedSubject)) {
953
												subjects.add(verifiedSubject);
954
											}
955
										}
956
									}
957
								}
958 7000 leinfelder
							}
959
						}
960 7060 leinfelder
						// add the groups they are a member of
961
						List<Subject> memberOfList = p.getIsMemberOfList();
962
						if (memberOfList != null) {
963
							for (Subject g : memberOfList) {
964 7000 leinfelder
								subjects.add(g);
965
							}
966
						}
967 7060 leinfelder
						// look at all the Groups to see if this person has membership defined there
968
						if (groupList != null) {
969
							for (Group group: groupList) {
970
								if (group.getHasMemberList() != null) {
971
									for (Subject member: group.getHasMemberList()) {
972
										// is the person a member?
973
										if (member.equals(p.getSubject())) {
974
											// add this group as a subject to check if it is not already there
975
											if (!subjects.contains(group.getSubject())) {
976
												subjects.add(group.getSubject());
977
											}
978
										}
979
									}
980
								}
981
							}
982
						}
983 7000 leinfelder
						break;
984
				}
985
			}
986
		}
987
988
		// add the authenticated symbolic since we have a session
989
		Subject authenticatedSubject = new Subject();
990
		authenticatedSubject.setValue(Constants.SUBJECT_AUTHENTICATED_USER);
991
		subjects.add(authenticatedSubject);
992
	}
993
994
    // add public subject for everyone
995 6293 leinfelder
    Subject publicSubject = new Subject();
996 6532 leinfelder
    publicSubject.setValue(Constants.SUBJECT_PUBLIC);
997 6293 leinfelder
    subjects.add(publicSubject);
998
999 6227 cjones
    // get the system metadata
1000 6241 cjones
    String pidStr = pid.getValue();
1001 6227 cjones
    SystemMetadata systemMetadata = null;
1002
    try {
1003 6574 cjones
        systemMetadata = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
1004
1005
    } catch (Exception e) {
1006
        // convert Hazelcast RuntimeException to NotFound
1007
        logMetacat.error("An error occurred while getting system metadata for identifier " +
1008
            pid.getValue() + ". The error message was: " + e.getMessage());
1009
        throw new NotFound("1800", "No record found for " + pidStr);
1010
1011 6858 cjones
    }
1012 6596 leinfelder
1013
    // throw not found if it was not found
1014
    if (systemMetadata == null) {
1015 6652 leinfelder
    	throw new NotFound("1800", "No system metadata could be found for given PID: " + pidStr);
1016 6596 leinfelder
    }
1017 6245 leinfelder
1018 6293 leinfelder
    // do we own it?
1019
    for (Subject s: subjects) {
1020 7039 cjones
      logMetacat.debug("Comparing \t" +
1021
                       systemMetadata.getRightsHolder().getValue() +
1022
                       " \tagainst \t" + s.getValue());
1023 6688 leinfelder
    	allowed = systemMetadata.getRightsHolder().equals(s);
1024 6293 leinfelder
    	if (allowed) {
1025
    		return allowed;
1026
    	}
1027
    }
1028
1029
    // otherwise check the access rules
1030 6245 leinfelder
    try {
1031
	    List<AccessRule> allows = systemMetadata.getAccessPolicy().getAllowList();
1032 6293 leinfelder
	    search: // label break
1033 6245 leinfelder
	    for (AccessRule accessRule: allows) {
1034
	      for (Subject s: subjects) {
1035 7059 leinfelder
	        logMetacat.debug("Checking allow access rule for subject: " + s.getValue());
1036 6816 leinfelder
	        if (accessRule.getSubjectList().contains(s)) {
1037 7059 leinfelder
	        	logMetacat.debug("Access rule contains subject: " + s.getValue());
1038 6830 leinfelder
	        	for (Permission p: accessRule.getPermissionList()) {
1039 7059 leinfelder
		        	logMetacat.debug("Checking permission: " + p.xmlValue());
1040 6830 leinfelder
	        		expandedPermissions = expandPermissions(p);
1041
	        		allowed = expandedPermissions.contains(permission);
1042
	        		if (allowed) {
1043 7059 leinfelder
			        	logMetacat.info("Permission granted: " + p.xmlValue() + " to " + s.getValue());
1044 6830 leinfelder
	        			break search; //label break
1045
	        		}
1046 6816 leinfelder
	        	}
1047
1048 6245 leinfelder
	        }
1049
	      }
1050
	    }
1051
    } catch (Exception e) {
1052
    	// catch all for errors - safe side should be to deny the access
1053
    	logMetacat.error("Problem checking authorization - defaulting to deny", e);
1054
		allowed = false;
1055 7039 cjones
1056
    }
1057 6227 cjones
1058 6245 leinfelder
    // throw or return?
1059 6227 cjones
    if (!allowed) {
1060 6245 leinfelder
      throw new NotAuthorized("1820", permission + " not allowed on " + pidStr);
1061 6227 cjones
    }
1062 6245 leinfelder
1063 6227 cjones
    return allowed;
1064 6241 cjones
1065 6227 cjones
  }
1066
1067 6256 cjones
  /*
1068 6227 cjones
   * parse a logEntry and get the relevant field from it
1069
   *
1070
   * @param fieldname
1071
   * @param entry
1072
   * @return
1073
   */
1074
  private String getLogEntryField(String fieldname, String entry) {
1075
    String begin = "<" + fieldname + ">";
1076
    String end = "</" + fieldname + ">";
1077
    // logMetacat.debug("looking for " + begin + " and " + end +
1078
    // " in entry " + entry);
1079
    String s = entry.substring(entry.indexOf(begin) + begin.length(), entry
1080
        .indexOf(end));
1081
    logMetacat.debug("entry " + fieldname + " : " + s);
1082
    return s;
1083
  }
1084 6174 cjones
1085 6257 cjones
  /**
1086 6241 cjones
   * Determine if a given object should be treated as an XML science metadata
1087
   * object.
1088
   *
1089
   * @param sysmeta - the SystemMetadata describing the object
1090
   * @return true if the object should be treated as science metadata
1091
   */
1092 6433 leinfelder
  public static boolean isScienceMetadata(SystemMetadata sysmeta) {
1093 6241 cjones
1094
    ObjectFormat objectFormat = null;
1095
    boolean isScienceMetadata = false;
1096
1097
    try {
1098 6561 leinfelder
      objectFormat = ObjectFormatCache.getInstance().getFormat(sysmeta.getFormatId());
1099 6433 leinfelder
      if ( objectFormat.getFormatType().equals("METADATA") ) {
1100 6402 cjones
      	isScienceMetadata = true;
1101
1102
      }
1103 6241 cjones
1104
1105
    } catch (ServiceFailure e) {
1106
      logMetacat.debug("There was a problem determining if the object identified by" +
1107
          sysmeta.getIdentifier().getValue() +
1108
          " is science metadata: " + e.getMessage());
1109
1110
    } catch (NotFound e) {
1111
      logMetacat.debug("There was a problem determining if the object identified by" +
1112
          sysmeta.getIdentifier().getValue() +
1113
          " is science metadata: " + e.getMessage());
1114
1115
    }
1116
1117
    return isScienceMetadata;
1118 6226 cjones
1119 6241 cjones
  }
1120
1121
  /**
1122
   * Insert or update an XML document into Metacat
1123
   *
1124
   * @param xml - the XML document to insert or update
1125
   * @param pid - the identifier to be used for the resulting object
1126
   *
1127
   * @return localId - the resulting docid of the document created or updated
1128
   *
1129
   */
1130 6807 leinfelder
  public String insertOrUpdateDocument(String xml, Identifier pid,
1131 6241 cjones
    Session session, String insertOrUpdate)
1132
    throws ServiceFailure {
1133
1134
  	logMetacat.debug("Starting to insert xml document...");
1135
    IdentifierManager im = IdentifierManager.getInstance();
1136
1137
    // generate pid/localId pair for sysmeta
1138
    String localId = null;
1139
1140
    if(insertOrUpdate.equals("insert")) {
1141
      localId = im.generateLocalId(pid.getValue(), 1);
1142
1143
    } else {
1144
      //localid should already exist in the identifier table, so just find it
1145
      try {
1146
        logMetacat.debug("Updating pid " + pid.getValue());
1147
        logMetacat.debug("looking in identifier table for pid " + pid.getValue());
1148
1149
        localId = im.getLocalId(pid.getValue());
1150
1151
        logMetacat.debug("localId: " + localId);
1152
        //increment the revision
1153
        String docid = localId.substring(0, localId.lastIndexOf("."));
1154
        String revS = localId.substring(localId.lastIndexOf(".") + 1, localId.length());
1155
        int rev = new Integer(revS).intValue();
1156
        rev++;
1157
        docid = docid + "." + rev;
1158
        localId = docid;
1159
        logMetacat.debug("incremented localId: " + localId);
1160
1161
      } catch(McdbDocNotFoundException e) {
1162
        throw new ServiceFailure("1030", "D1NodeService.insertOrUpdateDocument(): " +
1163
            "pid " + pid.getValue() +
1164
            " should have been in the identifier table, but it wasn't: " +
1165
            e.getMessage());
1166
1167
      }
1168
1169
    }
1170
1171 6443 leinfelder
    params = new Hashtable<String, String[]>();
1172 6241 cjones
    String[] action = new String[1];
1173
    action[0] = insertOrUpdate;
1174
    params.put("action", action);
1175
    String[] docid = new String[1];
1176
    docid[0] = localId;
1177
    params.put("docid", docid);
1178
    String[] doctext = new String[1];
1179
    doctext[0] = xml;
1180
    params.put("doctext", doctext);
1181
1182 6532 leinfelder
    String username = Constants.SUBJECT_PUBLIC;
1183 6241 cjones
    String[] groupnames = null;
1184 6302 leinfelder
    if (session != null ) {
1185
    	username = session.getSubject().getValue();
1186 6532 leinfelder
    	if (session.getSubjectInfo() != null) {
1187
    		List<Group> groupList = session.getSubjectInfo().getGroupList();
1188 6302 leinfelder
    		if (groupList != null) {
1189
    			groupnames = new String[groupList.size()];
1190
    			for (int i = 0; i > groupList.size(); i++ ) {
1191
    				groupnames[i] = groupList.get(i).getGroupName();
1192
    			}
1193
    		}
1194
    	}
1195 6241 cjones
    }
1196
1197
    // do the insert or update action
1198 6443 leinfelder
    handler = new MetacatHandler(new Timer());
1199 6542 leinfelder
    String result = handler.handleInsertOrUpdateAction(request.getRemoteAddr(), request.getHeader("User-Agent"), null,
1200 7128 leinfelder
                        null, params, username, groupnames, false, false);
1201 6241 cjones
1202
    if(result.indexOf("<error>") != -1) {
1203
    	String detailCode = "";
1204
    	if ( insertOrUpdate.equals("insert") ) {
1205 7017 leinfelder
    		// make sure to remove the mapping so that subsequent attempts do not fail with IdentifierNotUnique
1206
    		im.removeMapping(pid.getValue(), localId);
1207 6241 cjones
    		detailCode = "1190";
1208
1209
    	} else if ( insertOrUpdate.equals("update") ) {
1210
    		detailCode = "1310";
1211
1212
    	}
1213
        throw new ServiceFailure(detailCode,
1214
          "Error inserting or updating document: " + result);
1215
    }
1216
    logMetacat.debug("Finsished inserting xml document with id " + localId);
1217
1218
    return localId;
1219
  }
1220
1221
  /**
1222
   * Insert a data document
1223
   *
1224
   * @param object
1225
   * @param pid
1226
   * @param sessionData
1227
   * @throws ServiceFailure
1228
   * @returns localId of the data object inserted
1229
   */
1230 6807 leinfelder
  public String insertDataObject(InputStream object, Identifier pid,
1231 6241 cjones
          Session session) throws ServiceFailure {
1232
1233 6532 leinfelder
    String username = Constants.SUBJECT_PUBLIC;
1234 6241 cjones
    String[] groupnames = null;
1235 6302 leinfelder
    if (session != null ) {
1236
    	username = session.getSubject().getValue();
1237 6532 leinfelder
    	if (session.getSubjectInfo() != null) {
1238
    		List<Group> groupList = session.getSubjectInfo().getGroupList();
1239 6302 leinfelder
    		if (groupList != null) {
1240
    			groupnames = new String[groupList.size()];
1241
    			for (int i = 0; i > groupList.size(); i++ ) {
1242
    				groupnames[i] = groupList.get(i).getGroupName();
1243
    			}
1244
    		}
1245
    	}
1246 6241 cjones
    }
1247
1248
    // generate pid/localId pair for object
1249
    logMetacat.debug("Generating a pid/localId mapping");
1250
    IdentifierManager im = IdentifierManager.getInstance();
1251
    String localId = im.generateLocalId(pid.getValue(), 1);
1252 6719 leinfelder
1253
    // Save the data file to disk using "localId" as the name
1254
    String datafilepath = null;
1255
	try {
1256
		datafilepath = PropertyService.getProperty("application.datafilepath");
1257
	} catch (PropertyNotFoundException e) {
1258
		ServiceFailure sf = new ServiceFailure("1190", "Lookup data file path" + e.getMessage());
1259
		sf.initCause(e);
1260
		throw sf;
1261
	}
1262
    boolean locked = false;
1263
	try {
1264
		locked = DocumentImpl.getDataFileLockGrant(localId);
1265
	} catch (Exception e) {
1266
		ServiceFailure sf = new ServiceFailure("1190", "Could not lock file for writing:" + e.getMessage());
1267
		sf.initCause(e);
1268
		throw sf;
1269
	}
1270
1271
    logMetacat.debug("Case DATA: starting to write to disk.");
1272
	if (locked) {
1273
1274 6241 cjones
          File dataDirectory = new File(datafilepath);
1275
          dataDirectory.mkdirs();
1276
1277
          File newFile = writeStreamToFile(dataDirectory, localId, object);
1278
1279
          // TODO: Check that the file size matches SystemMetadata
1280
          // long size = newFile.length();
1281
          // if (size == 0) {
1282
          //     throw new IOException("Uploaded file is 0 bytes!");
1283
          // }
1284
1285
          // Register the file in the database (which generates an exception
1286
          // if the localId is not acceptable or other untoward things happen
1287
          try {
1288
            logMetacat.debug("Registering document...");
1289
            DocumentImpl.registerDocument(localId, "BIN", localId,
1290
                    username, groupnames);
1291
            logMetacat.debug("Registration step completed.");
1292
1293
          } catch (SQLException e) {
1294
            //newFile.delete();
1295
            logMetacat.debug("SQLE: " + e.getMessage());
1296
            e.printStackTrace(System.out);
1297
            throw new ServiceFailure("1190", "Registration failed: " +
1298
            		e.getMessage());
1299
1300
          } catch (AccessionNumberException e) {
1301
            //newFile.delete();
1302
            logMetacat.debug("ANE: " + e.getMessage());
1303
            e.printStackTrace(System.out);
1304
            throw new ServiceFailure("1190", "Registration failed: " +
1305
            	e.getMessage());
1306
1307
          } catch (Exception e) {
1308
            //newFile.delete();
1309
            logMetacat.debug("Exception: " + e.getMessage());
1310
            e.printStackTrace(System.out);
1311
            throw new ServiceFailure("1190", "Registration failed: " +
1312
            	e.getMessage());
1313
          }
1314
1315
          logMetacat.debug("Logging the creation event.");
1316 6542 leinfelder
          EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), username, localId, "create");
1317 6241 cjones
1318
          // Schedule replication for this data file
1319
          logMetacat.debug("Scheduling replication.");
1320
          ForceReplicationHandler frh = new ForceReplicationHandler(
1321
            localId, "create", false, null);
1322 6719 leinfelder
      }
1323
1324
      return localId;
1325 6241 cjones
1326
  }
1327 6255 cjones
1328 6256 cjones
  /**
1329
   * Insert a systemMetadata document and return its localId
1330
   */
1331 6807 leinfelder
  public void insertSystemMetadata(SystemMetadata sysmeta)
1332 6572 cjones
      throws ServiceFailure {
1333
1334
  	  logMetacat.debug("Starting to insert SystemMetadata...");
1335
      sysmeta.setDateSysMetadataModified(Calendar.getInstance().getTime());
1336
      logMetacat.debug("Inserting new system metadata with modified date " +
1337
          sysmeta.getDateSysMetadataModified());
1338
1339
      //insert the system metadata
1340
      try {
1341
        // note: the calling subclass handles the map hazelcast lock/unlock
1342
      	HazelcastService.getInstance().getSystemMetadataMap().put(sysmeta.getIdentifier(), sysmeta);
1343
1344
      } catch (Exception e) {
1345
          throw new ServiceFailure("1190", e.getMessage());
1346
1347
	    }
1348 6256 cjones
  }
1349
1350
  /**
1351 6255 cjones
   * Update a systemMetadata document
1352
   *
1353
   * @param sysMeta - the system metadata object in the system to update
1354
   */
1355 6858 cjones
    protected void updateSystemMetadata(SystemMetadata sysMeta)
1356
        throws ServiceFailure {
1357
1358
        logMetacat.debug("D1NodeService.updateSystemMetadata() called.");
1359
        sysMeta.setDateSysMetadataModified(new Date());
1360
        try {
1361
            HazelcastService.getInstance().getSystemMetadataMap().lock(sysMeta.getIdentifier());
1362
            HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);
1363
1364
        } catch (Exception e) {
1365
            throw new ServiceFailure("4862", e.getMessage());
1366
1367
        } finally {
1368
            HazelcastService.getInstance().getSystemMetadataMap().unlock(sysMeta.getIdentifier());
1369
1370
        }
1371
1372
    }
1373 6816 leinfelder
1374
  /**
1375
   * Given a Permission, returns a list of all permissions that it encompasses
1376
   * Permissions are hierarchical so that WRITE also allows READ.
1377
   * @param permission
1378
   * @return list of included Permissions for the given permission
1379
   */
1380
  protected List<Permission> expandPermissions(Permission permission) {
1381
	  	List<Permission> expandedPermissions = new ArrayList<Permission>();
1382
	    if (permission.equals(Permission.READ)) {
1383
	    	expandedPermissions.add(Permission.READ);
1384
	    }
1385
	    if (permission.equals(Permission.WRITE)) {
1386
	    	expandedPermissions.add(Permission.READ);
1387
	    	expandedPermissions.add(Permission.WRITE);
1388
	    }
1389
	    if (permission.equals(Permission.CHANGE_PERMISSION)) {
1390
	    	expandedPermissions.add(Permission.READ);
1391
	    	expandedPermissions.add(Permission.WRITE);
1392
	    	expandedPermissions.add(Permission.CHANGE_PERMISSION);
1393
	    }
1394
	    return expandedPermissions;
1395
  }
1396 6255 cjones
1397
  /*
1398 6241 cjones
   * Write a stream to a file
1399
   *
1400
   * @param dir - the directory to write to
1401
   * @param fileName - the file name to write to
1402
   * @param data - the object bytes as an input stream
1403
   *
1404
   * @return newFile - the new file created
1405
   *
1406
   * @throws ServiceFailure
1407
   */
1408
  private File writeStreamToFile(File dir, String fileName, InputStream data)
1409
    throws ServiceFailure {
1410
1411
    File newFile = new File(dir, fileName);
1412
    logMetacat.debug("Filename for write is: " + newFile.getAbsolutePath());
1413
1414
    try {
1415
        if (newFile.createNewFile()) {
1416
          // write data stream to desired file
1417
          OutputStream os = new FileOutputStream(newFile);
1418
          long length = IOUtils.copyLarge(data, os);
1419
          os.flush();
1420
          os.close();
1421
        } else {
1422
          logMetacat.debug("File creation failed, or file already exists.");
1423
          throw new ServiceFailure("1190", "File already exists: " + fileName);
1424
        }
1425
    } catch (FileNotFoundException e) {
1426
      logMetacat.debug("FNF: " + e.getMessage());
1427
      throw new ServiceFailure("1190", "File not found: " + fileName + " "
1428
                + e.getMessage());
1429
    } catch (IOException e) {
1430
      logMetacat.debug("IOE: " + e.getMessage());
1431
      throw new ServiceFailure("1190", "File was not written: " + fileName
1432
                + " " + e.getMessage());
1433
    }
1434
1435
    return newFile;
1436
  }
1437 7132 cjones
1438
  /*
1439
   * Returns a list of nodes that have been registered with the DataONE infrastructure
1440
   * that match the given session subject
1441
   * @return nodes - List of nodes from the registry with a matching session subject
1442
   *
1443
   * @throws ServiceFailure
1444
   * @throws NotImplemented
1445
   */
1446
  protected List<Node> listNodesBySubject(Subject subject)
1447
      throws ServiceFailure, NotImplemented {
1448 7136 leinfelder
      List<Node> nodeList = new ArrayList<Node>();
1449 7132 cjones
1450
      CNode cn = D1Client.getCN();
1451
      List<Node> nodes = cn.listNodes().getNodeList();
1452
1453
      // find the node in the node list
1454
      for ( Node node : nodes ) {
1455
1456
          List<Subject> nodeSubjects = node.getSubjectList();
1457 7139 leinfelder
          if (nodeSubjects != null) {
1458
	          // check if the session subject is in the node subject list
1459
	          for (Subject nodeSubject : nodeSubjects) {
1460
	              if ( nodeSubject.equals(subject) ) { // subject of session == node subject
1461
	                  nodeList.add(node);
1462
	              }
1463
	          }
1464
          }
1465 7132 cjones
      }
1466
1467
      return nodeList;
1468
1469
  }
1470
1471
1472 6241 cjones
}