Project

General

Profile

1
/**
2
 *  '$RCSfile$'
3
 *  Copyright: 2000-2011 Regents of the University of California and the
4
 *              National Center for Ecological Analysis and Synthesis
5
 *
6
 *   '$Author: tao $'
7
 *     '$Date: 2015-08-06 14:34:00 -0700 (Thu, 06 Aug 2015) $'
8
 *
9
 * This program is free software; you can redistribute it and/or modify
10
 * it under the terms of the GNU General Public License as published by
11
 * the Free Software Foundation; either version 2 of the License, or
12
 * (at your option) any later version.
13
 *
14
 * This program is distributed in the hope that it will be useful,
15
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 * GNU General Public License for more details.
18
 *
19
 * You should have received a copy of the GNU General Public License
20
 * along with this program; if not, write to the Free Software
21
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22
 */
23

    
24
package edu.ucsb.nceas.metacat.dataone;
25

    
26
import java.io.ByteArrayOutputStream;
27
import java.io.File;
28
import java.io.FileNotFoundException;
29
import java.io.FileOutputStream;
30
import java.io.IOException;
31
import java.io.InputStream;
32
import java.io.OutputStream;
33
import java.io.OutputStreamWriter;
34
import java.io.Writer;
35
import java.math.BigInteger;
36
import java.sql.PreparedStatement;
37
import java.sql.ResultSet;
38
import java.sql.SQLException;
39
import java.util.ArrayList;
40
import java.util.Calendar;
41
import java.util.Date;
42
import java.util.Hashtable;
43
import java.util.List;
44
import java.util.Set;
45
import java.util.Timer;
46
import java.util.Vector;
47
import java.util.concurrent.locks.Lock;
48

    
49
import javax.servlet.http.HttpServletRequest;
50

    
51
import org.apache.commons.io.IOUtils;
52
import org.apache.log4j.Logger;
53
import org.dataone.client.v2.CNode;
54
import org.dataone.client.v2.itk.D1Client;
55
import org.dataone.client.v2.formats.ObjectFormatCache;
56
import org.dataone.configuration.Settings;
57
import org.dataone.service.exceptions.BaseException;
58
import org.dataone.service.exceptions.IdentifierNotUnique;
59
import org.dataone.service.exceptions.InsufficientResources;
60
import org.dataone.service.exceptions.InvalidRequest;
61
import org.dataone.service.exceptions.InvalidSystemMetadata;
62
import org.dataone.service.exceptions.InvalidToken;
63
import org.dataone.service.exceptions.NotAuthorized;
64
import org.dataone.service.exceptions.NotFound;
65
import org.dataone.service.exceptions.NotImplemented;
66
import org.dataone.service.exceptions.ServiceFailure;
67
import org.dataone.service.exceptions.UnsupportedType;
68
import org.dataone.service.types.v1.AccessRule;
69
import org.dataone.service.types.v1.DescribeResponse;
70
import org.dataone.service.types.v1.Group;
71
import org.dataone.service.types.v1.Identifier;
72
import org.dataone.service.types.v1.ObjectFormatIdentifier;
73
import org.dataone.service.types.v1.ObjectList;
74
import org.dataone.service.types.v2.Log;
75
import org.dataone.service.types.v2.Node;
76
import org.dataone.service.types.v2.OptionList;
77
import org.dataone.service.types.v1.Event;
78
import org.dataone.service.types.v1.NodeReference;
79
import org.dataone.service.types.v1.NodeType;
80
import org.dataone.service.types.v2.ObjectFormat;
81
import org.dataone.service.types.v1.Permission;
82
import org.dataone.service.types.v1.Replica;
83
import org.dataone.service.types.v1.Session;
84
import org.dataone.service.types.v1.Subject;
85
import org.dataone.service.types.v2.SystemMetadata;
86
import org.dataone.service.types.v1.util.AuthUtils;
87
import org.dataone.service.types.v1.util.ChecksumUtil;
88
import org.dataone.service.util.Constants;
89

    
90
import edu.ucsb.nceas.metacat.AccessionNumber;
91
import edu.ucsb.nceas.metacat.AccessionNumberException;
92
import edu.ucsb.nceas.metacat.DBTransform;
93
import edu.ucsb.nceas.metacat.DocumentImpl;
94
import edu.ucsb.nceas.metacat.EventLog;
95
import edu.ucsb.nceas.metacat.IdentifierManager;
96
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
97
import edu.ucsb.nceas.metacat.MetacatHandler;
98
import edu.ucsb.nceas.metacat.client.InsufficientKarmaException;
99
import edu.ucsb.nceas.metacat.common.query.stream.ContentTypeByteArrayInputStream;
100
import edu.ucsb.nceas.metacat.database.DBConnection;
101
import edu.ucsb.nceas.metacat.database.DBConnectionPool;
102
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;
103
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;
104
import edu.ucsb.nceas.metacat.properties.PropertyService;
105
import edu.ucsb.nceas.metacat.replication.ForceReplicationHandler;
106
import edu.ucsb.nceas.metacat.util.SkinUtil;
107
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
108

    
109
public abstract class D1NodeService {
110
    
111
  public static final String DELETEDMESSAGE = "The object with the PID has been deleted from the node.";
112
  
113
  private static Logger logMetacat = Logger.getLogger(D1NodeService.class);
114

    
115
  /** For logging the operations */
116
  protected HttpServletRequest request;
117
  
118
  /* reference to the metacat handler */
119
  protected MetacatHandler handler;
120
  
121
  /* parameters set in the incoming request */
122
  private Hashtable<String, String[]> params;
123
  
124
  /**
125
   * limit paged results sets to a configured maximum
126
   */
127
  protected static int MAXIMUM_DB_RECORD_COUNT = 7000;
128
  
129
  static {
130
		try {
131
			MAXIMUM_DB_RECORD_COUNT = Integer.valueOf(PropertyService.getProperty("database.webResultsetSize"));
132
		} catch (Exception e) {
133
			logMetacat.warn("Could not set MAXIMUM_DB_RECORD_COUNT", e);
134
		}
135
	}
136
  
137
  /**
138
   * out-of-band session object to be used when not passed in as a method parameter
139
   */
140
  protected Session session;
141

    
142
  /**
143
   * Constructor - used to set the metacatUrl from a subclass extending D1NodeService
144
   * 
145
   * @param metacatUrl - the URL of the metacat service, including the ending /d1
146
   */
147
  public D1NodeService(HttpServletRequest request) {
148
		this.request = request;
149
	}
150

    
151
  /**
152
   * retrieve the out-of-band session
153
   * @return
154
   */
155
  	public Session getSession() {
156
		return session;
157
	}
158
  	
159
  	/**
160
  	 * Set the out-of-band session
161
  	 * @param session
162
  	 */
163
	public void setSession(Session session) {
164
		this.session = session;
165
	}
166

    
167
  /**
168
   * This method provides a lighter weight mechanism than 
169
   * getSystemMetadata() for a client to determine basic 
170
   * properties of the referenced object.
171
   * 
172
   * @param session - the Session object containing the credentials for the Subject
173
   * @param pid - the identifier of the object to be described
174
   * 
175
   * @return describeResponse - A set of values providing a basic description 
176
   *                            of the object.
177
   * 
178
   * @throws InvalidToken
179
   * @throws ServiceFailure
180
   * @throws NotAuthorized
181
   * @throws NotFound
182
   * @throws NotImplemented
183
   * @throws InvalidRequest
184
   */
185
  public DescribeResponse describe(Session session, Identifier pid) 
186
      throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
187
      
188
      String serviceFailureCode = "4931";
189
      Identifier sid = getPIDForSID(pid, serviceFailureCode);
190
      if(sid != null) {
191
          pid = sid;
192
      }
193

    
194
    // get system metadata and construct the describe response
195
      SystemMetadata sysmeta = getSystemMetadata(session, pid);
196
      DescribeResponse describeResponse = 
197
      	new DescribeResponse(sysmeta.getFormatId(), sysmeta.getSize(), 
198
      			sysmeta.getDateSysMetadataModified(),
199
      			sysmeta.getChecksum(), sysmeta.getSerialVersion());
200

    
201
      return describeResponse;
202

    
203
  }
204
  
205
  /**
206
   * Deletes an object from the Member Node, where the object is either a 
207
   * data object or a science metadata object.
208
   * 
209
   * @param session - the Session object containing the credentials for the Subject
210
   * @param pid - The object identifier to be deleted
211
   * 
212
   * @return pid - the identifier of the object used for the deletion
213
   * 
214
   * @throws InvalidToken
215
   * @throws ServiceFailure
216
   * @throws NotAuthorized
217
   * @throws NotFound
218
   * @throws NotImplemented
219
   * @throws InvalidRequest
220
   */
221
  public Identifier delete(Session session, Identifier pid) 
222
      throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
223
      
224
      String localId = null;
225
      if (session == null) {
226
      	throw new InvalidToken("1330", "No session has been provided");
227
      }
228
      // just for logging purposes
229
      String username = session.getSubject().getValue();
230

    
231
      // do we have a valid pid?
232
      if (pid == null || pid.getValue().trim().equals("")) {
233
          throw new ServiceFailure("1350", "The provided identifier was invalid.");
234
      }
235

    
236
      // check for the existing identifier
237
      try {
238
          localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
239
      } catch (McdbDocNotFoundException e) {
240
          throw new NotFound("1340", "The object with the provided " + "identifier was not found.");
241
      } catch (SQLException e) {
242
          throw new ServiceFailure("1350", "The object with the provided " + "identifier "+pid.getValue()+" couldn't be identified since "+e.getMessage());
243
      }
244
      
245
      try {
246
          // delete the document, as admin
247
          DocumentImpl.delete(localId, null, null, null, true);
248
          EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), username, localId, Event.DELETE.xmlValue());
249

    
250
          // archive it
251
          // DocumentImpl.delete() now sets this
252
          // see https://redmine.dataone.org/issues/3406
253
//          SystemMetadata sysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
254
//          sysMeta.setArchived(true);
255
//          sysMeta.setDateSysMetadataModified(Calendar.getInstance().getTime());
256
//          HazelcastService.getInstance().getSystemMetadataMap().put(pid, sysMeta);
257
          
258
      } catch (McdbDocNotFoundException e) {
259
          throw new NotFound("1340", "The provided identifier was invalid.");
260

    
261
      } catch (SQLException e) {
262
          throw new ServiceFailure("1350", "There was a problem deleting the object." + "The error message was: " + e.getMessage());
263

    
264
      } catch (InsufficientKarmaException e) {
265
          if ( logMetacat.isDebugEnabled() ) {
266
              e.printStackTrace();
267
          }
268
          throw new NotAuthorized("1320", "The provided identity does not have " + "permission to DELETE objects on the Member Node.");
269
      
270
      } catch (Exception e) { // for some reason DocumentImpl throws a general Exception
271
          throw new ServiceFailure("1350", "There was a problem deleting the object." + "The error message was: " + e.getMessage());
272
      }
273

    
274
      return pid;
275
  }
276
  
277
  /**
278
   * Low level, "are you alive" operation. A valid ping response is 
279
   * indicated by a HTTP status of 200.
280
   * 
281
   * @return true if the service is alive
282
   * 
283
   * @throws NotImplemented
284
   * @throws ServiceFailure
285
   * @throws InsufficientResources
286
   */
287
  public Date ping() 
288
      throws NotImplemented, ServiceFailure, InsufficientResources {
289

    
290
      // test if we can get a database connection
291
      int serialNumber = -1;
292
      DBConnection dbConn = null;
293
      try {
294
          dbConn = DBConnectionPool.getDBConnection("MNodeService.ping");
295
          serialNumber = dbConn.getCheckOutSerialNumber();
296
      } catch (SQLException e) {
297
      	ServiceFailure sf = new ServiceFailure("", e.getMessage());
298
      	sf.initCause(e);
299
          throw sf;
300
      } finally {
301
          // Return the database connection
302
          DBConnectionPool.returnDBConnection(dbConn, serialNumber);
303
      }
304

    
305
      return Calendar.getInstance().getTime();
306
  }
307
  
308
  /**
309
   * Adds a new object to the Node, where the object is either a data 
310
   * object or a science metadata object. This method is called by clients 
311
   * to create new data objects on Member Nodes or internally for Coordinating
312
   * Nodes
313
   * 
314
   * @param session - the Session object containing the credentials for the Subject
315
   * @param pid - The object identifier to be created
316
   * @param object - the object bytes
317
   * @param sysmeta - the system metadata that describes the object  
318
   * 
319
   * @return pid - the object identifier created
320
   * 
321
   * @throws InvalidToken
322
   * @throws ServiceFailure
323
   * @throws NotAuthorized
324
   * @throws IdentifierNotUnique
325
   * @throws UnsupportedType
326
   * @throws InsufficientResources
327
   * @throws InvalidSystemMetadata
328
   * @throws NotImplemented
329
   * @throws InvalidRequest
330
   */
331
  public Identifier create(Session session, Identifier pid, InputStream object,
332
    SystemMetadata sysmeta) 
333
    throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, 
334
    UnsupportedType, InsufficientResources, InvalidSystemMetadata, 
335
    NotImplemented, InvalidRequest {
336

    
337
    Identifier resultPid = null;
338
    String localId = null;
339
    boolean allowed = false;
340
    
341
    // check for null session
342
    if (session == null) {
343
    	throw new InvalidToken("4894", "Session is required to WRITE to the Node.");
344
    }
345
    Subject subject = session.getSubject();
346

    
347
    Subject publicSubject = new Subject();
348
    publicSubject.setValue(Constants.SUBJECT_PUBLIC);
349
	// be sure the user is authenticated for create()
350
    if (subject == null || subject.getValue() == null || 
351
        subject.equals(publicSubject) ) {
352
      throw new NotAuthorized("1100", "The provided identity does not have " +
353
        "permission to WRITE to the Node.");
354
      
355
    }
356
        
357
    // verify that pid == SystemMetadata.getIdentifier()
358
    logMetacat.debug("Comparing pid|sysmeta_pid: " + 
359
      pid.getValue() + "|" + sysmeta.getIdentifier().getValue());
360
    if (!pid.getValue().equals(sysmeta.getIdentifier().getValue())) {
361
        throw new InvalidSystemMetadata("1180", 
362
            "The supplied system metadata is invalid. " +
363
            "The identifier " + pid.getValue() + " does not match identifier" +
364
            "in the system metadata identified by " +
365
            sysmeta.getIdentifier().getValue() + ".");
366
        
367
    }
368
    
369

    
370
    logMetacat.debug("Checking if identifier exists: " + pid.getValue());
371
    // Check that the identifier does not already exist
372
    boolean idExists = false;
373
    try {
374
        idExists = IdentifierManager.getInstance().identifierExists(pid.getValue());
375
    } catch (SQLException e) {
376
        throw new ServiceFailure("1190", 
377
                                "The requested identifier " + pid.getValue() +
378
                                " couldn't be determined if it is unique since : "+e.getMessage());
379
    }
380
    if (idExists) {
381
	    	throw new IdentifierNotUnique("1120", 
382
			          "The requested identifier " + pid.getValue() +
383
			          " is already used by another object and" +
384
			          "therefore can not be used for this object. Clients should choose" +
385
			          "a new identifier that is unique and retry the operation or " +
386
			          "use CN.reserveIdentifier() to reserve one.");
387
    	
388
    }
389
    
390
    
391
    // TODO: this probably needs to be refined more
392
    try {
393
      allowed = isAuthorized(session, pid, Permission.WRITE);
394
            
395
    } catch (NotFound e) {
396
      // The identifier doesn't exist, writing should be fine.
397
      allowed = true;
398
    }
399
    
400
    if(!allowed) {
401
        throw new NotAuthorized("1100", "Provited Identity doesn't have the WRITE permission on the pid "+pid.getValue());
402
    }
403
    // verify checksum, only if we can reset the inputstream
404
    if (object.markSupported()) {
405
        logMetacat.debug("Checking checksum for: " + pid.getValue());
406
	    String checksumAlgorithm = sysmeta.getChecksum().getAlgorithm();
407
	    String checksumValue = sysmeta.getChecksum().getValue();
408
	    try {
409
			String computedChecksumValue = ChecksumUtil.checksum(object, checksumAlgorithm).getValue();
410
			// it's very important that we don't consume the stream
411
			object.reset();
412
			if (!computedChecksumValue.equals(checksumValue)) {
413
			    logMetacat.error("Checksum for " + pid.getValue() + " does not match system metadata, computed = " + computedChecksumValue );
414
				throw new InvalidSystemMetadata("4896", "Checksum given does not match that of the object");
415
			}
416
		} catch (Exception e) {
417
			String msg = "Error verifying checksum values";
418
	      	logMetacat.error(msg, e);
419
	        throw new ServiceFailure("1190", msg + ": " + e.getMessage());
420
		}
421
    } else {
422
    	logMetacat.warn("mark is not supported on the object's input stream - cannot verify checksum without consuming stream");
423
    }
424
    	
425
    // we have the go ahead
426
    //if ( allowed ) {
427
         
428
        // save the sysmeta
429
        try {
430
            // lock and unlock of the pid happens in the subclass
431
            HazelcastService.getInstance().getSystemMetadataMap().put(sysmeta.getIdentifier(), sysmeta);
432
           
433
        
434
        } catch (Exception e) {
435
            logMetacat.error("Problem creating system metadata: " + pid.getValue(), e);
436
            throw new ServiceFailure("1190", e.getMessage());
437
        }
438
      
439
        logMetacat.debug("Allowed to insert: " + pid.getValue());
440

    
441
      // Science metadata (XML) or science data object?
442
      // TODO: there are cases where certain object formats are science metadata
443
      // but are not XML (netCDF ...).  Handle this.
444
      if ( isScienceMetadata(sysmeta) ) {
445
        
446
        // CASE METADATA:
447
      	//String objectAsXML = "";
448
        try {
449
	        //objectAsXML = IOUtils.toString(object, "UTF-8");
450
	        localId = insertOrUpdateDocument(object,"UTF-8", pid, session, "insert");
451
	        //localId = im.getLocalId(pid.getValue());
452

    
453
        } catch (IOException e) {
454
            removeSystemMeta(pid);
455
        	String msg = "The Node is unable to create the object. " +
456
          "There was a problem converting the object to XML";
457
        	logMetacat.info(msg);
458
          throw new ServiceFailure("1190", msg + ": " + e.getMessage());
459

    
460
        } catch (ServiceFailure e) {
461
            removeSystemMeta(pid);
462
            throw e;
463
        } catch (Exception e) {
464
            removeSystemMeta(pid);
465
            throw new ServiceFailure("1190", "The node is unable to create the object: " + e.getMessage());
466
        }
467
                    
468
      } else {
469
	        
470
	      // DEFAULT CASE: DATA (needs to be checked and completed)
471
          try {
472
              localId = insertDataObject(object, pid, session);
473
          } catch (ServiceFailure e) {
474
              removeSystemMeta(pid);
475
              throw e;
476
          } catch (Exception e) {
477
              removeSystemMeta(pid);
478
              throw new ServiceFailure("1190", "The node is unable to create the object: " + e.getMessage());
479
          }
480
	      
481
      }   
482
    
483
    //}
484

    
485
    logMetacat.debug("Done inserting new object: " + pid.getValue());
486
    
487
    // setting the resulting identifier failed
488
    if (localId == null ) {
489
        removeSystemMeta(pid);
490
      throw new ServiceFailure("1190", "The Node is unable to create the object. ");
491
    }
492
    
493
    try {
494
        // submit for indexing
495
        MetacatSolrIndex.getInstance().submit(sysmeta.getIdentifier(), sysmeta, null, true);
496
    } catch (Exception e) {
497
        logMetacat.warn("Couldn't create solr index for object "+pid.getValue());
498
    }
499

    
500
    resultPid = pid;
501
    
502
    logMetacat.debug("create() complete for object: " + pid.getValue());
503

    
504
    return resultPid;
505
  }
506
  
507
  /*
508
   * Roll-back method when inserting data object fails.
509
   */
510
  protected void removeSystemMeta(Identifier id){
511
      HazelcastService.getInstance().getSystemMetadataMap().remove(id);
512
  }
513
  
514
  /*
515
   * Roll-back method when inserting data object fails.
516
   */
517
  protected void removeSolrIndex(SystemMetadata sysMeta) {
518
      sysMeta.setSerialVersion(sysMeta.getSerialVersion().add(BigInteger.ONE));
519
      sysMeta.setArchived(true);
520
      sysMeta.setDateSysMetadataModified(Calendar.getInstance().getTime());
521
      try {
522
          MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null, false);
523
      } catch (Exception e) {
524
          logMetacat.warn("Can't remove the solr index for pid "+sysMeta.getIdentifier().getValue());
525
      }
526
      
527
  }
528

    
529
  /**
530
   * Return the log records associated with a given event between the start and 
531
   * end dates listed given a particular Subject listed in the Session
532
   * 
533
   * @param session - the Session object containing the credentials for the Subject
534
   * @param fromDate - the start date of the desired log records
535
   * @param toDate - the end date of the desired log records
536
   * @param event - restrict log records of a specific event type
537
   * @param start - zero based offset from the first record in the 
538
   *                set of matching log records. Used to assist with 
539
   *                paging the response.
540
   * @param count - maximum number of log records to return in the response. 
541
   *                Used to assist with paging the response.
542
   * 
543
   * @return the desired log records
544
   * 
545
   * @throws InvalidToken
546
   * @throws ServiceFailure
547
   * @throws NotAuthorized
548
   * @throws InvalidRequest
549
   * @throws NotImplemented
550
   */
551
  public Log getLogRecords(Session session, Date fromDate, Date toDate, 
552
      String event, String pidFilter, Integer start, Integer count) throws InvalidToken, ServiceFailure,
553
      NotAuthorized, InvalidRequest, NotImplemented {
554

    
555
	  // only admin access to this method
556
	  // see https://redmine.dataone.org/issues/2855
557
	  if (!isAdminAuthorized(session)) {
558
		  throw new NotAuthorized("1460", "Only the CN or admin is allowed to harvest logs from this node");
559
	  }
560
    Log log = new Log();
561
    IdentifierManager im = IdentifierManager.getInstance();
562
    EventLog el = EventLog.getInstance();
563
    if ( fromDate == null ) {
564
      logMetacat.debug("setting fromdate from null");
565
      fromDate = new Date(1);
566
    }
567
    if ( toDate == null ) {
568
      logMetacat.debug("setting todate from null");
569
      toDate = new Date();
570
    }
571

    
572
    if ( start == null ) {
573
    	start = 0;	
574
    }
575
    
576
    if ( count == null ) {
577
    	count = 1000;
578
    }
579
    
580
    // safeguard against large requests
581
    if (count > MAXIMUM_DB_RECORD_COUNT) {
582
    	count = MAXIMUM_DB_RECORD_COUNT;
583
    }
584

    
585
    String[] filterDocid = null;
586
    if (pidFilter != null && !pidFilter.trim().equals("")) {
587
        //check if the given identifier is a sid. If it is sid, choose the current pid of the sid.
588
        Identifier pid = new Identifier();
589
        pid.setValue(pidFilter);
590
        String serviceFailureCode = "1490";
591
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
592
        if(sid != null) {
593
            pid = sid;
594
        }
595
        pidFilter = pid.getValue();
596
		try {
597
	      String localId = im.getLocalId(pidFilter);
598
	      filterDocid = new String[] {localId};
599
	    } catch (Exception ex) { 
600
	    	String msg = "Could not find localId for given pidFilter '" + pidFilter + "'";
601
	        logMetacat.warn(msg, ex);
602
	        //throw new InvalidRequest("1480", msg);
603
	        return log; //return 0 record
604
	    }
605
    }
606
    
607
    logMetacat.debug("fromDate: " + fromDate);
608
    logMetacat.debug("toDate: " + toDate);
609

    
610
    log = el.getD1Report(null, null, filterDocid, event,
611
        new java.sql.Timestamp(fromDate.getTime()),
612
        new java.sql.Timestamp(toDate.getTime()), false, start, count);
613
    
614
    logMetacat.info("getLogRecords");
615
    return log;
616
  }
617
    
618
  /**
619
   * Return the object identified by the given object identifier
620
   * 
621
   * @param session - the Session object containing the credentials for the Subject
622
   * @param pid - the object identifier for the given object
623
   * 
624
   * TODO: The D1 Authorization API doesn't provide information on which 
625
   * authentication system the Subject belongs to, and so it's not possible to
626
   * discern which Person or Group is a valid KNB LDAP DN.  Fix this.
627
   * 
628
   * @return inputStream - the input stream of the given object
629
   * 
630
   * @throws InvalidToken
631
   * @throws ServiceFailure
632
   * @throws NotAuthorized
633
   * @throws InvalidRequest
634
   * @throws NotImplemented
635
   */
636
  public InputStream get(Session session, Identifier pid) 
637
    throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, 
638
    NotImplemented {
639
    
640
    String serviceFailureCode = "1030";
641
    Identifier sid = getPIDForSID(pid, serviceFailureCode);
642
    if(sid != null) {
643
        pid = sid;
644
    }
645
    
646
    InputStream inputStream = null; // bytes to be returned
647
    handler = new MetacatHandler(new Timer());
648
    boolean allowed = false;
649
    String localId; // the metacat docid for the pid
650
    
651
    // get the local docid from Metacat
652
    try {
653
      localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
654
    
655
    } catch (McdbDocNotFoundException e) {
656
      throw new NotFound("1020", "The object specified by " + 
657
                         pid.getValue() +
658
                         " does not exist at this node.");
659
    } catch (SQLException e) {
660
        throw new ServiceFailure("1030", "The object specified by "+ pid.getValue()+
661
                                  " couldn't be identified at this node since "+e.getMessage());
662
    }
663
    
664
    // check for authorization
665
    try {
666
		allowed = isAuthorized(session, pid, Permission.READ);
667
	} catch (InvalidRequest e) {
668
		throw new ServiceFailure("1030", e.getDescription());
669
	}
670
    
671
    // if the person is authorized, perform the read
672
    if (allowed) {
673
      try {
674
        inputStream = handler.read(localId);
675
      } catch (McdbDocNotFoundException de) {
676
          String error ="";
677
          if(EventLog.getInstance().isDeleted(localId)) {
678
                error=DELETEDMESSAGE;
679
          }
680
          throw new NotFound("1020", "The object specified by " + 
681
                           pid.getValue() +
682
                           " does not exist at this node. "+error);
683
      } catch (Exception e) {
684
        throw new ServiceFailure("1030", "The object specified by " + 
685
            pid.getValue() +
686
            " could not be returned due to error: " +
687
            e.getMessage()+". ");
688
      }
689
    }
690

    
691
    // if we fail to set the input stream
692
    if ( inputStream == null ) {
693
        String error ="";
694
        if(EventLog.getInstance().isDeleted(localId)) {
695
              error=DELETEDMESSAGE;
696
        }
697
        throw new NotFound("1020", "The object specified by " + 
698
                         pid.getValue() +
699
                         " does not exist at this node. "+error);
700
    }
701
    
702
	// log the read event
703
    String principal = Constants.SUBJECT_PUBLIC;
704
    if (session != null && session.getSubject() != null) {
705
    	principal = session.getSubject().getValue();
706
    }
707
    EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), principal, localId, "read");
708
    
709
    return inputStream;
710
  }
711

    
712
  /**
713
   * Return the system metadata for a given object
714
   * 
715
   * @param session - the Session object containing the credentials for the Subject
716
   * @param pid - the object identifier for the given object
717
   * 
718
   * @return inputStream - the input stream of the given system metadata object
719
   * 
720
   * @throws InvalidToken
721
   * @throws ServiceFailure
722
   * @throws NotAuthorized
723
   * @throws NotFound
724
   * @throws InvalidRequest
725
   * @throws NotImplemented
726
   */
727
    public SystemMetadata getSystemMetadata(Session session, Identifier pid)
728
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
729
        NotImplemented {
730

    
731
        String serviceFailureCode = "1090";
732
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
733
        if(sid != null) {
734
            pid = sid;
735
        }
736
        boolean isAuthorized = false;
737
        SystemMetadata systemMetadata = null;
738
        List<Replica> replicaList = null;
739
        NodeReference replicaNodeRef = null;
740
        List<Node> nodeListBySubject = null;
741
        Subject subject = null;
742
        
743
        if (session != null ) {
744
            subject = session.getSubject();
745
        }
746
        
747
        // check normal authorization
748
        BaseException originalAuthorizationException = null;
749
        if (!isAuthorized) {
750
            try {
751
                isAuthorized = isAuthorized(session, pid, Permission.READ);
752

    
753
            } catch (InvalidRequest e) {
754
                throw new ServiceFailure("1090", e.getDescription());
755
            } catch (NotAuthorized nae) {
756
            	// catch this for later
757
            	originalAuthorizationException = nae;
758
			}
759
        }
760
        
761
        // get the system metadata first because we need the replica list for auth
762
        systemMetadata = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
763
        
764
        // check the replica information to expand access to MNs that might need it
765
        if (!isAuthorized) {
766
        	
767
	        try {
768
	        	
769
	            // if MNs are listed as replicas, allow access
770
	            if ( systemMetadata != null ) {
771
	                replicaList = systemMetadata.getReplicaList();
772
	                // only check if there are in fact replicas listed
773
	                if ( replicaList != null ) {
774
	                    
775
	                    if ( subject != null ) {
776
	                        // get the list of nodes with a matching node subject
777
	                        try {
778
	                            nodeListBySubject = listNodesBySubject(session.getSubject());
779
	
780
	                        } catch (BaseException e) {
781
	                            // Unexpected error contacting the CN via D1Client
782
	                            String msg = "Caught an unexpected error while trying "
783
	                                    + "to potentially authorize system metadata access "
784
	                                    + "based on the session subject. The error was "
785
	                                    + e.getMessage();
786
	                            logMetacat.error(msg);
787
	                            if (logMetacat.isDebugEnabled()) {
788
	                                e.printStackTrace();
789
	
790
	                            }
791
	                            // isAuthorized is still false 
792
	                        }
793
	
794
	                    }
795
	                    if (nodeListBySubject != null) {
796
	                        // compare node ids to replica node ids
797
	                        outer: for (Replica replica : replicaList) {
798
	                            replicaNodeRef = replica.getReplicaMemberNode();
799
	
800
	                            for (Node node : nodeListBySubject) {
801
	                                if (node.getIdentifier().equals(replicaNodeRef)) {
802
	                                    // node id via session subject matches a replica node
803
	                                    isAuthorized = true;
804
	                                    break outer;
805
	                                }
806
	                            }
807
	                        }
808
	                    }
809
	                }
810
	            }
811
	            
812
	            // if we still aren't authorized, then we are done
813
	            if (!isAuthorized) {
814
	                throw new NotAuthorized("1400", Permission.READ
815
	                        + " not allowed on " + pid.getValue());
816
	            }
817

    
818
	        } catch (RuntimeException e) {
819
	        	e.printStackTrace();
820
	            // convert hazelcast RuntimeException to ServiceFailure
821
	            throw new ServiceFailure("1090", "Unexpected error getting system metadata for: " + 
822
	                pid.getValue());	
823
	        }
824
	        
825
        }
826
        
827
        // It wasn't in the map
828
        if ( systemMetadata == null ) {
829
            String error ="";
830
            String localId = null;
831
            try {
832
                localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
833
              
834
             } catch (Exception e) {
835
                logMetacat.warn("Couldn't find the local id for the pid "+pid.getValue());
836
            }
837
            
838
            if(localId != null && EventLog.getInstance().isDeleted(localId)) {
839
                error = DELETEDMESSAGE;
840
            } else if (localId == null && EventLog.getInstance().isDeleted(pid.getValue())) {
841
                error = DELETEDMESSAGE;
842
            }
843
            throw new NotFound("1420", "No record found for: " + pid.getValue()+". "+error);
844
        }
845
        
846
        return systemMetadata;
847
    }
848
     
849
    
850
    /**
851
     * Test if the specified session represents the authoritative member node for the
852
     * given object specified by the identifier. According the the DataONE documentation, 
853
     * the authoritative member node has all the rights of the *rightsHolder*.
854
     * @param session - the Session object containing the credentials for the Subject
855
     * @param pid - the Identifier of the data object
856
     * @return true if the session represents the authoritative mn.
857
     * @throws ServiceFailure 
858
     * @throws NotImplemented 
859
     */
860
    public boolean isAuthoritativeMNodeAdmin(Session session, Identifier pid) {
861
        boolean allowed = false;
862
        //check the parameters
863
        if(session == null) {
864
            logMetacat.debug("D1NodeService.isAuthoritativeMNodeAdmin - the session object is null and return false.");
865
            return allowed;
866
        } else if (pid == null || pid.getValue() == null || pid.getValue().trim().equals("")) {
867
            logMetacat.debug("D1NodeService.isAuthoritativeMNodeAdmin - the Identifier object is null (not being specified) and return false.");
868
            return allowed;
869
        }
870
        
871
        //Get the subject from the session
872
        Subject subject = session.getSubject();
873
        if(subject != null) {
874
            //Get the authoritative member node info from the system metadata
875
            SystemMetadata sysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
876
            if(sysMeta != null) {
877
                NodeReference authoritativeMNode = sysMeta.getAuthoritativeMemberNode();
878
                if(authoritativeMNode != null) {
879
                        CNode cn = null;
880
                        try {
881
                            cn = D1Client.getCN();
882
                        } catch (BaseException e) {
883
                            logMetacat.error("D1NodeService.isAuthoritativeMNodeAdmin - couldn't connect to the CN since "+
884
                                            e.getDescription()+ ". The false value will be returned for the AuthoritativeMNodeAdmin.");
885
                            return allowed;
886
                        }
887
                        
888
                        if(cn != null) {
889
                            List<Node> nodes = null;
890
                            try {
891
                                nodes = cn.listNodes().getNodeList();
892
                            } catch (NotImplemented e) {
893
                                logMetacat.error("D1NodeService.isAuthoritativeMNodeAdmin - couldn't get the member nodes list from the CN since "+e.getDescription()+ 
894
                                                ". The false value will be returned for the AuthoritativeMNodeAdmin.");
895
                                return allowed;
896
                            } catch (ServiceFailure ee) {
897
                                logMetacat.error("D1NodeService.isAuthoritativeMNodeAdmin - couldn't get the member nodes list from the CN since "+ee.getDescription()+ 
898
                                                ". The false value will be returned for the AuthoritativeMNodeAdmin.");
899
                                return allowed;
900
                            }
901
                            if(nodes != null) {
902
                                for(Node node : nodes) {
903
                                    //find the authoritative node and get its subjects
904
                                    if (node.getType() == NodeType.MN && node.getIdentifier() != null && node.getIdentifier().equals(authoritativeMNode)) {
905
                                        List<Subject> nodeSubjects = node.getSubjectList();
906
                                        if(nodeSubjects != null) {
907
                                            // check if the session subject is in the node subject list
908
                                            for (Subject nodeSubject : nodeSubjects) {
909
                                                logMetacat.debug("D1NodeService.isAuthoritativeMNodeAdmin(), comparing subjects: " +
910
                                                    nodeSubject.getValue() + " and " + subject.getValue());
911
                                                if ( nodeSubject != null && nodeSubject.equals(subject) ) {
912
                                                    allowed = true; // subject of session == target node subject
913
                                                    break;
914
                                                }
915
                                            }              
916
                                        }
917
                                      
918
                                    }
919
                                }
920
                            }
921
                        }
922
                }
923
            }
924
        }
925
        return allowed;
926
    }
927
    
928
    
929
  /**
930
   * Test if the user identified by the provided token has administrative authorization 
931
   * 
932
   * @param session - the Session object containing the credentials for the Subject
933
   * 
934
   * @return true if the user is admin (mn itself or a cn )
935
   * 
936
   * @throws ServiceFailure
937
   * @throws InvalidToken
938
   * @throws NotFound
939
   * @throws NotAuthorized
940
   * @throws NotImplemented
941
   */
942
  public boolean isAdminAuthorized(Session session) 
943
      throws ServiceFailure, InvalidToken, NotAuthorized,
944
      NotImplemented {
945

    
946
      boolean allowed = false;
947
      
948
      // must have a session in order to check admin 
949
      if (session == null) {
950
         logMetacat.debug("In isAdminAuthorized(), session is null ");
951
         return false;
952
      }
953
      
954
      logMetacat.debug("In isAdminAuthorized(), checking CN or MN authorization for " +
955
           session.getSubject().getValue());
956
      
957
      // check if this is the node calling itself (MN)
958
      allowed = isNodeAdmin(session);
959
      
960
      // check the CN list
961
      if (!allowed) {
962
	      allowed = isCNAdmin(session);
963
      }
964
      
965
      return allowed;
966
  }
967
  
968
  /*
969
   * Determine if the specified session is a CN or not. Return true if it is; otherwise false.
970
   */
971
  protected boolean isCNAdmin (Session session) {
972
      boolean allowed = false;
973
      List<Node> nodes = null;
974

    
975
      try {
976
          // are we allowed to do this? only CNs are allowed
977
          CNode cn = D1Client.getCN();
978
          nodes = cn.listNodes().getNodeList();
979
      }
980
      catch (Throwable e) {
981
          logMetacat.warn(e.getMessage());
982
          return false;  
983
      }
984
          
985
      if ( nodes == null ) {
986
          return false;
987
          //throw new ServiceFailure("4852", "Couldn't get node list.");
988
      }
989
      
990
      // find the node in the node list
991
      for ( Node node : nodes ) {
992
          
993
          NodeReference nodeReference = node.getIdentifier();
994
          logMetacat.debug("In isAdminAuthorized(), Node reference is: " + nodeReference.getValue());
995
          
996
          Subject subject = session.getSubject();
997
          
998
          if (node.getType() == NodeType.CN) {
999
              List<Subject> nodeSubjects = node.getSubjectList();
1000
              
1001
              // check if the session subject is in the node subject list
1002
              for (Subject nodeSubject : nodeSubjects) {
1003
                  logMetacat.debug("In isAdminAuthorized(), comparing subjects: " +
1004
                      nodeSubject.getValue() + " and " + subject.getValue());
1005
                  if ( nodeSubject.equals(subject) ) {
1006
                      allowed = true; // subject of session == target node subject
1007
                      break;
1008
                      
1009
                  }
1010
              }              
1011
          }
1012
      }
1013
      return allowed;
1014
  }
1015
  
1016
  /**
1017
   * Test if the user identified by the provided token has administrative authorization 
1018
   * on this node because they are calling themselves
1019
   * 
1020
   * @param session - the Session object containing the credentials for the Subject
1021
   * 
1022
   * @return true if the user is this node
1023
   * @throws ServiceFailure 
1024
   * @throws NotImplemented 
1025
   */
1026
  public boolean isNodeAdmin(Session session) throws NotImplemented, ServiceFailure {
1027

    
1028
      boolean allowed = false;
1029
      
1030
      // must have a session in order to check admin 
1031
      if (session == null) {
1032
         logMetacat.debug("In isNodeAdmin(), session is null ");
1033
         return false;
1034
      }
1035
      
1036
      logMetacat.debug("In isNodeAdmin(), MN authorization for " +
1037
           session.getSubject().getValue());
1038
      
1039
      Node node = MNodeService.getInstance(request).getCapabilities();
1040
      NodeReference nodeReference = node.getIdentifier();
1041
      logMetacat.debug("In isNodeAdmin(), Node reference is: " + nodeReference.getValue());
1042
      
1043
      Subject subject = session.getSubject();
1044
      
1045
      if (node.getType() == NodeType.MN) {
1046
          List<Subject> nodeSubjects = node.getSubjectList();
1047
          
1048
          // check if the session subject is in the node subject list
1049
          for (Subject nodeSubject : nodeSubjects) {
1050
              logMetacat.debug("In isNodeAdmin(), comparing subjects: " +
1051
                  nodeSubject.getValue() + " and " + subject.getValue());
1052
              if ( nodeSubject.equals(subject) ) {
1053
                  allowed = true; // subject of session == this node's subect
1054
                  break;
1055
              }
1056
          }              
1057
      }
1058
      
1059
      return allowed;
1060
  }
1061
  
1062
  /**
1063
   * Test if the user identified by the provided token has authorization 
1064
   * for the operation on the specified object.
1065
   * Allowed subjects include:
1066
   * 1. CNs
1067
   * 2. Authoritative node
1068
   * 3. Owner of the object
1069
   * 4. Users with the specified permission in the access rules.
1070
   * 
1071
   * @param session - the Session object containing the credentials for the Subject
1072
   * @param pid - The identifer of the resource for which access is being checked
1073
   * @param operation - The type of operation which is being requested for the given pid
1074
   *
1075
   * @return true if the operation is allowed
1076
   * 
1077
   * @throws ServiceFailure
1078
   * @throws InvalidToken
1079
   * @throws NotFound
1080
   * @throws NotAuthorized
1081
   * @throws NotImplemented
1082
   * @throws InvalidRequest
1083
   */
1084
  public boolean isAuthorized(Session session, Identifier pid, Permission permission)
1085
    throws ServiceFailure, InvalidToken, NotFound, NotAuthorized,
1086
    NotImplemented, InvalidRequest {
1087

    
1088
    boolean allowed = false;
1089
    
1090
    if (permission == null) {
1091
    	throw new InvalidRequest("1761", "Permission was not provided or is invalid");
1092
    }
1093
    
1094
    // always allow CN access
1095
    if ( isAdminAuthorized(session) ) {
1096
        allowed = true;
1097
        return allowed;
1098
        
1099
    }
1100
    
1101
    String serviceFailureCode = "1760";
1102
    Identifier sid = getPIDForSID(pid, serviceFailureCode);
1103
    if(sid != null) {
1104
        pid = sid;
1105
    }
1106
    
1107
    // the authoritative member node of the pid always has the access as well.
1108
    if (isAuthoritativeMNodeAdmin(session, pid)) {
1109
        allowed = true;
1110
        return allowed;
1111
    }
1112
    
1113
    //is it the owner of the object or the access rules allow the user?
1114
    allowed = userHasPermission(session,  pid, permission );
1115
    
1116
    // throw or return?
1117
    if (!allowed) {
1118
     // track the identities we have checked against
1119
      StringBuffer includedSubjects = new StringBuffer();
1120
      Set<Subject> subjects = AuthUtils.authorizedClientSubjects(session);
1121
      for (Subject s: subjects) {
1122
             includedSubjects.append(s.getValue() + "; ");
1123
        }    
1124
      throw new NotAuthorized("1820", permission + " not allowed on " + pid.getValue() + " for subject[s]: " + includedSubjects.toString() );
1125
    }
1126
    
1127
    return allowed;
1128
    
1129
  }
1130
  
1131
  
1132
  /*
1133
   * Determine if a user has the permission to perform the specified permission.
1134
   * 1. Owner can have any permission.
1135
   * 2. Access table allow the user has the permission
1136
   */
1137
  protected boolean userHasPermission(Session userSession, Identifier pid, Permission permission ) throws NotFound{
1138
      boolean allowed = false;
1139
      // permissions are hierarchical
1140
      List<Permission> expandedPermissions = null;
1141
      // get the subject[s] from the session
1142
      //defer to the shared util for recursively compiling the subjects   
1143
      Set<Subject> subjects = AuthUtils.authorizedClientSubjects(userSession);
1144
          
1145
      // get the system metadata
1146
      String pidStr = pid.getValue();
1147
      SystemMetadata systemMetadata = null;
1148
      try {
1149
          systemMetadata = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
1150

    
1151
      } catch (Exception e) {
1152
          // convert Hazelcast RuntimeException to NotFound
1153
          logMetacat.error("An error occurred while getting system metadata for identifier " +
1154
              pid.getValue() + ". The error message was: " + e.getMessage());
1155
          throw new NotFound("1800", "No record found for " + pidStr);
1156
          
1157
      } 
1158
      
1159
      // throw not found if it was not found
1160
      if (systemMetadata == null) {
1161
          String localId = null;
1162
          String error = "No system metadata could be found for given PID: " + pidStr;
1163
          try {
1164
              localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1165
            
1166
           } catch (Exception e) {
1167
              logMetacat.warn("Couldn't find the local id for the pid "+pidStr);
1168
          }
1169
          
1170
          if(localId != null && EventLog.getInstance().isDeleted(localId)) {
1171
              error = error + ". "+DELETEDMESSAGE;
1172
          } else if (localId == null && EventLog.getInstance().isDeleted(pid.getValue())) {
1173
              error = error + ". "+DELETEDMESSAGE;
1174
          }
1175
          throw new NotFound("1800", error);
1176
      }
1177
          
1178
      // do we own it?
1179
      for (Subject s: subjects) {
1180
        logMetacat.debug("Comparing \t" + 
1181
                         systemMetadata.getRightsHolder().getValue() +
1182
                         " \tagainst \t" + s.getValue());
1183
          //includedSubjects.append(s.getValue() + "; ");
1184
          allowed = systemMetadata.getRightsHolder().equals(s);
1185
          if (allowed) {
1186
              return allowed;
1187
          }
1188
      }    
1189
      
1190
      // otherwise check the access rules
1191
      try {
1192
          List<AccessRule> allows = systemMetadata.getAccessPolicy().getAllowList();
1193
          search: // label break
1194
          for (AccessRule accessRule: allows) {
1195
            for (Subject s: subjects) {
1196
              logMetacat.debug("Checking allow access rule for subject: " + s.getValue());
1197
              if (accessRule.getSubjectList().contains(s)) {
1198
                  logMetacat.debug("Access rule contains subject: " + s.getValue());
1199
                  for (Permission p: accessRule.getPermissionList()) {
1200
                      logMetacat.debug("Checking permission: " + p.xmlValue());
1201
                      expandedPermissions = expandPermissions(p);
1202
                      allowed = expandedPermissions.contains(permission);
1203
                      if (allowed) {
1204
                          logMetacat.info("Permission granted: " + p.xmlValue() + " to " + s.getValue());
1205
                          break search; //label break
1206
                      }
1207
                  }
1208
                  
1209
              }
1210
            }
1211
          }
1212
      } catch (Exception e) {
1213
          // catch all for errors - safe side should be to deny the access
1214
          logMetacat.error("Problem checking authorization - defaulting to deny", e);
1215
          allowed = false;
1216
        
1217
      }
1218
      return allowed;
1219
  }
1220
  /*
1221
   * parse a logEntry and get the relevant field from it
1222
   * 
1223
   * @param fieldname
1224
   * @param entry
1225
   * @return
1226
   */
1227
  private String getLogEntryField(String fieldname, String entry) {
1228
    String begin = "<" + fieldname + ">";
1229
    String end = "</" + fieldname + ">";
1230
    // logMetacat.debug("looking for " + begin + " and " + end +
1231
    // " in entry " + entry);
1232
    String s = entry.substring(entry.indexOf(begin) + begin.length(), entry
1233
        .indexOf(end));
1234
    logMetacat.debug("entry " + fieldname + " : " + s);
1235
    return s;
1236
  }
1237

    
1238
  /** 
1239
   * Determine if a given object should be treated as an XML science metadata
1240
   * object. 
1241
   * 
1242
   * @param sysmeta - the SystemMetadata describing the object
1243
   * @return true if the object should be treated as science metadata
1244
   */
1245
  public static boolean isScienceMetadata(SystemMetadata sysmeta) {
1246
    
1247
    ObjectFormat objectFormat = null;
1248
    boolean isScienceMetadata = false;
1249
    
1250
    try {
1251
      objectFormat = ObjectFormatCache.getInstance().getFormat(sysmeta.getFormatId());
1252
      if ( objectFormat.getFormatType().equals("METADATA") ) {
1253
      	isScienceMetadata = true;
1254
      	
1255
      }
1256
      
1257
       
1258
    } catch (ServiceFailure e) {
1259
      logMetacat.debug("There was a problem determining if the object identified by" + 
1260
          sysmeta.getIdentifier().getValue() + 
1261
          " is science metadata: " + e.getMessage());
1262
    
1263
    } catch (NotFound e) {
1264
      logMetacat.debug("There was a problem determining if the object identified by" + 
1265
          sysmeta.getIdentifier().getValue() + 
1266
          " is science metadata: " + e.getMessage());
1267
    
1268
    }
1269
    
1270
    return isScienceMetadata;
1271

    
1272
  }
1273
  
1274
  /**
1275
   * Check fro whitespace in the given pid.
1276
   * null pids are also invalid by default
1277
   * @param pid
1278
   * @return
1279
   */
1280
  public static boolean isValidIdentifier(Identifier pid) {
1281
	  if (pid != null && pid.getValue() != null && pid.getValue().length() > 0) {
1282
		  return !pid.getValue().matches(".*\\s+.*");
1283
	  } 
1284
	  return false;
1285
  }
1286
  
1287
  
1288
  /**
1289
   * Insert or update an XML document into Metacat
1290
   * 
1291
   * @param xml - the XML document to insert or update
1292
   * @param pid - the identifier to be used for the resulting object
1293
   * 
1294
   * @return localId - the resulting docid of the document created or updated
1295
   * 
1296
   */
1297
  public String insertOrUpdateDocument(InputStream xml, String encoding,  Identifier pid, 
1298
    Session session, String insertOrUpdate) 
1299
    throws ServiceFailure, IOException {
1300
    
1301
  	logMetacat.debug("Starting to insert xml document...");
1302
    IdentifierManager im = IdentifierManager.getInstance();
1303

    
1304
    // generate pid/localId pair for sysmeta
1305
    String localId = null;
1306
    byte[] xmlBytes  = IOUtils.toByteArray(xml);
1307
    String xmlStr = new String(xmlBytes, encoding);
1308
    if(insertOrUpdate.equals("insert")) {
1309
      localId = im.generateLocalId(pid.getValue(), 1);
1310
      
1311
    } else {
1312
      //localid should already exist in the identifier table, so just find it
1313
      try {
1314
        logMetacat.debug("Updating pid " + pid.getValue());
1315
        logMetacat.debug("looking in identifier table for pid " + pid.getValue());
1316
        
1317
        localId = im.getLocalId(pid.getValue());
1318
        
1319
        logMetacat.debug("localId: " + localId);
1320
        //increment the revision
1321
        String docid = localId.substring(0, localId.lastIndexOf("."));
1322
        String revS = localId.substring(localId.lastIndexOf(".") + 1, localId.length());
1323
        int rev = new Integer(revS).intValue();
1324
        rev++;
1325
        docid = docid + "." + rev;
1326
        localId = docid;
1327
        logMetacat.debug("incremented localId: " + localId);
1328
      
1329
      } catch(McdbDocNotFoundException e) {
1330
        throw new ServiceFailure("1030", "D1NodeService.insertOrUpdateDocument(): " +
1331
            "pid " + pid.getValue() + 
1332
            " should have been in the identifier table, but it wasn't: " + 
1333
            e.getMessage());
1334
      
1335
      } catch (SQLException e) {
1336
          throw new ServiceFailure("1030", "D1NodeService.insertOrUpdateDocument() -"+
1337
                     " couldn't identify if the pid "+pid.getValue()+" is in the identifier table since "+e.getMessage());
1338
      }
1339
      
1340
    }
1341

    
1342
    params = new Hashtable<String, String[]>();
1343
    String[] action = new String[1];
1344
    action[0] = insertOrUpdate;
1345
    params.put("action", action);
1346
    String[] docid = new String[1];
1347
    docid[0] = localId;
1348
    params.put("docid", docid);
1349
    String[] doctext = new String[1];
1350
    doctext[0] = xmlStr;
1351
    params.put("doctext", doctext);
1352
    
1353
    String username = Constants.SUBJECT_PUBLIC;
1354
    String[] groupnames = null;
1355
    if (session != null ) {
1356
    	username = session.getSubject().getValue();
1357
    	if (session.getSubjectInfo() != null) {
1358
    		List<Group> groupList = session.getSubjectInfo().getGroupList();
1359
    		if (groupList != null) {
1360
    			groupnames = new String[groupList.size()];
1361
    			for (int i = 0; i < groupList.size(); i++ ) {
1362
    				groupnames[i] = groupList.get(i).getGroupName();
1363
    			}
1364
    		}
1365
    	}
1366
    }
1367
    
1368
    // do the insert or update action
1369
    handler = new MetacatHandler(new Timer());
1370
    String result = handler.handleInsertOrUpdateAction(request.getRemoteAddr(), request.getHeader("User-Agent"), null, 
1371
                        null, params, username, groupnames, false, false, xmlBytes);
1372
    
1373
    if(result.indexOf("<error>") != -1) {
1374
    	String detailCode = "";
1375
    	if ( insertOrUpdate.equals("insert") ) {
1376
    		// make sure to remove the mapping so that subsequent attempts do not fail with IdentifierNotUnique
1377
    		im.removeMapping(pid.getValue(), localId);
1378
    		detailCode = "1190";
1379
    		
1380
    	} else if ( insertOrUpdate.equals("update") ) {
1381
    		detailCode = "1310";
1382
    		
1383
    	}
1384
        throw new ServiceFailure(detailCode, 
1385
          "Error inserting or updating document: " + result);
1386
    }
1387
    logMetacat.debug("Finsished inserting xml document with id " + localId);
1388
    
1389
    return localId;
1390
  }
1391
  
1392
  /**
1393
   * Insert a data document
1394
   * 
1395
   * @param object
1396
   * @param pid
1397
   * @param sessionData
1398
   * @throws ServiceFailure
1399
   * @returns localId of the data object inserted
1400
   */
1401
  public String insertDataObject(InputStream object, Identifier pid, 
1402
          Session session) throws ServiceFailure {
1403
      
1404
    String username = Constants.SUBJECT_PUBLIC;
1405
    String[] groupnames = null;
1406
    if (session != null ) {
1407
    	username = session.getSubject().getValue();
1408
    	if (session.getSubjectInfo() != null) {
1409
    		List<Group> groupList = session.getSubjectInfo().getGroupList();
1410
    		if (groupList != null) {
1411
    			groupnames = new String[groupList.size()];
1412
    			for (int i = 0; i < groupList.size(); i++ ) {
1413
    				groupnames[i] = groupList.get(i).getGroupName();
1414
    			}
1415
    		}
1416
    	}
1417
    }
1418
  
1419
    // generate pid/localId pair for object
1420
    logMetacat.debug("Generating a pid/localId mapping");
1421
    IdentifierManager im = IdentifierManager.getInstance();
1422
    String localId = im.generateLocalId(pid.getValue(), 1);
1423
  
1424
    // Save the data file to disk using "localId" as the name
1425
    String datafilepath = null;
1426
	try {
1427
		datafilepath = PropertyService.getProperty("application.datafilepath");
1428
	} catch (PropertyNotFoundException e) {
1429
		ServiceFailure sf = new ServiceFailure("1190", "Lookup data file path" + e.getMessage());
1430
		sf.initCause(e);
1431
		throw sf;
1432
	}
1433
    boolean locked = false;
1434
	try {
1435
		locked = DocumentImpl.getDataFileLockGrant(localId);
1436
	} catch (Exception e) {
1437
		ServiceFailure sf = new ServiceFailure("1190", "Could not lock file for writing:" + e.getMessage());
1438
		sf.initCause(e);
1439
		throw sf;
1440
	}
1441

    
1442
    logMetacat.debug("Case DATA: starting to write to disk.");
1443
	if (locked) {
1444

    
1445
          File dataDirectory = new File(datafilepath);
1446
          dataDirectory.mkdirs();
1447
  
1448
          File newFile = writeStreamToFile(dataDirectory, localId, object);
1449
  
1450
          // TODO: Check that the file size matches SystemMetadata
1451
          // long size = newFile.length();
1452
          // if (size == 0) {
1453
          //     throw new IOException("Uploaded file is 0 bytes!");
1454
          // }
1455
  
1456
          // Register the file in the database (which generates an exception
1457
          // if the localId is not acceptable or other untoward things happen
1458
          try {
1459
            logMetacat.debug("Registering document...");
1460
            DocumentImpl.registerDocument(localId, "BIN", localId,
1461
                    username, groupnames);
1462
            logMetacat.debug("Registration step completed.");
1463
            
1464
          } catch (SQLException e) {
1465
            //newFile.delete();
1466
            logMetacat.debug("SQLE: " + e.getMessage());
1467
            e.printStackTrace(System.out);
1468
            throw new ServiceFailure("1190", "Registration failed: " + 
1469
            		e.getMessage());
1470
            
1471
          } catch (AccessionNumberException e) {
1472
            //newFile.delete();
1473
            logMetacat.debug("ANE: " + e.getMessage());
1474
            e.printStackTrace(System.out);
1475
            throw new ServiceFailure("1190", "Registration failed: " + 
1476
            	e.getMessage());
1477
            
1478
          } catch (Exception e) {
1479
            //newFile.delete();
1480
            logMetacat.debug("Exception: " + e.getMessage());
1481
            e.printStackTrace(System.out);
1482
            throw new ServiceFailure("1190", "Registration failed: " + 
1483
            	e.getMessage());
1484
          }
1485
  
1486
          logMetacat.debug("Logging the creation event.");
1487
          EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), username, localId, "create");
1488
  
1489
          // Schedule replication for this data file, the "insert" action is important here!
1490
          logMetacat.debug("Scheduling replication.");
1491
          ForceReplicationHandler frh = new ForceReplicationHandler(localId, "insert", false, null);
1492
      }
1493
      
1494
      return localId;
1495
    
1496
  }
1497

    
1498
  /**
1499
   * Insert a systemMetadata document and return its localId
1500
   */
1501
  public void insertSystemMetadata(SystemMetadata sysmeta) 
1502
      throws ServiceFailure {
1503
      
1504
  	  logMetacat.debug("Starting to insert SystemMetadata...");
1505
      sysmeta.setDateSysMetadataModified(Calendar.getInstance().getTime());
1506
      logMetacat.debug("Inserting new system metadata with modified date " + 
1507
          sysmeta.getDateSysMetadataModified());
1508
      
1509
      //insert the system metadata
1510
      try {
1511
        // note: the calling subclass handles the map hazelcast lock/unlock
1512
      	HazelcastService.getInstance().getSystemMetadataMap().put(sysmeta.getIdentifier(), sysmeta);
1513
      	// submit for indexing
1514
        MetacatSolrIndex.getInstance().submit(sysmeta.getIdentifier(), sysmeta, null, true);
1515
      } catch (Exception e) {
1516
          throw new ServiceFailure("1190", e.getMessage());
1517
          
1518
	    }  
1519
  }
1520
  
1521
  /**
1522
   * Retrieve the list of objects present on the MN that match the calling parameters
1523
   * 
1524
   * @param session - the Session object containing the credentials for the Subject
1525
   * @param startTime - Specifies the beginning of the time range from which 
1526
   *                    to return object (>=)
1527
   * @param endTime - Specifies the beginning of the time range from which 
1528
   *                  to return object (>=)
1529
   * @param objectFormat - Restrict results to the specified object format
1530
   * @param replicaStatus - Indicates if replicated objects should be returned in the list
1531
   * @param start - The zero-based index of the first value, relative to the 
1532
   *                first record of the resultset that matches the parameters.
1533
   * @param count - The maximum number of entries that should be returned in 
1534
   *                the response. The Member Node may return less entries 
1535
   *                than specified in this value.
1536
   * 
1537
   * @return objectList - the list of objects matching the criteria
1538
   * 
1539
   * @throws InvalidToken
1540
   * @throws ServiceFailure
1541
   * @throws NotAuthorized
1542
   * @throws InvalidRequest
1543
   * @throws NotImplemented
1544
   */
1545
  public ObjectList listObjects(Session session, Date startTime, Date endTime, ObjectFormatIdentifier objectFormatId, Identifier identifier, NodeReference nodeId, Integer start,
1546
          Integer count) throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken {
1547

    
1548
      ObjectList objectList = null;
1549

    
1550
      try {
1551
          // safeguard against large requests
1552
          if (count == null || count > MAXIMUM_DB_RECORD_COUNT) {
1553
              count = MAXIMUM_DB_RECORD_COUNT;
1554
          }
1555
          boolean isSid = false;
1556
          if(identifier != null) {
1557
              isSid = IdentifierManager.getInstance().systemMetadataSIDExists(identifier);
1558
          }
1559
          objectList = IdentifierManager.getInstance().querySystemMetadata(startTime, endTime, objectFormatId, nodeId, start, count, identifier, isSid);
1560
      } catch (Exception e) {
1561
          throw new ServiceFailure("1580", "Error querying system metadata: " + e.getMessage());
1562
      }
1563

    
1564
      return objectList;
1565
  }
1566

    
1567
  /**
1568
   * Update a systemMetadata document
1569
   * 
1570
   * @param sysMeta - the system metadata object in the system to update
1571
   */
1572
    protected void updateSystemMetadata(SystemMetadata sysMeta)
1573
        throws ServiceFailure {
1574

    
1575
        logMetacat.debug("D1NodeService.updateSystemMetadata() called.");
1576
        sysMeta.setDateSysMetadataModified(new Date());
1577
        try {
1578
            HazelcastService.getInstance().getSystemMetadataMap().lock(sysMeta.getIdentifier());
1579
            HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);
1580
            // submit for indexing
1581
            MetacatSolrIndex.getInstance().submit(sysMeta.getIdentifier(), sysMeta, null, true);
1582
        } catch (Exception e) {
1583
            throw new ServiceFailure("4862", e.getMessage());
1584

    
1585
        } finally {
1586
            HazelcastService.getInstance().getSystemMetadataMap().unlock(sysMeta.getIdentifier());
1587

    
1588
        }
1589

    
1590
    }
1591
    
1592
    /**
1593
     * Update the system metadata of the specified pid
1594
     * @param session - the identity of the client which calls the method
1595
     * @param pid - the identifier of the object which will be updated
1596
     * @param sysmeta - the new system metadata  
1597
     * @return
1598
     * @throws NotImplemented
1599
     * @throws NotAuthorized
1600
     * @throws ServiceFailure
1601
     * @throws InvalidRequest
1602
     * @throws InvalidSystemMetadata
1603
     * @throws InvalidToken
1604
     */
1605
	public boolean updateSystemMetadata(Session session, Identifier pid,
1606
			SystemMetadata sysmeta) throws NotImplemented, NotAuthorized,
1607
			ServiceFailure, InvalidRequest, InvalidSystemMetadata, InvalidToken {
1608
		
1609
		// The lock to be used for this identifier
1610
      Lock lock = null;
1611
     
1612
      // verify that guid == SystemMetadata.getIdentifier()
1613
      logMetacat.debug("Comparing guid|sysmeta_guid: " + pid.getValue() + 
1614
          "|" + sysmeta.getIdentifier().getValue());
1615
      
1616
      if (!pid.getValue().equals(sysmeta.getIdentifier().getValue())) {
1617
          throw new InvalidRequest("4863", 
1618
              "The identifier in method call (" + pid.getValue() + 
1619
              ") does not match identifier in system metadata (" +
1620
              sysmeta.getIdentifier().getValue() + ").");
1621
      }
1622
      //compare serial version.
1623
      
1624
      //check the sid
1625
      SystemMetadata currentSysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
1626
      logMetacat.debug("The current dateUploaded is ============"+currentSysmeta.getDateUploaded());
1627
      logMetacat.debug("the dateUploaded in the new system metadata is "+sysmeta.getDateUploaded());
1628
      logMetacat.debug("The current dateUploaded is (by time) ============"+currentSysmeta.getDateUploaded().getTime());
1629
      logMetacat.debug("the dateUploaded in the new system metadata is (by time) "+sysmeta.getDateUploaded().getTime());
1630
      if(currentSysmeta == null ) {
1631
          //do we need throw an exception?
1632
          
1633
      } else {
1634
          
1635
          BigInteger newVersion = sysmeta.getSerialVersion();
1636
          if(newVersion == null) {
1637
              throw new InvalidSystemMetadata("4956", "The serial version can't be null in the new system metadata");
1638
          }
1639
          BigInteger currentVersion = currentSysmeta.getSerialVersion();
1640
          if(currentVersion != null && newVersion.compareTo(currentVersion) <= 0) {
1641
              throw new InvalidSystemMetadata("4956", "The serial version in the new system metadata is "+newVersion.toString()+
1642
                      " which is less than or equals the previous version "+currentVersion.toString()+". This is illegal in the updateSystemMetadata method.");
1643
          }
1644
          Identifier currentSid = currentSysmeta.getSeriesId();
1645
          if(currentSid != null) {
1646
              //new sid must match the current sid
1647
              Identifier newSid = sysmeta.getSeriesId();
1648
              if (!isValidIdentifier(newSid)) {
1649
                  throw new InvalidSystemMetadata("4956", "The series id in the system metadata is invalid in the request.");
1650
              } else {
1651
                  if(!newSid.getValue().equals(currentSid.getValue())) {
1652
                      throw new InvalidSystemMetadata("4956", "The series id "+newSid.getValue() +" in the system metadata doesn't match the current sid "+currentSid.getValue());
1653
                  }
1654
              }
1655
          } else {
1656
              //current system metadata doesn't have a sid. So we can have those scenarios
1657
              //1. The new sid may be null as well
1658
              //2. If the new sid does exist, it may be an identifier which hasn't bee used.
1659
              //3. If the new sid does exist, it may be an sid which equals the SID it obsoletes
1660
              //4. If the new sid does exist, it may be an sid which equauls the SID it was obsoleted by
1661
              Identifier newSid = sysmeta.getSeriesId();
1662
              if(newSid != null) {
1663
                  //It matches the rules of the checkSidInModifyingSystemMetadata
1664
                  checkSidInModifyingSystemMetadata(sysmeta, "4956", "4868");
1665
              }
1666
          }
1667
          checkModifiedImmutableFields(currentSysmeta, sysmeta);
1668
          checkOneTimeSettableSysmMetaFields(currentSysmeta, sysmeta);
1669
          if(currentSysmeta.getObsoletes() == null && sysmeta.getObsoletes() != null) {
1670
              //we are setting a value to the obsoletes field, so we should make sure if there is not object obsoletes the value
1671
              String obsoletes = existsInObsoletes(sysmeta.getObsoletes());
1672
              if( obsoletes != null) {
1673
                  throw new InvalidSystemMetadata("4956", "There is an object with id "+obsoletes +
1674
                          " already obsoletes the pid "+sysmeta.getObsoletes().getValue() +". You can't set the object "+pid.getValue()+" to obsolete the pid "+sysmeta.getObsoletes().getValue()+" again.");
1675
              }
1676
          }
1677
          
1678
          if(currentSysmeta.getObsoletedBy() == null && sysmeta.getObsoletedBy() != null) {
1679
              //we are setting a value to the obsoletedBy field, so we should make sure that the no another object obsoletes the pid we are updating. 
1680
              String obsoletedBy = existsInObsoletedBy(sysmeta.getObsoletedBy());
1681
              if( obsoletedBy != null) {
1682
                  throw new InvalidSystemMetadata("4956", "There is an object with id "+obsoletedBy +
1683
                          " already is obsoleted by the pid "+sysmeta.getObsoletedBy().getValue() +". You can't set the pid "+pid.getValue()+" to be obsoleted by the pid "+sysmeta.getObsoletedBy().getValue()+" again.");
1684
              }
1685
          }
1686
      }
1687
      
1688
      // do the actual update
1689
      this.updateSystemMetadata(sysmeta);
1690
      
1691
      try {
1692
    	  String localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1693
    	  EventLog.getInstance().log(request.getRemoteAddr(), 
1694
    	          request.getHeader("User-Agent"), session.getSubject().getValue(), 
1695
    	          localId, "updateSystemMetadata");
1696
      } catch (McdbDocNotFoundException e) {
1697
    	  // do nothing, no localId to log with
1698
    	  logMetacat.warn("Could not log 'updateSystemMetadata' event because no localId was found for pid: " + pid.getValue());
1699
      } catch (SQLException e) {
1700
          logMetacat.warn("Could not log 'updateSystemMetadata' event because the localId couldn't be identified for the pid: " + pid.getValue());
1701
      }
1702
      return true;
1703
	}
1704
	
1705
	
1706
	/*
1707
	 * Check if the newMeta modifies an immutable field. 
1708
	 */
1709
	private void checkModifiedImmutableFields(SystemMetadata orgMeta, SystemMetadata newMeta) throws InvalidRequest, InvalidSystemMetadata{
1710
	    if(orgMeta != null && newMeta != null) {
1711
	        if(newMeta.getIdentifier() == null) {
1712
	            throw new InvalidSystemMetadata("4956", "The new version of the system metadata is invalid since the identifier is null");
1713
	        }
1714
	        if(!orgMeta.getIdentifier().equals(newMeta.getIdentifier())) {
1715
	            throw new InvalidRequest("4869","The request is trying to modify an immutable field in the SystemMeta: the new system meta's identifier "+newMeta.getIdentifier().getValue()+" is "+
1716
	                  "different to the orginal one "+orgMeta.getIdentifier().getValue());
1717
	        }
1718
	        if(newMeta.getSize() == null) {
1719
	            throw new InvalidSystemMetadata("4956", "The new version of the system metadata is invalid since the size is null");
1720
	        }
1721
	        if(!orgMeta.getSize().equals(newMeta.getSize())) {
1722
	            throw new InvalidRequest("4869", "The request is trying to modify an immutable field in the SystemMeta: the new system meta's size "+newMeta.getSize().longValue()+" is "+
1723
	                      "different to the orginal one "+orgMeta.getSize().longValue());
1724
	        }
1725
	        if(newMeta.getChecksum()!= null && orgMeta.getChecksum() != null && !orgMeta.getChecksum().getValue().equals(newMeta.getChecksum().getValue())) {
1726
	            logMetacat.error("The request is trying to modify an immutable field in the SystemMeta: the new system meta's checksum "+newMeta.getChecksum().getValue()+" is "+
1727
                        "different to the orginal one "+orgMeta.getChecksum().getValue());
1728
	            throw new InvalidRequest("4869", "The request is trying to modify an immutable field in the SystemMeta: the new system meta's checksum "+newMeta.getChecksum().getValue()+" is "+
1729
                        "different to the orginal one "+orgMeta.getChecksum().getValue());
1730
	        }
1731
	        if(orgMeta.getSubmitter() != null && newMeta.getSubmitter() != null && !orgMeta.getSubmitter().equals(newMeta.getSubmitter())) {
1732
	            throw new InvalidRequest("4869", "The request is trying to modify an immutable field in the SystemMeta: the new system meta's submitter "+newMeta.getSubmitter().getValue()+" is "+
1733
                        "different to the orginal one "+orgMeta.getSubmitter().getValue());
1734
	        }
1735
	        
1736
	        if(orgMeta.getDateUploaded() != null && newMeta.getDateUploaded() != null && orgMeta.getDateUploaded().getTime() != newMeta.getDateUploaded().getTime()) {
1737
	            throw new InvalidRequest("4869", "The request is trying to modify an immutable field in the SystemMeta: the new system meta's date of uploaded "+newMeta.getDateUploaded()+" is "+
1738
                        "different to the orginal one "+orgMeta.getDateUploaded());
1739
	        }
1740
	        
1741
	        if(orgMeta.getOriginMemberNode() != null && newMeta.getOriginMemberNode() != null && !orgMeta.getOriginMemberNode().equals(newMeta.getOriginMemberNode())) {
1742
	            throw new InvalidRequest("4869", "The request is trying to modify an immutable field in the SystemMeta: the new system meta's orginal member node  "+newMeta.getOriginMemberNode().getValue()+" is "+
1743
                        "different to the orginal one "+orgMeta.getOriginMemberNode().getValue());
1744
	        }
1745
	        
1746
	        if(orgMeta.getSeriesId() != null && newMeta.getSeriesId() != null && !orgMeta.getSeriesId().equals(newMeta.getSeriesId())) {
1747
                throw new InvalidRequest("4869", "The request is trying to modify an immutable field in the SystemMeta: the new system meta's series id  "+newMeta.getSeriesId().getValue()+" is "+
1748
                        "different to the orginal one "+orgMeta.getSeriesId().getValue());
1749
            }
1750
	        
1751
	    }
1752
	}
1753
	
1754
	/*
1755
	 * Some fields in the system metadata, such as obsoletes or obsoletedBy can be set only once. 
1756
	 * After set, they are not allowed to be changed.
1757
	 */
1758
	private void checkOneTimeSettableSysmMetaFields(SystemMetadata orgMeta, SystemMetadata newMeta) throws InvalidRequest {
1759
	    if(orgMeta.getObsoletedBy() != null ) {
1760
	        if(newMeta.getObsoletedBy() == null || !orgMeta.getObsoletedBy().equals(newMeta.getObsoletedBy())) {
1761
	            throw new InvalidRequest("4869", "The request is trying to reset the obsoletedBy field in the system metadata of the object "
1762
	                    + orgMeta.getIdentifier().getValue() +". This is illegal since the obsoletedBy filed is set, you can't change it again.");
1763
	        }
1764
        }
1765
	    if(orgMeta.getObsoletes() != null) {
1766
	        if(newMeta.getObsoletes() == null || !orgMeta.getObsoletes().equals(newMeta.getObsoletes())) {
1767
	            throw new InvalidRequest("4869", "The request is trying to reset the obsoletes field in the system metadata of the object"+
1768
	               orgMeta.getIdentifier().getValue()+". This is illegal since the obsoletes filed is set, you can't change it again.");
1769
	        }
1770
	    }
1771
	}
1772
  
1773
  /**
1774
   * Given a Permission, returns a list of all permissions that it encompasses
1775
   * Permissions are hierarchical so that WRITE also allows READ.
1776
   * @param permission
1777
   * @return list of included Permissions for the given permission
1778
   */
1779
  protected List<Permission> expandPermissions(Permission permission) {
1780
	  	List<Permission> expandedPermissions = new ArrayList<Permission>();
1781
	    if (permission.equals(Permission.READ)) {
1782
	    	expandedPermissions.add(Permission.READ);
1783
	    }
1784
	    if (permission.equals(Permission.WRITE)) {
1785
	    	expandedPermissions.add(Permission.READ);
1786
	    	expandedPermissions.add(Permission.WRITE);
1787
	    }
1788
	    if (permission.equals(Permission.CHANGE_PERMISSION)) {
1789
	    	expandedPermissions.add(Permission.READ);
1790
	    	expandedPermissions.add(Permission.WRITE);
1791
	    	expandedPermissions.add(Permission.CHANGE_PERMISSION);
1792
	    }
1793
	    return expandedPermissions;
1794
  }
1795

    
1796
  /*
1797
   * Write a stream to a file
1798
   * 
1799
   * @param dir - the directory to write to
1800
   * @param fileName - the file name to write to
1801
   * @param data - the object bytes as an input stream
1802
   * 
1803
   * @return newFile - the new file created
1804
   * 
1805
   * @throws ServiceFailure
1806
   */
1807
  private File writeStreamToFile(File dir, String fileName, InputStream data) 
1808
    throws ServiceFailure {
1809
    
1810
    File newFile = new File(dir, fileName);
1811
    logMetacat.debug("Filename for write is: " + newFile.getAbsolutePath());
1812

    
1813
    try {
1814
        if (newFile.createNewFile()) {
1815
          // write data stream to desired file
1816
          OutputStream os = new FileOutputStream(newFile);
1817
          long length = IOUtils.copyLarge(data, os);
1818
          os.flush();
1819
          os.close();
1820
        } else {
1821
          logMetacat.debug("File creation failed, or file already exists.");
1822
          throw new ServiceFailure("1190", "File already exists: " + fileName);
1823
        }
1824
    } catch (FileNotFoundException e) {
1825
      logMetacat.debug("FNF: " + e.getMessage());
1826
      throw new ServiceFailure("1190", "File not found: " + fileName + " " 
1827
                + e.getMessage());
1828
    } catch (IOException e) {
1829
      logMetacat.debug("IOE: " + e.getMessage());
1830
      throw new ServiceFailure("1190", "File was not written: " + fileName 
1831
                + " " + e.getMessage());
1832
    }
1833

    
1834
    return newFile;
1835
  }
1836

    
1837
  /*
1838
   * Returns a list of nodes that have been registered with the DataONE infrastructure
1839
   * that match the given session subject
1840
   * @return nodes - List of nodes from the registry with a matching session subject
1841
   * 
1842
   * @throws ServiceFailure
1843
   * @throws NotImplemented
1844
   */
1845
  protected List<Node> listNodesBySubject(Subject subject) 
1846
      throws ServiceFailure, NotImplemented {
1847
      List<Node> nodeList = new ArrayList<Node>();
1848
      
1849
      CNode cn = D1Client.getCN();
1850
      List<Node> nodes = cn.listNodes().getNodeList();
1851
      
1852
      // find the node in the node list
1853
      for ( Node node : nodes ) {
1854
          
1855
          List<Subject> nodeSubjects = node.getSubjectList();
1856
          if (nodeSubjects != null) {    
1857
	          // check if the session subject is in the node subject list
1858
	          for (Subject nodeSubject : nodeSubjects) {
1859
	              if ( nodeSubject.equals(subject) ) { // subject of session == node subject
1860
	                  nodeList.add(node);  
1861
	              }                              
1862
	          }
1863
          }
1864
      }
1865
      
1866
      return nodeList;
1867
      
1868
  }
1869

    
1870
  /**
1871
   * Archives an object, where the object is either a 
1872
   * data object or a science metadata object.
1873
   * 
1874
   * @param session - the Session object containing the credentials for the Subject
1875
   * @param pid - The object identifier to be archived
1876
   * 
1877
   * @return pid - the identifier of the object used for the archiving
1878
   * 
1879
   * @throws InvalidToken
1880
   * @throws ServiceFailure
1881
   * @throws NotAuthorized
1882
   * @throws NotFound
1883
   * @throws NotImplemented
1884
   * @throws InvalidRequest
1885
   */
1886
  public Identifier archive(Session session, Identifier pid) 
1887
      throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
1888

    
1889
      String localId = null;
1890
      boolean allowed = false;
1891
      String username = Constants.SUBJECT_PUBLIC;
1892
      String[] groupnames = null;
1893
      if (session == null) {
1894
      	throw new InvalidToken("1330", "No session has been provided");
1895
      } else {
1896
          username = session.getSubject().getValue();
1897
          if (session.getSubjectInfo() != null) {
1898
              List<Group> groupList = session.getSubjectInfo().getGroupList();
1899
              if (groupList != null) {
1900
                  groupnames = new String[groupList.size()];
1901
                  for (int i = 0; i < groupList.size(); i++) {
1902
                      groupnames[i] = groupList.get(i).getGroupName();
1903
                  }
1904
              }
1905
          }
1906
      }
1907

    
1908
      // do we have a valid pid?
1909
      if (pid == null || pid.getValue().trim().equals("")) {
1910
          throw new ServiceFailure("1350", "The provided identifier was invalid.");
1911
      }
1912
      
1913
      String serviceFailureCode = "1350";
1914
      Identifier sid = getPIDForSID(pid, serviceFailureCode);
1915
      if(sid != null) {
1916
          pid = sid;
1917
      }
1918

    
1919
      // check for the existing identifier
1920
      try {
1921
          localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1922
      } catch (McdbDocNotFoundException e) {
1923
          throw new NotFound("1340", "The object with the provided " + "identifier was not found.");
1924
      } catch (SQLException e) {
1925
          throw new ServiceFailure("1350", "The object with the provided identifier "+pid.getValue()+" couldn't be identified since "+e.getMessage());
1926
      }
1927

    
1928
      // does the subject have archive (a D1 CHANGE_PERMISSION level) privileges on the pid?
1929
      try {
1930
			allowed = isAuthorized(session, pid, Permission.CHANGE_PERMISSION);
1931
		} catch (InvalidRequest e) {
1932
          throw new ServiceFailure("1350", e.getDescription());
1933
		}
1934
          
1935

    
1936
      if (allowed) {
1937
          try {
1938
              // archive the document
1939
              DocumentImpl.delete(localId, null, null, null, false);
1940
              EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), username, localId, Event.DELETE.xmlValue());
1941

    
1942
              // archive it
1943
              SystemMetadata sysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
1944
              sysMeta.setArchived(true);
1945
              sysMeta.setDateSysMetadataModified(Calendar.getInstance().getTime());
1946
              HazelcastService.getInstance().getSystemMetadataMap().put(pid, sysMeta);
1947
              // submit for indexing
1948
              // DocumentImpl call above should do this.
1949
              // see: https://projects.ecoinformatics.org/ecoinfo/issues/6030
1950
              //HazelcastService.getInstance().getIndexQueue().add(sysMeta);
1951
              
1952
          } catch (McdbDocNotFoundException e) {
1953
              throw new NotFound("1340", "The provided identifier was invalid.");
1954

    
1955
          } catch (SQLException e) {
1956
              throw new ServiceFailure("1350", "There was a problem archiving the object." + "The error message was: " + e.getMessage());
1957

    
1958
          } catch (InsufficientKarmaException e) {
1959
              throw new NotAuthorized("1320", "The provided identity does not have " + "permission to archive this object.");
1960

    
1961
          } catch (Exception e) { // for some reason DocumentImpl throws a general Exception
1962
              throw new ServiceFailure("1350", "There was a problem archiving the object." + "The error message was: " + e.getMessage());
1963
          }
1964

    
1965
      } else {
1966
          throw new NotAuthorized("1320", "The provided identity does not have " + "permission to archive the object on the Node.");
1967
      }
1968

    
1969
      return pid;
1970
  }
1971
  
1972
  
1973
  /**
1974
   * A utility method for v1 api to check the specified identifier exists as a pid
1975
   * @param identifier  the specified identifier
1976
   * @param serviceFailureCode  the detail error code for the service failure exception
1977
   * @param noFoundCode  the detail error code for the not found exception
1978
   * @throws ServiceFailure
1979
   * @throws NotFound
1980
   */
1981
  public void checkV1SystemMetaPidExist(Identifier identifier, String serviceFailureCode, String serviceFailureMessage,  
1982
          String noFoundCode, String notFoundMessage) throws ServiceFailure, NotFound {
1983
      boolean exists = false;
1984
      try {
1985
          exists = IdentifierManager.getInstance().systemMetadataPIDExists(identifier);
1986
      } catch (SQLException e) {
1987
          throw new ServiceFailure(serviceFailureCode, serviceFailureMessage+" since "+e.getMessage());
1988
      }
1989
      if(!exists) {
1990
         //the v1 method only handles a pid. so it should throw a not-found exception.
1991
          // check if the pid was deleted.
1992
          try {
1993
              String localId = IdentifierManager.getInstance().getLocalId(identifier.getValue());
1994
              if(EventLog.getInstance().isDeleted(localId)) {
1995
                  notFoundMessage=notFoundMessage+" "+DELETEDMESSAGE;
1996
              } 
1997
            } catch (Exception e) {
1998
              logMetacat.info("Couldn't determine if the not-found identifier "+identifier.getValue()+" was deleted since "+e.getMessage());
1999
            }
2000
            throw new NotFound(noFoundCode, notFoundMessage);
2001
      }
2002
  }
2003
  
2004
  /**
2005
   * Utility method to get the PID for an SID. If the specified identifier is not an SID
2006
   * , null will be returned.
2007
   * @param sid  the specified sid
2008
   * @param serviceFailureCode  the detail error code for the service failure exception
2009
   * @return the pid for the sid. If the specified identifier is not an SID, null will be returned.
2010
   * @throws ServiceFailure
2011
   */
2012
  protected Identifier getPIDForSID(Identifier sid, String serviceFailureCode) throws ServiceFailure {
2013
      Identifier id = null;
2014
      String serviceFailureMessage = "The PID "+" couldn't be identified for the sid " + sid.getValue();
2015
      try {
2016
          //determine if the given pid is a sid or not.
2017
          if(IdentifierManager.getInstance().systemMetadataSIDExists(sid)) {
2018
              try {
2019
                  //set the header pid for the sid if the identifier is a sid.
2020
                  id = IdentifierManager.getInstance().getHeadPID(sid);
2021
              } catch (SQLException sqle) {
2022
                  throw new ServiceFailure(serviceFailureCode, serviceFailureMessage+" since "+sqle.getMessage());
2023
              }
2024
              
2025
          }
2026
      } catch (SQLException e) {
2027
          throw new ServiceFailure(serviceFailureCode, serviceFailureMessage + " since "+e.getMessage());
2028
      }
2029
      return id;
2030
  }
2031

    
2032
  /*
2033
   * Determine if the sid is legitimate in CN.create and CN.registerSystemMetadata methods. It also is used as a part of rules of the updateSystemMetadata method. Here are the rules:
2034
   * A. If the sysmeta doesn't have an SID, nothing needs to be checked for the SID.
2035
   * B. If the sysmeta does have an SID, it may be an identifier which doesn't exist in the system.
2036
   * C. If the sysmeta does have an SID and it exists as an SID in the system, those scenarios are acceptable:
2037
   *    i. The sysmeta has an obsoletes field, the SID has the same value as the SID of the system metadata of the obsoleting pid.
2038
   *    ii. The sysmeta has an obsoletedBy field, the SID has the same value as the SID of the system metadata of the obsoletedBy pid. 
2039
   */
2040
  protected boolean checkSidInModifyingSystemMetadata(SystemMetadata sysmeta, String invalidSystemMetadataCode, String serviceFailureCode) throws InvalidSystemMetadata, ServiceFailure{
2041
      boolean pass = false;
2042
      if(sysmeta == null) {
2043
          throw new InvalidSystemMetadata(invalidSystemMetadataCode, "The system metadata is null in the request.");
2044
      }
2045
      Identifier sid = sysmeta.getSeriesId();
2046
      if(sid != null) {
2047
          // the series id exists
2048
          if (!isValidIdentifier(sid)) {
2049
              throw new InvalidSystemMetadata(invalidSystemMetadataCode, "The series id in the system metadata is invalid in the request.");
2050
          }
2051
          Identifier pid = sysmeta.getIdentifier();
2052
          if (!isValidIdentifier(pid)) {
2053
              throw new InvalidSystemMetadata(invalidSystemMetadataCode, "The pid in the system metadata is invalid in the request.");
2054
          }
2055
          //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario )
2056
          if(sid.getValue().equals(pid.getValue())) {
2057
              throw new InvalidSystemMetadata(invalidSystemMetadataCode, "The series id "+sid.getValue()+" in the system metadata shouldn't have the same value of the pid.");
2058
          }
2059
          try {
2060
              if (IdentifierManager.getInstance().identifierExists(sid.getValue())) {
2061
                  //the sid exists in system
2062
                  if(sysmeta.getObsoletes() != null) {
2063
                      SystemMetadata obsoletesSysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(sysmeta.getObsoletes());
2064
                      if(obsoletesSysmeta != null) {
2065
                          Identifier obsoletesSid = obsoletesSysmeta.getSeriesId();
2066
                          if(obsoletesSid != null && obsoletesSid.getValue() != null && !obsoletesSid.getValue().trim().equals("")) {
2067
                              if(sid.getValue().equals(obsoletesSid.getValue())) {
2068
                                  pass = true;// the i of rule C
2069
                              }
2070
                          }
2071
                      } else {
2072
                           logMetacat.warn("D1NodeService.checkSidInModifyingSystemMetacat - Can't find the system metadata for the pid "+sysmeta.getObsoletes().getValue()+
2073
                                                                         " which is the value of the obsoletes. So we can't check if the sid " +sid.getValue()+" is legitimate ");
2074
                      }
2075
                  }
2076
                  if(!pass) {
2077
                      // the sid doesn't match the sid of the obsoleting identifier. So we check the obsoletedBy
2078
                      if(sysmeta.getObsoletedBy() != null) {
2079
                          SystemMetadata obsoletedBySysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(sysmeta.getObsoletedBy());
2080
                          if(obsoletedBySysmeta != null) {
2081
                              Identifier obsoletedBySid = obsoletedBySysmeta.getSeriesId();
2082
                              if(obsoletedBySid != null && obsoletedBySid.getValue() != null && !obsoletedBySid.getValue().trim().equals("")) {
2083
                                  if(sid.getValue().equals(obsoletedBySid.getValue())) {
2084
                                      pass = true;// the ii of the rule C
2085
                                  }
2086
                              }
2087
                          } else {
2088
                              logMetacat.warn("D1NodeService.checkSidInModifyingSystemMetacat - Can't find the system metadata for the pid "+sysmeta.getObsoletes().getValue() 
2089
                                                                            +" which is the value of the obsoletedBy. So we can't check if the sid "+sid.getValue()+" is legitimate.");
2090
                          }
2091
                      }
2092
                  }
2093
                  if(!pass) {
2094
                      throw new InvalidSystemMetadata(invalidSystemMetadataCode, "The series id "+sid.getValue()+
2095
                              " in the system metadata exists in the system. And it doesn't match either previous object's sid or the next object's sid.");
2096
                  }
2097
              } else {
2098
                  pass = true; //Rule B
2099
              }
2100
          } catch (SQLException e) {
2101
              throw new ServiceFailure(serviceFailureCode, "Can't determine if the sid in the system metadata is unique or not since "+e.getMessage());
2102
          }
2103
          
2104
      } else {
2105
          //no sid. Rule A.
2106
          pass = true;
2107
      }
2108
      return pass;
2109
      
2110
  }
2111
  
2112
  //@Override
2113
  public OptionList listViews(Session arg0) throws InvalidToken,
2114
          ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented {
2115
      OptionList views = new OptionList();
2116
      views.setKey("views");
2117
      views.setDescription("List of views for objects on the node");
2118
      Vector<String> skinNames = null;
2119
      try {
2120
          skinNames = SkinUtil.getSkinNames();
2121
      } catch (PropertyNotFoundException e) {
2122
          throw new ServiceFailure("2841", e.getMessage());
2123
      }
2124
      for (String skinName: skinNames) {
2125
          views.addOption(skinName);
2126
      }
2127
      return views;
2128
  }
2129
  
2130
  public OptionList listViews() throws InvalidToken,
2131
  ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented { 
2132
      return listViews(null);
2133
  }
2134

    
2135
  //@Override
2136
  public InputStream view(Session session, String format, Identifier pid)
2137
          throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest,
2138
          NotImplemented, NotFound {
2139
      InputStream resultInputStream = null;
2140
      
2141
      String serviceFailureCode = "2831";
2142
      Identifier sid = getPIDForSID(pid, serviceFailureCode);
2143
      if(sid != null) {
2144
          pid = sid;
2145
      }
2146
      
2147
      SystemMetadata sysMeta = this.getSystemMetadata(session, pid);
2148
      InputStream object = this.get(session, pid);
2149

    
2150
      try {
2151
          // can only transform metadata, really
2152
          ObjectFormat objectFormat = ObjectFormatCache.getInstance().getFormat(sysMeta.getFormatId());
2153
          if (objectFormat.getFormatType().equals("METADATA")) {
2154
              // transform
2155
              DBTransform transformer = new DBTransform();
2156
              String documentContent = IOUtils.toString(object, "UTF-8");
2157
              String sourceType = objectFormat.getFormatId().getValue();
2158
              String targetType = "-//W3C//HTML//EN";
2159
              ByteArrayOutputStream baos = new ByteArrayOutputStream();
2160
              Writer writer = new OutputStreamWriter(baos , "UTF-8");
2161
              // TODO: include more params?
2162
              Hashtable<String, String[]> params = new Hashtable<String, String[]>();
2163
              String localId = null;
2164
              try {
2165
                  localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
2166
              } catch (McdbDocNotFoundException e) {
2167
                  throw new NotFound("1020", e.getMessage());
2168
              }
2169
              params.put("qformat", new String[] {format});               
2170
              params.put("docid", new String[] {localId});
2171
              params.put("pid", new String[] {pid.getValue()});
2172
              transformer.transformXMLDocument(
2173
                      documentContent , 
2174
                      sourceType, 
2175
                      targetType , 
2176
                      format, 
2177
                      writer, 
2178
                      params, 
2179
                      null //sessionid
2180
                      );
2181
              
2182
              // finally, get the HTML back
2183
              resultInputStream = new ContentTypeByteArrayInputStream(baos.toByteArray());
2184
              ((ContentTypeByteArrayInputStream) resultInputStream).setContentType("text/html");
2185
  
2186
          } else {
2187
              // just return the raw bytes
2188
              resultInputStream = object;
2189
          }
2190
      } catch (IOException e) {
2191
          // report as service failure
2192
          ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2193
          sf.initCause(e);
2194
          throw sf;
2195
      } catch (PropertyNotFoundException e) {
2196
          // report as service failure
2197
          ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2198
          sf.initCause(e);
2199
          throw sf;
2200
      } catch (SQLException e) {
2201
          // report as service failure
2202
          ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2203
          sf.initCause(e);
2204
          throw sf;
2205
      } catch (ClassNotFoundException e) {
2206
          // report as service failure
2207
          ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2208
          sf.initCause(e);
2209
          throw sf;
2210
      }
2211
      
2212
      return resultInputStream;
2213
  } 
2214
  
2215
  /*
2216
   * Determine if the given identifier exists in the obsoletes field in the system metadata table.
2217
   * If the return value is not null, the given identifier exists in the given cloumn. The return value is 
2218
   * the guid of the first row.
2219
   */
2220
  protected String existsInObsoletes(Identifier id) throws InvalidRequest, ServiceFailure{
2221
      String guid = existsInFields("obsoletes", id);
2222
      return guid;
2223
  }
2224
  
2225
  /*
2226
   * Determine if the given identifier exists in the obsoletes field in the system metadata table.
2227
   * If the return value is not null, the given identifier exists in the given cloumn. The return value is 
2228
   * the guid of the first row.
2229
   */
2230
  protected String existsInObsoletedBy(Identifier id) throws InvalidRequest, ServiceFailure{
2231
      String guid = existsInFields("obsoleted_by", id);
2232
      return guid;
2233
  }
2234

    
2235
  /*
2236
   * Determine if the given identifier exists in the given column in the system metadata table. 
2237
   * If the return value is not null, the given identifier exists in the given cloumn. The return value is 
2238
   * the guid of the first row.
2239
   */
2240
  private String existsInFields(String column, Identifier id) throws InvalidRequest, ServiceFailure {
2241
      String guid = null;
2242
      if(id == null ) {
2243
          throw new InvalidRequest("4863", "The given identifier is null and we can't determine if the guid exists in the field "+column+" in the systemmetadata table");
2244
      }
2245
      String sql = "SELECT guid FROM systemmetadata WHERE "+column+" = ?";
2246
      int serialNumber = -1;
2247
      DBConnection dbConn = null;
2248
      PreparedStatement stmt = null;
2249
      ResultSet result = null;
2250
      try {
2251
          dbConn = 
2252
                  DBConnectionPool.getDBConnection("D1NodeService.existsInFields");
2253
          serialNumber = dbConn.getCheckOutSerialNumber();
2254
          stmt = dbConn.prepareStatement(sql);
2255
          stmt.setString(1, id.getValue());
2256
          result = stmt.executeQuery();
2257
          if(result.next()) {
2258
              guid = result.getString(1);
2259
          }
2260
          stmt.close();
2261
      } catch (SQLException e) {
2262
          e.printStackTrace();
2263
          throw new ServiceFailure("4862","We can't determine if the id "+id.getValue()+" exists in field "+column+" in the systemmetadata table since "+e.getMessage());
2264
      } finally {
2265
          // Return database connection to the pool
2266
          DBConnectionPool.returnDBConnection(dbConn, serialNumber);
2267
          if(stmt != null) {
2268
              try {
2269
                  stmt.close();
2270
              } catch (SQLException e) {
2271
                  logMetacat.warn("We can close the PreparedStatment in D1NodeService.existsInFields since "+e.getMessage());
2272
              }
2273
          }
2274
          
2275
      }
2276
      return guid;
2277
      
2278
  }
2279
}
(2-2/8)