Project

General

Profile

1 6179 cjones
/**
2
 *  '$RCSfile$'
3
 *  Copyright: 2000-2011 Regents of the University of California and the
4
 *              National Center for Ecological Analysis and Synthesis
5
 *
6
 *   '$Author:  $'
7
 *     '$Date:  $'
8
 *
9
 * This program is free software; you can redistribute it and/or modify
10
 * it under the terms of the GNU General Public License as published by
11
 * the Free Software Foundation; either version 2 of the License, or
12
 * (at your option) any later version.
13
 *
14
 * This program is distributed in the hope that it will be useful,
15
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 * GNU General Public License for more details.
18
 *
19
 * You should have received a copy of the GNU General Public License
20
 * along with this program; if not, write to the Free Software
21
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22
 */
23
24
package edu.ucsb.nceas.metacat.dataone;
25
26 7417 leinfelder
import java.io.ByteArrayInputStream;
27 7860 leinfelder
import java.io.ByteArrayOutputStream;
28 7850 leinfelder
import java.io.File;
29 8800 leinfelder
import java.io.FileInputStream;
30 7850 leinfelder
import java.io.FileOutputStream;
31 6228 cjones
import java.io.IOException;
32 6179 cjones
import java.io.InputStream;
33 8190 leinfelder
import java.io.InputStreamReader;
34 7860 leinfelder
import java.io.OutputStreamWriter;
35 9387 walker
import java.io.StringReader;
36 7849 leinfelder
import java.io.UnsupportedEncodingException;
37 7860 leinfelder
import java.io.Writer;
38 7021 leinfelder
import java.math.BigInteger;
39 7849 leinfelder
import java.net.URISyntaxException;
40 9352 walker
import java.nio.charset.Charset;
41 6228 cjones
import java.security.NoSuchAlgorithmException;
42 7860 leinfelder
import java.sql.SQLException;
43 7417 leinfelder
import java.util.ArrayList;
44 6525 leinfelder
import java.util.Calendar;
45 6179 cjones
import java.util.Date;
46 8437 walker
import java.util.HashMap;
47 7680 tao
import java.util.HashSet;
48 7860 leinfelder
import java.util.Hashtable;
49 6250 cjones
import java.util.List;
50 7849 leinfelder
import java.util.Map;
51 7417 leinfelder
import java.util.Set;
52 6389 leinfelder
import java.util.Timer;
53 7489 leinfelder
import java.util.UUID;
54 7418 leinfelder
import java.util.Vector;
55 10104 tao
import java.util.concurrent.ExecutorService;
56
import java.util.concurrent.Executors;
57 6179 cjones
58 6542 leinfelder
import javax.servlet.http.HttpServletRequest;
59 9387 walker
import javax.xml.transform.Result;
60
import javax.xml.transform.Source;
61
import javax.xml.transform.TransformerException;
62
import javax.xml.transform.TransformerFactory;
63
import javax.xml.transform.dom.DOMSource;
64
import javax.xml.transform.stream.StreamResult;
65 6542 leinfelder
66 6258 cjones
import org.apache.commons.io.IOUtils;
67 9997 tao
import org.apache.commons.lang.NotImplementedException;
68 6179 cjones
import org.apache.log4j.Logger;
69 8810 leinfelder
import org.dataone.client.v2.CNode;
70
import org.dataone.client.v2.itk.D1Client;
71
import org.dataone.client.v2.MNode;
72
import org.dataone.client.v2.formats.ObjectFormatCache;
73 6552 leinfelder
import org.dataone.client.auth.CertificateManager;
74 8810 leinfelder
import org.dataone.client.v2.formats.ObjectFormatInfo;
75 6552 leinfelder
import org.dataone.configuration.Settings;
76 7849 leinfelder
import org.dataone.ore.ResourceMapFactory;
77 6795 cjones
import org.dataone.service.exceptions.BaseException;
78 6179 cjones
import org.dataone.service.exceptions.IdentifierNotUnique;
79
import org.dataone.service.exceptions.InsufficientResources;
80
import org.dataone.service.exceptions.InvalidRequest;
81
import org.dataone.service.exceptions.InvalidSystemMetadata;
82
import org.dataone.service.exceptions.InvalidToken;
83
import org.dataone.service.exceptions.NotAuthorized;
84
import org.dataone.service.exceptions.NotFound;
85
import org.dataone.service.exceptions.NotImplemented;
86
import org.dataone.service.exceptions.ServiceFailure;
87 6185 leinfelder
import org.dataone.service.exceptions.SynchronizationFailed;
88 6179 cjones
import org.dataone.service.exceptions.UnsupportedType;
89 8810 leinfelder
import org.dataone.service.mn.tier1.v2.MNCore;
90
import org.dataone.service.mn.tier1.v2.MNRead;
91
import org.dataone.service.mn.tier2.v2.MNAuthorization;
92
import org.dataone.service.mn.tier3.v2.MNStorage;
93
import org.dataone.service.mn.tier4.v2.MNReplication;
94
import org.dataone.service.mn.v2.MNPackage;
95
import org.dataone.service.mn.v2.MNQuery;
96
import org.dataone.service.mn.v2.MNView;
97 8591 leinfelder
import org.dataone.service.types.v1.AccessRule;
98 6366 leinfelder
import org.dataone.service.types.v1.Checksum;
99 7144 leinfelder
import org.dataone.service.types.v1.DescribeResponse;
100 6366 leinfelder
import org.dataone.service.types.v1.Event;
101
import org.dataone.service.types.v1.Identifier;
102 8810 leinfelder
import org.dataone.service.types.v2.Log;
103
import org.dataone.service.types.v2.LogEntry;
104
import org.dataone.service.types.v2.OptionList;
105 6366 leinfelder
import org.dataone.service.types.v1.MonitorInfo;
106
import org.dataone.service.types.v1.MonitorList;
107 8810 leinfelder
import org.dataone.service.types.v2.Node;
108
import org.dataone.service.types.v2.NodeList;
109 6366 leinfelder
import org.dataone.service.types.v1.NodeReference;
110
import org.dataone.service.types.v1.NodeState;
111
import org.dataone.service.types.v1.NodeType;
112 8810 leinfelder
import org.dataone.service.types.v2.ObjectFormat;
113 9353 tao
import org.dataone.service.types.v1.Group;
114 6366 leinfelder
import org.dataone.service.types.v1.ObjectFormatIdentifier;
115
import org.dataone.service.types.v1.ObjectList;
116
import org.dataone.service.types.v1.Permission;
117
import org.dataone.service.types.v1.Ping;
118 9340 tao
import org.dataone.service.types.v1.Replica;
119 6528 cjones
import org.dataone.service.types.v1.ReplicationStatus;
120 6366 leinfelder
import org.dataone.service.types.v1.Schedule;
121
import org.dataone.service.types.v1.Service;
122
import org.dataone.service.types.v1.Services;
123
import org.dataone.service.types.v1.Session;
124
import org.dataone.service.types.v1.Subject;
125
import org.dataone.service.types.v1.Synchronization;
126 8810 leinfelder
import org.dataone.service.types.v2.SystemMetadata;
127 7417 leinfelder
import org.dataone.service.types.v1.util.AuthUtils;
128 6366 leinfelder
import org.dataone.service.types.v1.util.ChecksumUtil;
129 7417 leinfelder
import org.dataone.service.types.v1_1.QueryEngineDescription;
130
import org.dataone.service.types.v1_1.QueryEngineList;
131 7418 leinfelder
import org.dataone.service.types.v1_1.QueryField;
132 6476 jones
import org.dataone.service.util.Constants;
133 8594 leinfelder
import org.dataone.service.util.TypeMarshaller;
134 7849 leinfelder
import org.dspace.foresite.OREException;
135
import org.dspace.foresite.OREParserException;
136
import org.dspace.foresite.ORESerialiserException;
137
import org.dspace.foresite.ResourceMap;
138 8437 walker
import org.ecoinformatics.datamanager.parser.DataPackage;
139
import org.ecoinformatics.datamanager.parser.Entity;
140
import org.ecoinformatics.datamanager.parser.generic.DataPackageParserInterface;
141
import org.ecoinformatics.datamanager.parser.generic.Eml200DataPackageParser;
142 9387 walker
import org.w3c.dom.Document;
143 6179 cjones
144 7448 leinfelder
import edu.ucsb.nceas.ezid.EZIDException;
145 7417 leinfelder
import edu.ucsb.nceas.metacat.DBQuery;
146 7860 leinfelder
import edu.ucsb.nceas.metacat.DBTransform;
147 6234 cjones
import edu.ucsb.nceas.metacat.EventLog;
148 6230 cjones
import edu.ucsb.nceas.metacat.IdentifierManager;
149 6234 cjones
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
150 7417 leinfelder
import edu.ucsb.nceas.metacat.MetaCatServlet;
151 6389 leinfelder
import edu.ucsb.nceas.metacat.MetacatHandler;
152 9478 tao
import edu.ucsb.nceas.metacat.ReadOnlyChecker;
153 7772 tao
import edu.ucsb.nceas.metacat.common.query.EnabledQueryEngines;
154 7757 leinfelder
import edu.ucsb.nceas.metacat.common.query.stream.ContentTypeByteArrayInputStream;
155 6648 leinfelder
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;
156 7662 tao
import edu.ucsb.nceas.metacat.index.MetacatSolrEngineDescriptionHandler;
157 7620 tao
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;
158 10172 leinfelder
import edu.ucsb.nceas.metacat.mdq.MDQClient;
159 6340 cjones
import edu.ucsb.nceas.metacat.properties.PropertyService;
160 7418 leinfelder
import edu.ucsb.nceas.metacat.shared.MetacatUtilException;
161 8026 leinfelder
import edu.ucsb.nceas.metacat.util.DeleteOnCloseFileInputStream;
162 7441 leinfelder
import edu.ucsb.nceas.metacat.util.DocumentUtil;
163 8810 leinfelder
import edu.ucsb.nceas.metacat.util.SkinUtil;
164 6542 leinfelder
import edu.ucsb.nceas.metacat.util.SystemUtil;
165 6340 cjones
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
166 8190 leinfelder
import edu.ucsb.nceas.utilities.XMLUtilities;
167 8800 leinfelder
import edu.ucsb.nceas.utilities.export.HtmlToPdf;
168 7850 leinfelder
import gov.loc.repository.bagit.Bag;
169
import gov.loc.repository.bagit.BagFactory;
170
import gov.loc.repository.bagit.writer.impl.ZipWriter;
171 6230 cjones
172 6179 cjones
/**
173
 * Represents Metacat's implementation of the DataONE Member Node
174
 * service API. Methods implement the various MN* interfaces, and methods common
175
 * to both Member Node and Coordinating Node interfaces are found in the
176
 * D1NodeService base class.
177 6288 cjones
 *
178
 * Implements:
179
 * MNCore.ping()
180
 * MNCore.getLogRecords()
181
 * MNCore.getObjectStatistics()
182
 * MNCore.getOperationStatistics()
183
 * MNCore.getStatus()
184
 * MNCore.getCapabilities()
185
 * MNRead.get()
186
 * MNRead.getSystemMetadata()
187
 * MNRead.describe()
188
 * MNRead.getChecksum()
189
 * MNRead.listObjects()
190
 * MNRead.synchronizationFailed()
191
 * MNAuthorization.isAuthorized()
192
 * MNAuthorization.setAccessPolicy()
193
 * MNStorage.create()
194
 * MNStorage.update()
195
 * MNStorage.delete()
196 9177 tao
 * MNStorage.updateSystemMetadata()
197 6288 cjones
 * MNReplication.replicate()
198
 *
199 6179 cjones
 */
200 6599 cjones
public class MNodeService extends D1NodeService
201 8810 leinfelder
    implements MNAuthorization, MNCore, MNRead, MNReplication, MNStorage, MNQuery, MNView, MNPackage {
202 6179 cjones
203 7772 tao
    //private static final String PATHQUERY = "pathquery";
204 7849 leinfelder
	public static final String UUID_SCHEME = "UUID";
205
	public static final String DOI_SCHEME = "DOI";
206 7489 leinfelder
	private static final String UUID_PREFIX = "urn:uuid:";
207 9387 walker
208
	private static String XPATH_EML_ID = "/eml:eml/@packageId";
209 7418 leinfelder
210
	/* the logger instance */
211 6475 jones
    private Logger logMetacat = null;
212 6795 cjones
213
    /* A reference to a remote Memeber Node */
214 9295 tao
    //private MNode mn;
215 6795 cjones
216
    /* A reference to a Coordinating Node */
217
    private CNode cn;
218 10104 tao
219
    // shared executor
220
    private static ExecutorService executor = null;
221 6241 cjones
222 10104 tao
    static {
223
        // use a shared executor service with nThreads == one less than available processors
224
        int availableProcessors = Runtime.getRuntime().availableProcessors();
225
        int nThreads = availableProcessors * 1;
226
        nThreads--;
227
        nThreads = Math.max(1, nThreads);
228
        executor = Executors.newFixedThreadPool(nThreads);
229
    }
230 6795 cjones
231 10104 tao
232 6475 jones
    /**
233
     * Singleton accessor to get an instance of MNodeService.
234
     *
235
     * @return instance - the instance of MNodeService
236
     */
237 6542 leinfelder
    public static MNodeService getInstance(HttpServletRequest request) {
238
        return new MNodeService(request);
239 6179 cjones
    }
240
241 6475 jones
    /**
242
     * Constructor, private for singleton access
243
     */
244 6542 leinfelder
    private MNodeService(HttpServletRequest request) {
245
        super(request);
246 6475 jones
        logMetacat = Logger.getLogger(MNodeService.class);
247 6552 leinfelder
248
        // set the Member Node certificate file location
249
        CertificateManager.getInstance().setCertificateLocation(Settings.getConfiguration().getString("D1Client.certificate.file"));
250 6310 cjones
    }
251 6475 jones
252
    /**
253
     * Deletes an object from the Member Node, where the object is either a
254
     * data object or a science metadata object.
255
     *
256
     * @param session - the Session object containing the credentials for the Subject
257
     * @param pid - The object identifier to be deleted
258
     *
259
     * @return pid - the identifier of the object used for the deletion
260
     *
261
     * @throws InvalidToken
262
     * @throws ServiceFailure
263
     * @throws NotAuthorized
264
     * @throws NotFound
265
     * @throws NotImplemented
266
     * @throws InvalidRequest
267
     */
268
    @Override
269
    public Identifier delete(Session session, Identifier pid)
270 6610 cjones
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
271 6475 jones
272 9478 tao
        if(isReadOnlyMode()) {
273 9481 tao
            throw new ServiceFailure("2902", ReadOnlyChecker.DATAONEERROR);
274 9478 tao
        }
275 7162 leinfelder
    	// only admin of  the MN or the CN is allowed a full delete
276
        boolean allowed = false;
277 7330 leinfelder
        allowed = isAdminAuthorized(session);
278 8360 tao
279 9050 tao
        String serviceFailureCode = "2902";
280
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
281
        if(sid != null) {
282
            pid = sid;
283
        }
284
285 8360 tao
        //check if it is the authoritative member node
286
        if(!allowed) {
287
            allowed = isAuthoritativeMNodeAdmin(session, pid);
288
        }
289
290 7162 leinfelder
        if (!allowed) {
291 7245 cjones
            throw new NotAuthorized("1320", "The provided identity does not have " + "permission to delete objects on the Node.");
292 7162 leinfelder
        }
293
294 7077 leinfelder
    	// defer to superclass implementation
295
        return super.delete(session, pid);
296 6250 cjones
    }
297
298 6475 jones
    /**
299
     * Updates an existing object by creating a new object identified by
300
     * newPid on the Member Node which explicitly obsoletes the object
301
     * identified by pid through appropriate changes to the SystemMetadata
302
     * of pid and newPid
303
     *
304
     * @param session - the Session object containing the credentials for the Subject
305
     * @param pid - The identifier of the object to be updated
306
     * @param object - the new object bytes
307
     * @param sysmeta - the new system metadata describing the object
308
     *
309
     * @return newPid - the identifier of the new object
310
     *
311
     * @throws InvalidToken
312
     * @throws ServiceFailure
313
     * @throws NotAuthorized
314
     * @throws NotFound
315
     * @throws NotImplemented
316
     * @throws IdentifierNotUnique
317
     * @throws UnsupportedType
318
     * @throws InsufficientResources
319
     * @throws InvalidSystemMetadata
320
     * @throws InvalidRequest
321
     */
322
    @Override
323 6575 cjones
    public Identifier update(Session session, Identifier pid, InputStream object,
324
        Identifier newPid, SystemMetadata sysmeta)
325
        throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique,
326
        UnsupportedType, InsufficientResources, NotFound,
327
        InvalidSystemMetadata, NotImplemented, InvalidRequest {
328 9781 cjones
329 10118 tao
        long startTime = System.currentTimeMillis();
330
331 9781 cjones
        if(isReadOnlyMode()) {
332
            throw new ServiceFailure("1310", ReadOnlyChecker.DATAONEERROR);
333
        }
334
335
        //transform a sid to a pid if it is applicable
336
        String serviceFailureCode = "1310";
337
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
338
        if(sid != null) {
339
            pid = sid;
340
        }
341
342
        String localId = null;
343
        boolean allowed = false;
344
        boolean isScienceMetadata = false;
345
346
        if (session == null) {
347
        	throw new InvalidToken("1210", "No session has been provided");
348
        }
349
        Subject subject = session.getSubject();
350
351
        // verify the pid is valid format
352
        if (!isValidIdentifier(pid)) {
353
        	throw new InvalidRequest("1202", "The provided identifier is invalid.");
354
        }
355
356
        // verify the new pid is valid format
357
        if (!isValidIdentifier(newPid)) {
358
            throw new InvalidRequest("1202", "The provided identifier is invalid.");
359
        }
360
361
        // make sure that the newPid doesn't exists
362
        boolean idExists = true;
363 9600 tao
        try {
364 9781 cjones
            idExists = IdentifierManager.getInstance().identifierExists(newPid.getValue());
365
        } catch (SQLException e) {
366
            throw new ServiceFailure("1310",
367
                                    "The requested identifier " + newPid.getValue() +
368
                                    " couldn't be determined if it is unique since : "+e.getMessage());
369
        }
370
        if (idExists) {
371
                throw new IdentifierNotUnique("1220",
372
                          "The requested identifier " + newPid.getValue() +
373
                          " is already used by another object and" +
374
                          "therefore can not be used for this object. Clients should choose" +
375
                          "a new identifier that is unique and retry the operation or " +
376
                          "use CN.reserveIdentifier() to reserve one.");
377 9044 tao
378 9781 cjones
        }
379
380
381
382
        // check for the existing identifier
383
        try {
384
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
385 6575 cjones
386 9781 cjones
        } catch (McdbDocNotFoundException e) {
387
            throw new InvalidRequest("1202", "The object with the provided " +
388
                "identifier was not found.");
389 6575 cjones
390 9781 cjones
        } catch (SQLException ee) {
391
            throw new ServiceFailure("1310", "The object with the provided " +
392
                    "identifier "+pid.getValue()+" can't be identified since - "+ee.getMessage());
393
        }
394
395 10118 tao
        long end =System.currentTimeMillis();
396
        logMetacat.debug("MNodeService.update - the time spending on checking the validation of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end- startTime)+ " milli seconds.");
397
398 9781 cjones
        // set the originating node
399
        NodeReference originMemberNode = this.getCapabilities().getIdentifier();
400
        sysmeta.setOriginMemberNode(originMemberNode);
401
402
        // set the submitter to match the certificate
403
        sysmeta.setSubmitter(subject);
404
        // set the dates
405
        Date now = Calendar.getInstance().getTime();
406
        sysmeta.setDateSysMetadataModified(now);
407
        sysmeta.setDateUploaded(now);
408
409
        // make sure serial version is set to something
410
        BigInteger serialVersion = sysmeta.getSerialVersion();
411
        if (serialVersion == null) {
412
        	sysmeta.setSerialVersion(BigInteger.ZERO);
413
        }
414 10118 tao
        long startTime2 = System.currentTimeMillis();
415 9781 cjones
        // does the subject have WRITE ( == update) priveleges on the pid?
416
        //allowed = isAuthorized(session, pid, Permission.WRITE);
417
        //CN having the permission is allowed; user with the write permission and calling on the authoritative node is allowed.
418 9997 tao
        try {
419
            allowed = allowUpdating(session, pid, Permission.WRITE);
420
        }   catch (NotFound e) {
421
            throw new NotFound("1280", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
422
        } catch(ServiceFailure e) {
423
            throw new ServiceFailure("1310", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
424
        } catch(NotAuthorized e) {
425
            throw new NotAuthorized("1200", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
426
        } catch(NotImplemented e) {
427
            throw new NotImplemented("1201","Can't determine if the client has the permission to update he object with id "+pid.getValue()+" since "+e.getDescription());
428
        } catch(InvalidRequest e) {
429
            throw new InvalidRequest("1202", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
430
        } catch(InvalidToken e) {
431
            throw new InvalidToken("1210", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
432
        }
433
434 10118 tao
        end =System.currentTimeMillis();
435
        logMetacat.debug("MNodeService.update - the time spending on checking if the user has the permission to update the old pid "+pid.getValue()+" with the new pid "+newPid.getValue()+" is "+(end- startTime2)+ " milli seconds.");
436
437 9781 cjones
        if (allowed) {
438 10118 tao
            long startTime3 = System.currentTimeMillis();
439 9781 cjones
        	// check quality of SM
440
        	if (sysmeta.getObsoletedBy() != null) {
441
        		throw new InvalidSystemMetadata("1300", "Cannot include obsoletedBy when updating object");
442
        	}
443
        	if (sysmeta.getObsoletes() != null && !sysmeta.getObsoletes().getValue().equals(pid.getValue())) {
444
        		throw new InvalidSystemMetadata("1300", "The identifier provided in obsoletes does not match old Identifier");
445
        	}
446
447
            // get the existing system metadata for the object
448
            SystemMetadata existingSysMeta = getSystemMetadata(session, pid);
449
            //System.out.println("the archive is "+existingSysMeta.getArchived());
450
            //Base on documentation, we can't update an archived object:
451
            //The update operation MUST fail with Exceptions.InvalidRequest on objects that have the Types.SystemMetadata.archived property set to true.
452
            if(existingSysMeta.getArchived() != null && existingSysMeta.getArchived()) {
453
                throw new InvalidRequest("1202","An archived object"+pid.getValue()+" can't be updated");
454 9200 tao
            }
455 9781 cjones
456
            // check for previous update
457
            // see: https://redmine.dataone.org/issues/3336
458
            Identifier existingObsoletedBy = existingSysMeta.getObsoletedBy();
459
            if (existingObsoletedBy != null) {
460
            	throw new InvalidRequest("1202",
461
            			"The previous identifier has already been made obsolete by: " + existingObsoletedBy.getValue());
462 7400 leinfelder
            }
463 10118 tao
            end =System.currentTimeMillis();
464
            logMetacat.debug("MNodeService.update - the time spending on checking the quality of the system metadata of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end- startTime3)+ " milli seconds.");
465
466 9781 cjones
            //check the sid in the system metadata. If it exists, it should be non-exist or match the old sid in the previous system metadata.
467
            Identifier sidInSys = sysmeta.getSeriesId();
468
            if(sidInSys != null) {
469
                if (!isValidIdentifier(sidInSys)) {
470
                    throw new InvalidSystemMetadata("1300", "The provided series id in the system metadata is invalid.");
471 9046 tao
                }
472 9781 cjones
                Identifier previousSid = existingSysMeta.getSeriesId();
473
                if(previousSid != null) {
474
                    // there is a previous sid, if the new sid doesn't match it, the new sid should be non-existing.
475
                    if(!sidInSys.getValue().equals(previousSid.getValue())) {
476 9044 tao
                        try {
477
                            idExists = IdentifierManager.getInstance().identifierExists(sidInSys.getValue());
478
                        } catch (SQLException e) {
479
                            throw new ServiceFailure("1310",
480
                                                    "The requested identifier " + sidInSys.getValue() +
481
                                                    " couldn't be determined if it is unique since : "+e.getMessage());
482
                        }
483
                        if(idExists) {
484 9781 cjones
                            throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata doesn't match the previous series id "
485
                                                            +previousSid.getValue()+", so it should NOT exist. However, it was used by another object.");
486 9044 tao
                        }
487
                    }
488 9781 cjones
                } else {
489
                    // there is no previous sid, the new sid should be non-existing.
490
                    try {
491
                        idExists = IdentifierManager.getInstance().identifierExists(sidInSys.getValue());
492
                    } catch (SQLException e) {
493
                        throw new ServiceFailure("1310",
494
                                                "The requested identifier " + sidInSys.getValue() +
495
                                                " couldn't be determined if it is unique since : "+e.getMessage());
496 9601 tao
                    }
497 9781 cjones
                    if(idExists) {
498
                        throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata should NOT exist since the previous series id is null."
499
                                                        +"However, it was used by another object.");
500
                    }
501 9601 tao
                }
502 9781 cjones
                //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario)
503
                if(sidInSys.getValue().equals(newPid.getValue())) {
504
                    throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata shouldn't have the same value of the pid.");
505
                }
506
            }
507 10118 tao
508
            long end2 =System.currentTimeMillis();
509
            logMetacat.debug("MNodeService.update - the time spending on checking the sid validation of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end2- end)+ " milli seconds.");
510
511 9781 cjones
512
            isScienceMetadata = isScienceMetadata(sysmeta);
513
514
            // do we have XML metadata or a data object?
515
            if (isScienceMetadata) {
516
517
                // update the science metadata XML document
518
                // TODO: handle non-XML metadata/data documents (like netCDF)
519
                // TODO: don't put objects into memory using stream to string
520
                //String objectAsXML = "";
521
                try {
522
                    //objectAsXML = IOUtils.toString(object, "UTF-8");
523 9783 leinfelder
                	String formatId = null;
524
                	if(sysmeta.getFormatId() != null) {
525
                	    formatId = sysmeta.getFormatId().getValue();
526
                	}
527 10280 tao
                    localId = insertOrUpdateDocument(object, "UTF-8", pid, session, "update", formatId, sysmeta.getChecksum());
528 9781 cjones
529
                    // register the newPid and the generated localId
530
                    if (newPid != null) {
531
                        IdentifierManager.getInstance().createMapping(newPid.getValue(), localId);
532 9044 tao
                    }
533 9781 cjones
534
                } catch (IOException e) {
535 10129 tao
                    String msg = "The Node is unable to create the object: "+pid.getValue() + "There was a problem converting the object to XML";
536
                    logMetacat.error(msg, e);
537 9781 cjones
                    throw new ServiceFailure("1310", msg + ": " + e.getMessage());
538
539 10129 tao
                }  catch (PropertyNotFoundException e) {
540
                    String msg = "The Node is unable to create the object. " +pid.getValue()+ " since the properties are not configured well "+e.getMessage();
541
                    logMetacat.error(msg, e);
542
                    throw new ServiceFailure("1310", msg);
543 9044 tao
                }
544 9781 cjones
545 6475 jones
            } else {
546 9781 cjones
547
                // update the data object
548 10280 tao
                localId = insertDataObject(object, newPid, session, sysmeta.getChecksum());
549 9781 cjones
550 6475 jones
            }
551 9781 cjones
552 10118 tao
            long end3 =System.currentTimeMillis();
553
            logMetacat.debug("MNodeService.update - the time spending on saving the object with the new pid "+newPid.getValue()+" is "+(end3- end2)+ " milli seconds.");
554
555 9781 cjones
            // add the newPid to the obsoletedBy list for the existing sysmeta
556
            existingSysMeta.setObsoletedBy(newPid);
557
            //increase version
558
            BigInteger current = existingSysMeta.getSerialVersion();
559
            //System.out.println("the current version is "+current);
560
            current = current.add(BigInteger.ONE);
561
            //System.out.println("the new current version is "+current);
562
            existingSysMeta.setSerialVersion(current);
563
            // then update the existing system metadata
564
            updateSystemMetadata(existingSysMeta);
565
566
            // prep the new system metadata, add pid to the affected lists
567
            sysmeta.setObsoletes(pid);
568
            //sysmeta.addDerivedFrom(pid);
569
570
            // and insert the new system metadata
571
            insertSystemMetadata(sysmeta);
572
573
            // log the update event
574
            EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), subject.getValue(), localId, Event.UPDATE.toString());
575
576 10172 leinfelder
            // queue for QC reporting
577
            MDQClient.submit(sysmeta);
578
579 10118 tao
            long end4 =System.currentTimeMillis();
580
            logMetacat.debug("MNodeService.update - the time spending on updating/saving system metadata  of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" and saving the log information is "+(end4- end3)+ " milli seconds.");
581
582 9781 cjones
            // attempt to register the identifier - it checks if it is a doi
583
            try {
584
    			DOIService.getInstance().registerDOI(sysmeta);
585
    		} catch (Exception e) {
586
                throw new ServiceFailure("1190", "Could not register DOI: " + e.getMessage());
587
    		}
588 10118 tao
            long end5 =System.currentTimeMillis();
589
            logMetacat.debug("MNodeService.update - the time spending on registering the doi (if it is doi ) of the new pid "+newPid.getValue()+" is "+(end5- end4)+ " milli seconds.");
590
591 9781 cjones
        } else {
592
            throw new NotAuthorized("1200", "The provided identity does not have " + "permission to UPDATE the object identified by " + pid.getValue()
593
                    + " on the Member Node.");
594 9600 tao
        }
595 9781 cjones
596 10118 tao
        long end6 =System.currentTimeMillis();
597
        logMetacat.debug("MNodeService.update - the total time of updating the old pid " +pid.getValue() +" whth the new pid "+newPid.getValue()+" is "+(end6- startTime)+ " milli seconds.");
598 6475 jones
        return newPid;
599 6250 cjones
    }
600 6254 cjones
601 6475 jones
    public Identifier create(Session session, Identifier pid, InputStream object, SystemMetadata sysmeta) throws InvalidToken, ServiceFailure, NotAuthorized,
602
            IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest {
603 9781 cjones
604
        if(isReadOnlyMode()) {
605
            throw new ServiceFailure("1190", ReadOnlyChecker.DATAONEERROR);
606
        }
607
        // check for null session
608
        if (session == null) {
609
          throw new InvalidToken("1110", "Session is required to WRITE to the Node.");
610
        }
611
        // verify the pid is valid format
612
        if (!isValidIdentifier(pid)) {
613
            throw new InvalidRequest("1102", "The provided identifier is invalid.");
614
        }
615
        // set the submitter to match the certificate
616
        sysmeta.setSubmitter(session.getSubject());
617
        // set the originating node
618
        NodeReference originMemberNode = this.getCapabilities().getIdentifier();
619
        sysmeta.setOriginMemberNode(originMemberNode);
620
621
        // if no authoritative MN, set it to the same
622
        if (sysmeta.getAuthoritativeMemberNode() == null) {
623
        	sysmeta.setAuthoritativeMemberNode(originMemberNode);
624
        }
625
626
        sysmeta.setArchived(false);
627
628
        // set the dates
629
        Date now = Calendar.getInstance().getTime();
630
        sysmeta.setDateSysMetadataModified(now);
631
        sysmeta.setDateUploaded(now);
632
633
        // set the serial version
634
        sysmeta.setSerialVersion(BigInteger.ZERO);
635
636
        // check that we are not attempting to subvert versioning
637
        if (sysmeta.getObsoletes() != null && sysmeta.getObsoletes().getValue() != null) {
638
            throw new InvalidSystemMetadata("1180",
639
              "The supplied system metadata is invalid. " +
640
              "The obsoletes field cannot have a value when creating entries.");
641
        }
642
643
        if (sysmeta.getObsoletedBy() != null && sysmeta.getObsoletedBy().getValue() != null) {
644
            throw new InvalidSystemMetadata("1180",
645
              "The supplied system metadata is invalid. " +
646
              "The obsoletedBy field cannot have a value when creating entries.");
647
        }
648
649
        // verify the sid in the system metadata
650
        Identifier sid = sysmeta.getSeriesId();
651
        boolean idExists = false;
652
        if(sid != null) {
653
            if (!isValidIdentifier(sid)) {
654
                throw new InvalidSystemMetadata("1180", "The provided series id is invalid.");
655 9068 tao
            }
656 9781 cjones
            try {
657
                idExists = IdentifierManager.getInstance().identifierExists(sid.getValue());
658
            } catch (SQLException e) {
659
                throw new ServiceFailure("1190",
660
                                        "The series identifier " + sid.getValue() +
661
                                        " in the system metadata couldn't be determined if it is unique since : "+e.getMessage());
662 9068 tao
            }
663 9781 cjones
            if (idExists) {
664
                    throw new InvalidSystemMetadata("1180",
665
                              "The series identifier " + sid.getValue() +
666
                              " is already used by another object and" +
667
                              "therefore can not be used for this object. Clients should choose" +
668
                              "a new identifier that is unique and retry the operation or " +
669
                              "use CN.reserveIdentifier() to reserve one.");
670
671 9068 tao
            }
672 9781 cjones
            //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario )
673
            if(sid.getValue().equals(pid.getValue())) {
674
                throw new InvalidSystemMetadata("1180", "The series id "+sid.getValue()+" in the system metadata shouldn't have the same value of the pid.");
675 9068 tao
            }
676 9600 tao
        }
677 9781 cjones
678
        // call the shared impl
679
        Identifier resultPid = super.create(session, pid, object, sysmeta);
680
681
        // attempt to register the identifier - it checks if it is a doi
682
        try {
683
			DOIService.getInstance().registerDOI(sysmeta);
684
		} catch (Exception e) {
685
			ServiceFailure sf = new ServiceFailure("1190", "Could not register DOI: " + e.getMessage());
686
			sf.initCause(e);
687
            throw sf;
688
		}
689
690 10172 leinfelder
        // queue for QC reporting
691
        MDQClient.submit(sysmeta);
692
693 7507 leinfelder
        // return
694
		return resultPid ;
695 6475 jones
    }
696 6250 cjones
697 6475 jones
    /**
698
     * Called by a Coordinating Node to request that the Member Node create a
699
     * copy of the specified object by retrieving it from another Member
700
     * Node and storing it locally so that it can be made accessible to
701
     * the DataONE system.
702
     *
703
     * @param session - the Session object containing the credentials for the Subject
704
     * @param sysmeta - Copy of the CN held system metadata for the object
705
     * @param sourceNode - A reference to node from which the content should be
706
     *                     retrieved. The reference should be resolved by
707
     *                     checking the CN node registry.
708
     *
709
     * @return true if the replication succeeds
710
     *
711
     * @throws ServiceFailure
712
     * @throws NotAuthorized
713
     * @throws NotImplemented
714
     * @throws UnsupportedType
715
     * @throws InsufficientResources
716
     * @throws InvalidRequest
717
     */
718
    @Override
719 6786 cjones
    public boolean replicate(Session session, SystemMetadata sysmeta,
720
            NodeReference sourceNode) throws NotImplemented, ServiceFailure,
721
            NotAuthorized, InvalidRequest, InsufficientResources,
722
            UnsupportedType {
723 9480 tao
        /*if(isReadOnlyMode()) {
724 9478 tao
            throw new InvalidRequest("2153", "The Metacat member node is on the read-only mode and your request can't be fulfiled. Please try again later.");
725 9480 tao
        }*/
726 6786 cjones
727 6875 cjones
        if (session != null && sysmeta != null && sourceNode != null) {
728
            logMetacat.info("MNodeService.replicate() called with parameters: \n" +
729
                            "\tSession.Subject      = "                           +
730
                            session.getSubject().getValue() + "\n"                +
731 7082 cjones
                            "\tidentifier           = "                           +
732
                            sysmeta.getIdentifier().getValue()                    +
733 6875 cjones
                            "\n" + "\tSource NodeReference ="                     +
734
                            sourceNode.getValue());
735
        }
736 6475 jones
        boolean result = false;
737 6786 cjones
        String nodeIdStr = null;
738 6651 cjones
        NodeReference nodeId = null;
739 6786 cjones
740 6795 cjones
        // get the referenced object
741
        Identifier pid = sysmeta.getIdentifier();
742 9068 tao
        // verify the pid is valid format
743
        if (!isValidIdentifier(pid)) {
744
            throw new InvalidRequest("2153", "The provided identifier in the system metadata is invalid.");
745
        }
746 6795 cjones
747
        // get from the membernode
748
        // TODO: switch credentials for the server retrieval?
749
        this.cn = D1Client.getCN();
750
        InputStream object = null;
751
        Session thisNodeSession = null;
752
        SystemMetadata localSystemMetadata = null;
753
        BaseException failure = null;
754 6818 cjones
        String localId = null;
755
756 6795 cjones
        // TODO: check credentials
757
        // cannot be called by public
758 7063 leinfelder
        if (session == null || session.getSubject() == null) {
759 7082 cjones
            String msg = "No session was provided to replicate identifier " +
760
            sysmeta.getIdentifier().getValue();
761 8414 tao
            logMetacat.error(msg);
762 7192 cjones
            throw new NotAuthorized("2152", msg);
763
764 6795 cjones
        }
765
766
767 6651 cjones
        // get the local node id
768
        try {
769 7030 cjones
            nodeIdStr = PropertyService.getProperty("dataone.nodeId");
770 6651 cjones
            nodeId = new NodeReference();
771
            nodeId.setValue(nodeIdStr);
772 6786 cjones
773 6651 cjones
        } catch (PropertyNotFoundException e1) {
774 7030 cjones
            String msg = "Couldn't get dataone.nodeId property: " + e1.getMessage();
775 6795 cjones
            failure = new ServiceFailure("2151", msg);
776 8414 tao
            //setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
777 6795 cjones
            logMetacat.error(msg);
778 8414 tao
            //return true;
779
            throw new ServiceFailure("2151", msg);
780 6786 cjones
781 6651 cjones
        }
782 6795 cjones
783 6475 jones
        try {
784 6786 cjones
            try {
785 9544 tao
                // do we already have a replica?
786 6822 cjones
                try {
787 9544 tao
                    localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
788
                    // if we have a local id, get the local object
789
                    try {
790
                        object = MetacatHandler.read(localId);
791
                    } catch (Exception e) {
792
                        // NOTE: we may already know about this ID because it could be a data file described by a metadata file
793
                        // https://redmine.dataone.org/issues/2572
794
                        // TODO: fix this so that we don't prevent ourselves from getting replicas
795
796
                        // let the CN know that the replication failed
797
                        logMetacat.warn("Object content not found on this node despite having localId: " + localId);
798
                        String msg = "Can't read the object bytes properly, replica is invalid.";
799
                        ServiceFailure serviceFailure = new ServiceFailure("2151", msg);
800
                        setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, serviceFailure);
801
                        logMetacat.warn(msg);
802
                        throw serviceFailure;
803
804
                    }
805
806
                } catch (McdbDocNotFoundException e) {
807
                    logMetacat.info("No replica found. Continuing.");
808 6822 cjones
809 9544 tao
                } catch (SQLException ee) {
810
                    throw new ServiceFailure("2151", "Couldn't identify the local id of the object with the specified identifier "
811
                                            +pid.getValue()+" since - "+ee.getMessage());
812 6822 cjones
                }
813 9544 tao
814
                // no local replica, get a replica
815
                if ( object == null ) {
816
                    /*boolean success = true;
817
                    try {
818
                        //use the v2 ping api to connect the source node
819
                        mn.ping();
820
                    } catch (Exception e) {
821
                        success = false;
822
                    }*/
823
                    D1NodeVersionChecker checker = new D1NodeVersionChecker(sourceNode);
824
                    String nodeVersion = checker.getVersion("MNRead");
825
                    if(nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V1)) {
826
                        //The source node is a v1 node, we use the v1 api
827
                        org.dataone.client.v1.MNode mNodeV1 =  org.dataone.client.v1.itk.D1Client.getMN(sourceNode);
828
                        object = mNodeV1.getReplica(thisNodeSession, pid);
829
                    } else if (nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V2)){
830
                     // session should be null to use the default certificate
831
                        // location set in the Certificate manager
832
                        MNode mn = D1Client.getMN(sourceNode);
833
                        object = mn.getReplica(thisNodeSession, pid);
834
                    } else {
835 10249 tao
                        throw new ServiceFailure("2151", "The version of MNRead service is "+nodeVersion+" in the source node "+sourceNode.getValue()+" and it is not supported. Please check the information in the cn");
836 9544 tao
                    }
837
838
                    logMetacat.info("MNodeService.getReplica() called for identifier "
839
                                    + pid.getValue());
840 6822 cjones
841 9544 tao
                }
842
843
            } catch (InvalidToken e) {
844
                String msg = "Could not retrieve object to replicate (InvalidToken): "+ e.getMessage();
845
                failure = new ServiceFailure("2151", msg);
846
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
847
                logMetacat.error(msg);
848
                throw new ServiceFailure("2151", msg);
849
850
            } catch (NotFound e) {
851
                String msg = "Could not retrieve object to replicate (NotFound): "+ e.getMessage();
852
                failure = new ServiceFailure("2151", msg);
853
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
854
                logMetacat.error(msg);
855
                throw new ServiceFailure("2151", msg);
856
857
            } catch (NotAuthorized e) {
858
                String msg = "Could not retrieve object to replicate (NotAuthorized): "+ e.getMessage();
859
                failure = new ServiceFailure("2151", msg);
860
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
861
                logMetacat.error(msg);
862
                throw new ServiceFailure("2151", msg);
863
            } catch (NotImplemented e) {
864
                String msg = "Could not retrieve object to replicate (mn.getReplica NotImplemented): "+ e.getMessage();
865
                failure = new ServiceFailure("2151", msg);
866
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
867
                logMetacat.error(msg);
868
                throw new ServiceFailure("2151", msg);
869
            } catch (ServiceFailure e) {
870
                String msg = "Could not retrieve object to replicate (ServiceFailure): "+ e.getMessage();
871
                failure = new ServiceFailure("2151", msg);
872
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
873
                logMetacat.error(msg);
874
                throw new ServiceFailure("2151", msg);
875
            } catch (InsufficientResources e) {
876
                String msg = "Could not retrieve object to replicate (InsufficientResources): "+ e.getMessage();
877
                failure = new ServiceFailure("2151", msg);
878
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
879
                logMetacat.error(msg);
880
                throw new ServiceFailure("2151", msg);
881 6819 cjones
            }
882 9544 tao
883
            // verify checksum on the object, if supported
884 10249 tao
            logMetacat.info("MNodeService.replicate - the class of object inputstream is "+object.getClass().getCanonicalName()+". Does it support the reset method? The answer is "+object.markSupported());
885 9544 tao
886
            // add it to local store
887
            Identifier retPid;
888 6786 cjones
            try {
889 9544 tao
                // skip the MN.create -- this mutates the system metadata and we don't want it to
890
                if ( localId == null ) {
891
                    // TODO: this will fail if we already "know" about the identifier
892
                    // FIXME: see https://redmine.dataone.org/issues/2572
893
                    retPid = super.create(session, pid, object, sysmeta);
894
                    result = (retPid.getValue().equals(pid.getValue()));
895
                }
896
897 6786 cjones
            } catch (Exception e) {
898 9544 tao
                String msg = "Could not save object to local store (" + e.getClass().getName() + "): " + e.getMessage();
899 8414 tao
                failure = new ServiceFailure("2151", msg);
900
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
901 9544 tao
                logMetacat.error(msg);
902 8414 tao
                throw new ServiceFailure("2151", msg);
903 9544 tao
904 6786 cjones
            }
905 9544 tao
        } finally {
906
            IOUtils.closeQuietly(object);
907 6693 leinfelder
        }
908 9544 tao
909 6786 cjones
910 6795 cjones
        // finish by setting the replication status
911
        setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.COMPLETED, null);
912 6475 jones
        return result;
913
914 6250 cjones
    }
915 9287 tao
916
    /*
917
     * If the given node supports v2 replication.
918
     */
919
    private boolean supportV2Replication(Node node) throws InvalidRequest {
920
        return supportVersionReplication(node, "v2");
921
    }
922
923
    /*
924
     * If the given node support the the given version replication. Return true if it does.
925
     */
926
    private boolean supportVersionReplication(Node node, String version) throws InvalidRequest{
927
        boolean support = false;
928
        if(node == null) {
929
            throw new InvalidRequest("2153", "There is no capacity information about the node in the replicate.");
930
        } else {
931
            Services services = node.getServices();
932
            if(services == null) {
933
                throw new InvalidRequest("2153", "Can't get replica from a node which doesn't have the replicate service.");
934
            } else {
935
               List<Service> list = services.getServiceList();
936
               if(list == null) {
937
                   throw new InvalidRequest("2153", "Can't get replica from a node which doesn't have the replicate service.");
938
               } else {
939
                   for(Service service : list) {
940
                       if(service != null && service.getName() != null && service.getName().equals("MNReplication") &&
941
                               service.getVersion() != null && service.getVersion().equalsIgnoreCase(version) && service.getAvailable() == true ) {
942
                           support = true;
943
944
                       }
945
                   }
946
               }
947
            }
948
        }
949
        return support;
950
    }
951 6179 cjones
952 6475 jones
    /**
953
     * Return the object identified by the given object identifier
954
     *
955
     * @param session - the Session object containing the credentials for the Subject
956
     * @param pid - the object identifier for the given object
957
     *
958
     * @return inputStream - the input stream of the given object
959
     *
960
     * @throws InvalidToken
961
     * @throws ServiceFailure
962
     * @throws NotAuthorized
963
     * @throws InvalidRequest
964
     * @throws NotImplemented
965
     */
966
    @Override
967 6610 cjones
    public InputStream get(Session session, Identifier pid)
968
    throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
969 6258 cjones
970 6475 jones
        return super.get(session, pid);
971 6258 cjones
972 6259 cjones
    }
973 6258 cjones
974 6475 jones
    /**
975
     * Returns a Checksum for the specified object using an accepted hashing algorithm
976
     *
977
     * @param session - the Session object containing the credentials for the Subject
978
     * @param pid - the object identifier for the given object
979
     * @param algorithm -  the name of an algorithm that will be used to compute
980
     *                     a checksum of the bytes of the object
981
     *
982
     * @return checksum - the checksum of the given object
983
     *
984
     * @throws InvalidToken
985
     * @throws ServiceFailure
986
     * @throws NotAuthorized
987
     * @throws NotFound
988
     * @throws InvalidRequest
989
     * @throws NotImplemented
990
     */
991
    @Override
992 6610 cjones
    public Checksum getChecksum(Session session, Identifier pid, String algorithm)
993
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
994
        InvalidRequest, NotImplemented {
995 6258 cjones
996 6475 jones
        Checksum checksum = null;
997 9040 tao
        String serviceFailure = "1410";
998
        String notFound = "1420";
999
        //Checkum only handles the pid, not sid
1000
        checkV1SystemMetaPidExist(pid, serviceFailure, "The checksum for the object specified by "+pid.getValue()+" couldn't be returned ",  notFound,
1001
                "The object specified by "+pid.getValue()+" does not exist at this node.");
1002 6475 jones
        InputStream inputStream = get(session, pid);
1003
1004 6259 cjones
        try {
1005 6475 jones
            checksum = ChecksumUtil.checksum(inputStream, algorithm);
1006
1007
        } catch (NoSuchAlgorithmException e) {
1008
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned due to an internal error: "
1009
                    + e.getMessage());
1010 6259 cjones
        } catch (IOException e) {
1011 6475 jones
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned due to an internal error: "
1012
                    + e.getMessage());
1013 6259 cjones
        }
1014 6382 cjones
1015 6475 jones
        if (checksum == null) {
1016
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned.");
1017
        }
1018 6258 cjones
1019 6475 jones
        return checksum;
1020 6259 cjones
    }
1021 6179 cjones
1022 6475 jones
    /**
1023
     * Return the system metadata for a given object
1024
     *
1025
     * @param session - the Session object containing the credentials for the Subject
1026
     * @param pid - the object identifier for the given object
1027
     *
1028
     * @return inputStream - the input stream of the given system metadata object
1029
     *
1030
     * @throws InvalidToken
1031
     * @throws ServiceFailure
1032
     * @throws NotAuthorized
1033
     * @throws NotFound
1034
     * @throws InvalidRequest
1035
     * @throws NotImplemented
1036
     */
1037
    @Override
1038 6610 cjones
    public SystemMetadata getSystemMetadata(Session session, Identifier pid)
1039
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
1040
        NotImplemented {
1041 6341 leinfelder
1042 6475 jones
        return super.getSystemMetadata(session, pid);
1043
    }
1044 6341 leinfelder
1045 6475 jones
    /**
1046
     * Retrieve the list of objects present on the MN that match the calling parameters
1047
     *
1048
     * @param session - the Session object containing the credentials for the Subject
1049
     * @param startTime - Specifies the beginning of the time range from which
1050
     *                    to return object (>=)
1051
     * @param endTime - Specifies the beginning of the time range from which
1052
     *                  to return object (>=)
1053
     * @param objectFormat - Restrict results to the specified object format
1054
     * @param replicaStatus - Indicates if replicated objects should be returned in the list
1055
     * @param start - The zero-based index of the first value, relative to the
1056
     *                first record of the resultset that matches the parameters.
1057
     * @param count - The maximum number of entries that should be returned in
1058
     *                the response. The Member Node may return less entries
1059
     *                than specified in this value.
1060
     *
1061
     * @return objectList - the list of objects matching the criteria
1062
     *
1063
     * @throws InvalidToken
1064
     * @throws ServiceFailure
1065
     * @throws NotAuthorized
1066
     * @throws InvalidRequest
1067
     * @throws NotImplemented
1068
     */
1069
    @Override
1070 8810 leinfelder
    public ObjectList listObjects(Session session, Date startTime, Date endTime, ObjectFormatIdentifier objectFormatId, Identifier identifier, Boolean replicaStatus, Integer start,
1071 6475 jones
            Integer count) throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken {
1072 9249 tao
        NodeReference nodeId = null;
1073
        if(!replicaStatus) {
1074
            //not include those objects whose authoritative node is not this mn
1075
            nodeId = new NodeReference();
1076
            try {
1077
                String currentNodeId = PropertyService.getInstance().getProperty("dataone.nodeId"); // return only pids for which this mn
1078
                nodeId.setValue(currentNodeId);
1079
            } catch(Exception e) {
1080
                throw new ServiceFailure("1580", e.getMessage());
1081
            }
1082
        }
1083
        return super.listObjects(session, startTime, endTime, objectFormatId, identifier, nodeId, start, count);
1084 6229 cjones
    }
1085 6179 cjones
1086 6475 jones
    /**
1087 6476 jones
     * Return a description of the node's capabilities and services.
1088 6475 jones
     *
1089
     * @return node - the technical capabilities of the Member Node
1090
     *
1091
     * @throws ServiceFailure
1092
     * @throws NotAuthorized
1093
     * @throws InvalidRequest
1094
     * @throws NotImplemented
1095
     */
1096
    @Override
1097 6610 cjones
    public Node getCapabilities()
1098
        throws NotImplemented, ServiceFailure {
1099 6179 cjones
1100 6475 jones
        String nodeName = null;
1101
        String nodeId = null;
1102 6492 jones
        String subject = null;
1103 6938 cjones
        String contactSubject = null;
1104 6475 jones
        String nodeDesc = null;
1105 6476 jones
        String nodeTypeString = null;
1106
        NodeType nodeType = null;
1107 9253 tao
        List<String> mnCoreServiceVersions = null;
1108
        List<String> mnReadServiceVersions = null;
1109
        List<String> mnAuthorizationServiceVersions = null;
1110
        List<String> mnStorageServiceVersions = null;
1111
        List<String> mnReplicationServiceVersions = null;
1112 6179 cjones
1113 6475 jones
        boolean nodeSynchronize = false;
1114
        boolean nodeReplicate = false;
1115 9253 tao
        List<String> mnCoreServiceAvailables = null;
1116
        List<String> mnReadServiceAvailables = null;
1117
        List<String> mnAuthorizationServiceAvailables = null;
1118
        List<String> mnStorageServiceAvailables = null;
1119
        List<String> mnReplicationServiceAvailables = null;
1120 6179 cjones
1121 6475 jones
        try {
1122
            // get the properties of the node based on configuration information
1123 9253 tao
            nodeName = Settings.getConfiguration().getString("dataone.nodeName");
1124
            nodeId = Settings.getConfiguration().getString("dataone.nodeId");
1125
            subject = Settings.getConfiguration().getString("dataone.subject");
1126
            contactSubject = Settings.getConfiguration().getString("dataone.contactSubject");
1127
            nodeDesc = Settings.getConfiguration().getString("dataone.nodeDescription");
1128
            nodeTypeString = Settings.getConfiguration().getString("dataone.nodeType");
1129 6476 jones
            nodeType = NodeType.convert(nodeTypeString);
1130 9253 tao
            nodeSynchronize = new Boolean(Settings.getConfiguration().getString("dataone.nodeSynchronize")).booleanValue();
1131
            nodeReplicate = new Boolean(Settings.getConfiguration().getString("dataone.nodeReplicate")).booleanValue();
1132 6475 jones
1133 6476 jones
            // Set the properties of the node based on configuration information and
1134
            // calls to current status methods
1135 9809 tao
            String serviceName = SystemUtil.getContextURL() + "/" + PropertyService.getProperty("dataone.serviceName");
1136 6476 jones
            Node node = new Node();
1137 6542 leinfelder
            node.setBaseURL(serviceName + "/" + nodeTypeString);
1138 6476 jones
            node.setDescription(nodeDesc);
1139 6475 jones
1140 6476 jones
            // set the node's health information
1141
            node.setState(NodeState.UP);
1142
1143
            // set the ping response to the current value
1144
            Ping canPing = new Ping();
1145
            canPing.setSuccess(false);
1146
            try {
1147 6803 leinfelder
            	Date pingDate = ping();
1148
                canPing.setSuccess(pingDate != null);
1149
            } catch (BaseException e) {
1150 6476 jones
                e.printStackTrace();
1151 6803 leinfelder
                // guess it can't be pinged
1152 6476 jones
            }
1153 6610 cjones
1154 6476 jones
            node.setPing(canPing);
1155 6475 jones
1156 6476 jones
            NodeReference identifier = new NodeReference();
1157
            identifier.setValue(nodeId);
1158
            node.setIdentifier(identifier);
1159 6492 jones
            Subject s = new Subject();
1160
            s.setValue(subject);
1161
            node.addSubject(s);
1162 6938 cjones
            Subject contact = new Subject();
1163
            contact.setValue(contactSubject);
1164
            node.addContactSubject(contact);
1165 6476 jones
            node.setName(nodeName);
1166
            node.setReplicate(nodeReplicate);
1167
            node.setSynchronize(nodeSynchronize);
1168 6475 jones
1169 6476 jones
            // services: MNAuthorization, MNCore, MNRead, MNReplication, MNStorage
1170
            Services services = new Services();
1171 6475 jones
1172 9253 tao
            mnCoreServiceVersions = Settings.getConfiguration().getList("dataone.mnCore.serviceVersion");
1173
            mnCoreServiceAvailables = Settings.getConfiguration().getList("dataone.mnCore.serviceAvailable");
1174
            if(mnCoreServiceVersions != null && mnCoreServiceAvailables != null && mnCoreServiceVersions.size() == mnCoreServiceAvailables.size()) {
1175
                for(int i=0; i<mnCoreServiceVersions.size(); i++) {
1176
                    String version = mnCoreServiceVersions.get(i);
1177
                    boolean available = new Boolean(mnCoreServiceAvailables.get(i)).booleanValue();
1178
                    Service sMNCore = new Service();
1179
                    sMNCore.setName("MNCore");
1180
                    sMNCore.setVersion(version);
1181
                    sMNCore.setAvailable(available);
1182
                    services.addService(sMNCore);
1183
                }
1184
            }
1185
1186
            mnReadServiceVersions = Settings.getConfiguration().getList("dataone.mnRead.serviceVersion");
1187
            mnReadServiceAvailables = Settings.getConfiguration().getList("dataone.mnRead.serviceAvailable");
1188
            if(mnReadServiceVersions != null && mnReadServiceAvailables != null && mnReadServiceVersions.size()==mnReadServiceAvailables.size()) {
1189
                for(int i=0; i<mnReadServiceVersions.size(); i++) {
1190
                    String version = mnReadServiceVersions.get(i);
1191
                    boolean available = new Boolean(mnReadServiceAvailables.get(i)).booleanValue();
1192
                    Service sMNRead = new Service();
1193
                    sMNRead.setName("MNRead");
1194
                    sMNRead.setVersion(version);
1195
                    sMNRead.setAvailable(available);
1196
                    services.addService(sMNRead);
1197
                }
1198
            }
1199
1200
            mnAuthorizationServiceVersions = Settings.getConfiguration().getList("dataone.mnAuthorization.serviceVersion");
1201
            mnAuthorizationServiceAvailables = Settings.getConfiguration().getList("dataone.mnAuthorization.serviceAvailable");
1202
            if(mnAuthorizationServiceVersions != null && mnAuthorizationServiceAvailables != null && mnAuthorizationServiceVersions.size()==mnAuthorizationServiceAvailables.size()) {
1203
                for(int i=0; i<mnAuthorizationServiceVersions.size(); i++) {
1204
                    String version = mnAuthorizationServiceVersions.get(i);
1205
                    boolean available = new Boolean(mnAuthorizationServiceAvailables.get(i)).booleanValue();
1206
                    Service sMNAuthorization = new Service();
1207
                    sMNAuthorization.setName("MNAuthorization");
1208
                    sMNAuthorization.setVersion(version);
1209
                    sMNAuthorization.setAvailable(available);
1210
                    services.addService(sMNAuthorization);
1211
                }
1212
            }
1213
1214
            mnStorageServiceVersions = Settings.getConfiguration().getList("dataone.mnStorage.serviceVersion");
1215
            mnStorageServiceAvailables = Settings.getConfiguration().getList("dataone.mnStorage.serviceAvailable");
1216
            if(mnStorageServiceVersions != null && mnStorageServiceAvailables != null && mnStorageServiceVersions.size() == mnStorageServiceAvailables.size()) {
1217
                for(int i=0; i<mnStorageServiceVersions.size(); i++) {
1218
                    String version = mnStorageServiceVersions.get(i);
1219
                    boolean available = new Boolean(mnStorageServiceAvailables.get(i)).booleanValue();
1220
                    Service sMNStorage = new Service();
1221
                    sMNStorage.setName("MNStorage");
1222
                    sMNStorage.setVersion(version);
1223
                    sMNStorage.setAvailable(available);
1224
                    services.addService(sMNStorage);
1225
                }
1226
            }
1227
1228
            mnReplicationServiceVersions = Settings.getConfiguration().getList("dataone.mnReplication.serviceVersion");
1229
            mnReplicationServiceAvailables = Settings.getConfiguration().getList("dataone.mnReplication.serviceAvailable");
1230
            if(mnReplicationServiceVersions != null && mnReplicationServiceAvailables != null && mnReplicationServiceVersions.size() == mnReplicationServiceAvailables.size()) {
1231
                for (int i=0; i<mnReplicationServiceVersions.size(); i++) {
1232
                    String version = mnReplicationServiceVersions.get(i);
1233
                    boolean available = new Boolean(mnReplicationServiceAvailables.get(i)).booleanValue();
1234
                    Service sMNReplication = new Service();
1235
                    sMNReplication.setName("MNReplication");
1236
                    sMNReplication.setVersion(version);
1237
                    sMNReplication.setAvailable(available);
1238
                    services.addService(sMNReplication);
1239
                }
1240
            }
1241
1242 6476 jones
            node.setServices(services);
1243 6475 jones
1244 6476 jones
            // Set the schedule for synchronization
1245
            Synchronization synchronization = new Synchronization();
1246
            Schedule schedule = new Schedule();
1247
            Date now = new Date();
1248 6689 leinfelder
            schedule.setYear(PropertyService.getProperty("dataone.nodeSynchronization.schedule.year"));
1249
            schedule.setMon(PropertyService.getProperty("dataone.nodeSynchronization.schedule.mon"));
1250
            schedule.setMday(PropertyService.getProperty("dataone.nodeSynchronization.schedule.mday"));
1251
            schedule.setWday(PropertyService.getProperty("dataone.nodeSynchronization.schedule.wday"));
1252
            schedule.setHour(PropertyService.getProperty("dataone.nodeSynchronization.schedule.hour"));
1253
            schedule.setMin(PropertyService.getProperty("dataone.nodeSynchronization.schedule.min"));
1254
            schedule.setSec(PropertyService.getProperty("dataone.nodeSynchronization.schedule.sec"));
1255 6476 jones
            synchronization.setSchedule(schedule);
1256
            synchronization.setLastHarvested(now);
1257
            synchronization.setLastCompleteHarvest(now);
1258
            node.setSynchronization(synchronization);
1259 6475 jones
1260 6476 jones
            node.setType(nodeType);
1261
            return node;
1262 6475 jones
1263 6476 jones
        } catch (PropertyNotFoundException pnfe) {
1264
            String msg = "MNodeService.getCapabilities(): " + "property not found: " + pnfe.getMessage();
1265
            logMetacat.error(msg);
1266
            throw new ServiceFailure("2162", msg);
1267
        }
1268 6228 cjones
    }
1269 6179 cjones
1270 8810 leinfelder
1271 6179 cjones
1272 6475 jones
    /**
1273
     * A callback method used by a CN to indicate to a MN that it cannot
1274
     * complete synchronization of the science metadata identified by pid.  Log
1275
     * the event in the metacat event log.
1276
     *
1277
     * @param session
1278
     * @param syncFailed
1279
     *
1280
     * @throws ServiceFailure
1281
     * @throws NotAuthorized
1282
     * @throws NotImplemented
1283
     */
1284
    @Override
1285 6991 leinfelder
    public boolean synchronizationFailed(Session session, SynchronizationFailed syncFailed)
1286 6610 cjones
        throws NotImplemented, ServiceFailure, NotAuthorized {
1287 6179 cjones
1288 6475 jones
        String localId;
1289 7075 cjones
        Identifier pid;
1290 10218 tao
        if ( syncFailed.getPid() != null ) {
1291 7075 cjones
            pid = new Identifier();
1292
            pid.setValue(syncFailed.getPid());
1293
            boolean allowed;
1294
1295
            //are we allowed? only CNs
1296
            try {
1297 7142 leinfelder
                allowed = isAdminAuthorized(session);
1298 7075 cjones
                if ( !allowed ){
1299
                    throw new NotAuthorized("2162",
1300
                            "Not allowed to call synchronizationFailed() on this node.");
1301
                }
1302
            } catch (InvalidToken e) {
1303
                throw new NotAuthorized("2162",
1304
                        "Not allowed to call synchronizationFailed() on this node.");
1305 6331 leinfelder
1306 7075 cjones
            }
1307
1308
        } else {
1309
            throw new ServiceFailure("2161", "The identifier cannot be null.");
1310
1311
        }
1312
1313 6475 jones
        try {
1314 7075 cjones
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1315 6475 jones
        } catch (McdbDocNotFoundException e) {
1316 7075 cjones
            throw new ServiceFailure("2161", "The identifier specified by " +
1317
                    syncFailed.getPid() + " was not found on this node.");
1318 6179 cjones
1319 9024 tao
        } catch (SQLException e) {
1320
            throw new ServiceFailure("2161", "Couldn't identify the local id of the identifier specified by " +
1321
                    syncFailed.getPid() + " since "+e.getMessage());
1322 6475 jones
        }
1323
        // TODO: update the CN URL below when the CNRead.SynchronizationFailed
1324
        // method is changed to include the URL as a parameter
1325 9380 rnahf
        logMetacat.warn("Synchronization for the object identified by " +
1326 7075 cjones
                pid.getValue() + " failed from " + syncFailed.getNodeId() +
1327 9380 rnahf
                " with message: " + syncFailed.getDescription() +
1328
                ". Logging the event to the Metacat EventLog as a 'syncFailed' event.");
1329 6475 jones
        // TODO: use the event type enum when the SYNCHRONIZATION_FAILED event is added
1330 6532 leinfelder
        String principal = Constants.SUBJECT_PUBLIC;
1331 6506 leinfelder
        if (session != null && session.getSubject() != null) {
1332 6575 cjones
          principal = session.getSubject().getValue();
1333 6506 leinfelder
        }
1334
        try {
1335 6575 cjones
          EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), principal, localId, "synchronization_failed");
1336 6506 leinfelder
        } catch (Exception e) {
1337 7075 cjones
            throw new ServiceFailure("2161", "Could not log the error for: " + pid.getValue());
1338 6991 leinfelder
        }
1339 6475 jones
        //EventLog.getInstance().log("CN URL WILL GO HERE",
1340
        //  session.getSubject().getValue(), localId, Event.SYNCHRONIZATION_FAILED);
1341 6991 leinfelder
        return true;
1342 6179 cjones
1343 6260 cjones
    }
1344
1345 6475 jones
    /**
1346
     * Essentially a get() but with different logging behavior
1347
     */
1348
    @Override
1349 6540 cjones
    public InputStream getReplica(Session session, Identifier pid)
1350 9314 tao
        throws NotAuthorized, NotImplemented, ServiceFailure, InvalidToken, NotFound {
1351 6179 cjones
1352 6540 cjones
        logMetacat.info("MNodeService.getReplica() called.");
1353
1354 6653 leinfelder
        // cannot be called by public
1355
        if (session == null) {
1356
        	throw new InvalidToken("2183", "No session was provided.");
1357
        }
1358
1359 6631 cjones
        logMetacat.info("MNodeService.getReplica() called with parameters: \n" +
1360
             "\tSession.Subject      = " + session.getSubject().getValue() + "\n" +
1361
             "\tIdentifier           = " + pid.getValue());
1362
1363 6475 jones
        InputStream inputStream = null; // bytes to be returned
1364
        handler = new MetacatHandler(new Timer());
1365
        boolean allowed = false;
1366
        String localId; // the metacat docid for the pid
1367 6179 cjones
1368 6475 jones
        // get the local docid from Metacat
1369
        try {
1370
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1371
        } catch (McdbDocNotFoundException e) {
1372 9314 tao
            throw new NotFound("2185", "The object specified by " +
1373 6610 cjones
                    pid.getValue() + " does not exist at this node.");
1374
1375 9024 tao
        } catch (SQLException e) {
1376
            throw new ServiceFailure("2181", "The local id of the object specified by " +
1377
                    pid.getValue() + " couldn't be identified since "+e.getMessage());
1378 6475 jones
        }
1379 6234 cjones
1380 6552 leinfelder
        Subject targetNodeSubject = session.getSubject();
1381 6185 leinfelder
1382 6552 leinfelder
        // check for authorization to replicate, null session to act as this source MN
1383 6610 cjones
        try {
1384 6777 leinfelder
            allowed = D1Client.getCN().isNodeAuthorized(null, targetNodeSubject, pid);
1385 6610 cjones
        } catch (InvalidToken e1) {
1386
            throw new ServiceFailure("2181", "Could not determine if node is authorized: "
1387
                + e1.getMessage());
1388
1389
        } catch (NotFound e1) {
1390 9314 tao
            throw new NotFound("2185", "Could not find the object "+pid.getValue()+" in this node - "
1391 6610 cjones
                    + e1.getMessage());
1392 6384 cjones
1393 6610 cjones
        } catch (InvalidRequest e1) {
1394
            throw new ServiceFailure("2181", "Could not determine if node is authorized: "
1395
                    + e1.getMessage());
1396
1397
        }
1398
1399 6540 cjones
        logMetacat.info("Called D1Client.isNodeAuthorized(). Allowed = " + allowed +
1400
            " for identifier " + pid.getValue());
1401
1402 6475 jones
        // if the person is authorized, perform the read
1403
        if (allowed) {
1404
            try {
1405 6986 jones
                inputStream = MetacatHandler.read(localId);
1406 6475 jones
            } catch (Exception e) {
1407 9314 tao
                throw new ServiceFailure("2181", "The object specified by " +
1408 6610 cjones
                    pid.getValue() + "could not be returned due to error: " + e.getMessage());
1409 6475 jones
            }
1410 9314 tao
        } else {
1411
            throw new NotAuthorized("2182", "The pid "+pid.getValue()+" is not authorized to be read by the client.");
1412 6475 jones
        }
1413 6384 cjones
1414 6475 jones
        // if we fail to set the input stream
1415
        if (inputStream == null) {
1416 6610 cjones
            throw new ServiceFailure("2181", "The object specified by " +
1417 9314 tao
                pid.getValue() + " can't be returned from the node.");
1418 6475 jones
        }
1419
1420
        // log the replica event
1421
        String principal = null;
1422
        if (session.getSubject() != null) {
1423
            principal = session.getSubject().getValue();
1424
        }
1425 6576 cjones
        EventLog.getInstance().log(request.getRemoteAddr(),
1426
            request.getHeader("User-Agent"), principal, localId, "replicate");
1427 6475 jones
1428
        return inputStream;
1429
    }
1430 9340 tao
1431 6573 cjones
    /**
1432 6600 cjones
     * A method to notify the Member Node that the authoritative copy of
1433 6599 cjones
     * system metadata on the Coordinating Nodes has changed.
1434 9340 tao
     *
1435 6599 cjones
     * @param session   Session information that contains the identity of the
1436
     *                  calling user as retrieved from the X.509 certificate
1437
     *                  which must be traceable to the CILogon service.
1438
     * @param serialVersion   The serialVersion of the system metadata
1439
     * @param dateSysMetaLastModified  The time stamp for when the system metadata was changed
1440
     * @throws NotImplemented
1441
     * @throws ServiceFailure
1442
     * @throws NotAuthorized
1443
     * @throws InvalidRequest
1444
     * @throws InvalidToken
1445
     */
1446 6991 leinfelder
    public boolean systemMetadataChanged(Session session, Identifier pid,
1447 6599 cjones
        long serialVersion, Date dateSysMetaLastModified)
1448
        throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest,
1449
        InvalidToken {
1450 9340 tao
        boolean needCheckAuthoriativeNode = true;
1451
        return systemMetadataChanged(needCheckAuthoriativeNode, session, pid,serialVersion, dateSysMetaLastModified);
1452
    }
1453
1454
    /**
1455
     * A method to notify the Member Node that the authoritative copy of
1456
     * system metadata on the Coordinating Nodes has changed.
1457
     * @param needCheckAuthoriativeNode  this is for the dataone version 2. In the
1458
     * version 2, there are two scenarios:
1459
     * 1. If the node is the authoritative node, it only accepts serial version and replica list.
1460
     * 2. If the node is a replica, it accepts everything.
1461
     * For the v1, api, the parameter should be false.
1462
     * @param session   Session information that contains the identity of the
1463
     *                  calling user as retrieved from the X.509 certificate
1464
     *                  which must be traceable to the CILogon service.
1465
     * @param serialVersion   The serialVersion of the system metadata
1466
     * @param dateSysMetaLastModified  The time stamp for when the system metadata was changed
1467
     * @throws NotImplemented
1468
     * @throws ServiceFailure
1469
     * @throws NotAuthorized
1470
     * @throws InvalidRequest
1471
     * @throws InvalidToken
1472
     */
1473
    public boolean systemMetadataChanged(boolean needCheckAuthoriativeNode, Session session, Identifier pid,
1474
        long serialVersion, Date dateSysMetaLastModified)
1475
        throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest,
1476
        InvalidToken {
1477 6599 cjones
1478 9479 tao
        /*if(isReadOnlyMode()) {
1479 9478 tao
            throw new InvalidRequest("1334", "The Metacat member node is on the read-only mode and your request can't be fulfiled. Please try again later.");
1480 9479 tao
        }*/
1481 7600 cjones
        // cannot be called by public
1482
        if (session == null) {
1483 9051 tao
        	throw new InvalidToken("1332", "No session was provided.");
1484 7600 cjones
        }
1485
1486 9051 tao
        String serviceFailureCode = "1333";
1487
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
1488
        if(sid != null) {
1489
            pid = sid;
1490
        }
1491
1492 6600 cjones
        SystemMetadata currentLocalSysMeta = null;
1493
        SystemMetadata newSysMeta = null;
1494
        CNode cn = D1Client.getCN();
1495
        NodeList nodeList = null;
1496
        Subject callingSubject = null;
1497
        boolean allowed = false;
1498
1499
        // are we allowed to call this?
1500
        callingSubject = session.getSubject();
1501
        nodeList = cn.listNodes();
1502
1503
        for(Node node : nodeList.getNodeList()) {
1504
            // must be a CN
1505
            if ( node.getType().equals(NodeType.CN)) {
1506
               List<Subject> subjectList = node.getSubjectList();
1507
               // the calling subject must be in the subject list
1508
               if ( subjectList.contains(callingSubject)) {
1509
                   allowed = true;
1510
1511
               }
1512
1513
            }
1514
        }
1515
1516
        if (!allowed ) {
1517
            String msg = "The subject identified by " + callingSubject.getValue() +
1518
              " is not authorized to call this service.";
1519
            throw new NotAuthorized("1331", msg);
1520
1521
        }
1522
        try {
1523 9340 tao
            HazelcastService.getInstance().getSystemMetadataMap().lock(pid);
1524 6600 cjones
1525 9340 tao
            // compare what we have locally to what is sent in the change notification
1526 6600 cjones
            try {
1527 9340 tao
                currentLocalSysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
1528
1529
            } catch (RuntimeException e) {
1530
                String msg = "SystemMetadata for pid " + pid.getValue() +
1531
                  " couldn't be updated because it couldn't be found locally: " +
1532
                  e.getMessage();
1533 6692 leinfelder
                logMetacat.error(msg);
1534 9340 tao
                ServiceFailure sf = new ServiceFailure("1333", msg);
1535 6692 leinfelder
                sf.initCause(e);
1536 9340 tao
                throw sf;
1537 6600 cjones
            }
1538 6692 leinfelder
1539 9340 tao
            if(currentLocalSysMeta == null) {
1540
                throw new InvalidRequest("1334", "We can't find the system metadata in the node for the id "+pid.getValue());
1541
            }
1542 9382 tao
            if (currentLocalSysMeta.getSerialVersion().longValue() <= serialVersion ) {
1543 9340 tao
                try {
1544
                    newSysMeta = cn.getSystemMetadata(null, pid);
1545
                } catch (NotFound e) {
1546
                    // huh? you just said you had it
1547
                	String msg = "On updating the local copy of system metadata " +
1548
                    "for pid " + pid.getValue() +", the CN reports it is not found." +
1549
                    " The error message was: " + e.getMessage();
1550
                    logMetacat.error(msg);
1551
                    //ServiceFailure sf = new ServiceFailure("1333", msg);
1552
                    InvalidRequest sf = new InvalidRequest("1334", msg);
1553
                    sf.initCause(e);
1554
                    throw sf;
1555 9056 tao
                }
1556 9340 tao
1557
                //check about the sid in the system metadata
1558
                Identifier newSID = newSysMeta.getSeriesId();
1559
                if(newSID != null) {
1560
                    if (!isValidIdentifier(newSID)) {
1561
                        throw new InvalidRequest("1334", "The series identifier in the new system metadata is invalid.");
1562
                    }
1563
                    Identifier currentSID = currentLocalSysMeta.getSeriesId();
1564
                    if( currentSID != null && currentSID.getValue() != null) {
1565
                        if(!newSID.getValue().equals(currentSID.getValue())) {
1566
                            //newSID doesn't match the currentSID. The newSID shouldn't be used.
1567
                            try {
1568
                                if(IdentifierManager.getInstance().identifierExists(newSID.getValue())) {
1569
                                    throw new InvalidRequest("1334", "The series identifier "+newSID.getValue()+" in the new system metadata has been used by another object.");
1570
                                }
1571
                            } catch (SQLException sql) {
1572
                                throw new ServiceFailure("1333", "Couldn't determine if the SID "+newSID.getValue()+" in the system metadata exists in the node since "+sql.getMessage());
1573
                            }
1574
1575
                        }
1576
                    } else {
1577
                        //newSID shouldn't be used
1578 9056 tao
                        try {
1579
                            if(IdentifierManager.getInstance().identifierExists(newSID.getValue())) {
1580
                                throw new InvalidRequest("1334", "The series identifier "+newSID.getValue()+" in the new system metadata has been used by another object.");
1581
                            }
1582
                        } catch (SQLException sql) {
1583
                            throw new ServiceFailure("1333", "Couldn't determine if the SID "+newSID.getValue()+" in the system metadata exists in the node since "+sql.getMessage());
1584
                        }
1585
                    }
1586 9340 tao
                }
1587
                // update the local copy of system metadata for the pid
1588
                try {
1589
                    if(needCheckAuthoriativeNode) {
1590
                        //this is for the v2 api.
1591
                        if(isAuthoritativeNode(pid)) {
1592
                            //this is the authoritative node, so we only accept replica and serial version
1593 9375 tao
                            logMetacat.debug("MNodeService.systemMetadataChanged - this is the authoritative node for the pid "+pid.getValue());
1594 9340 tao
                            List<Replica> replicas = newSysMeta.getReplicaList();
1595
                            newSysMeta = currentLocalSysMeta;
1596
                            newSysMeta.setSerialVersion(new BigInteger((new Long(serialVersion)).toString()));
1597
                            newSysMeta.setReplicaList(replicas);
1598 9375 tao
                        } else {
1599
                            //we need to archive the object in the replica node
1600
                            logMetacat.debug("MNodeService.systemMetadataChanged - this is NOT the authoritative node for the pid "+pid.getValue());
1601
                            logMetacat.debug("MNodeService.systemMetadataChanged - the new value of archive is "+newSysMeta.getArchived()+" for the pid "+pid.getValue());
1602
                            logMetacat.debug("MNodeService.systemMetadataChanged - the local value of archive is "+currentLocalSysMeta.getArchived()+" for the pid "+pid.getValue());
1603
                            if (newSysMeta.getArchived() != null && newSysMeta.getArchived() == true  &&
1604
                                    ((currentLocalSysMeta.getArchived() != null && currentLocalSysMeta.getArchived() == false ) || currentLocalSysMeta.getArchived() == null)){
1605
                                logMetacat.debug("MNodeService.systemMetadataChanged - start to archive object "+pid.getValue());
1606
                                boolean logArchive = false;
1607
                                boolean needUpdateModificationDate = false;
1608
                                try {
1609
                                    archiveObject(logArchive, session, pid, newSysMeta, needUpdateModificationDate);
1610
                                } catch (NotFound e) {
1611
                                    throw new InvalidRequest("1334", "Can't find the pid "+pid.getValue()+" for archive.");
1612
                                }
1613
1614 9384 tao
                            } else if((newSysMeta.getArchived() == null || newSysMeta.getArchived() == false) && (currentLocalSysMeta.getArchived() != null && currentLocalSysMeta.getArchived() == true )) {
1615
                                throw new InvalidRequest("1334", "The pid "+pid.getValue()+" has been archived and it can't be reset to false.");
1616 9375 tao
                            }
1617 9056 tao
                        }
1618
                    }
1619 9340 tao
                    HazelcastService.getInstance().getSystemMetadataMap().put(newSysMeta.getIdentifier(), newSysMeta);
1620
                    logMetacat.info("Updated local copy of system metadata for pid " +
1621
                        pid.getValue() + " after change notification from the CN.");
1622
1623
                    // TODO: consider inspecting the change for archive
1624
                    // see: https://projects.ecoinformatics.org/ecoinfo/issues/6417
1625
    //                if (newSysMeta.getArchived() != null && newSysMeta.getArchived().booleanValue()) {
1626
    //                	try {
1627
    //						this.archive(session, newSysMeta.getIdentifier());
1628
    //					} catch (NotFound e) {
1629
    //						// do we care? nothing to do about it now
1630
    //						logMetacat.error(e.getMessage(), e);
1631
    //					}
1632
    //                }
1633
1634
                } catch (RuntimeException e) {
1635
                    String msg = "SystemMetadata for pid " + pid.getValue() +
1636
                      " couldn't be updated: " +
1637
                      e.getMessage();
1638
                    logMetacat.error(msg);
1639
                    ServiceFailure sf = new ServiceFailure("1333", msg);
1640
                    sf.initCause(e);
1641
                    throw sf;
1642 9056 tao
                }
1643 6600 cjones
1644 9375 tao
                try {
1645
                    String localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1646
                    EventLog.getInstance().log(request.getRemoteAddr(),
1647
                            request.getHeader("User-Agent"), session.getSubject().getValue(),
1648
                            localId, "updateSystemMetadata");
1649
                } catch (Exception e) {
1650
                    // do nothing, no localId to log with
1651
                    logMetacat.warn("MNodeService.systemMetadataChanged - Could not log 'updateSystemMetadata' event because no localId was found for pid: " + pid.getValue());
1652
                }
1653
1654 9340 tao
1655 6600 cjones
            }
1656 9340 tao
        } finally {
1657
            HazelcastService.getInstance().getSystemMetadataMap().unlock(pid);
1658
        }
1659
1660 9382 tao
        if (currentLocalSysMeta.getSerialVersion().longValue() <= serialVersion ) {
1661 9248 leinfelder
            // attempt to re-register the identifier (it checks if it is a doi)
1662
            try {
1663 9340 tao
                DOIService.getInstance().registerDOI(newSysMeta);
1664
            } catch (Exception e) {
1665
                logMetacat.warn("Could not [re]register DOI: " + e.getMessage(), e);
1666
            }
1667 9248 leinfelder
1668 8464 leinfelder
            // submit for indexing
1669
            try {
1670 9340 tao
                MetacatSolrIndex.getInstance().submit(newSysMeta.getIdentifier(), newSysMeta, null, true);
1671
            } catch (Exception e) {
1672 8464 leinfelder
                logMetacat.error("Could not submit changed systemMetadata for indexing, pid: " + newSysMeta.getIdentifier().getValue(), e);
1673 9340 tao
            }
1674 6600 cjones
        }
1675
1676 6991 leinfelder
        return true;
1677
1678 6599 cjones
    }
1679
1680 6795 cjones
    /*
1681
     * Set the replication status for the object on the Coordinating Node
1682
     *
1683
     * @param session - the session for the this target node
1684
     * @param pid - the identifier of the object being updated
1685
     * @param nodeId - the identifier of this target node
1686
     * @param status - the replication status to set
1687
     * @param failure - the exception to include, if any
1688
     */
1689
    private void setReplicationStatus(Session session, Identifier pid,
1690
        NodeReference nodeId, ReplicationStatus status, BaseException failure)
1691
        throws ServiceFailure, NotImplemented, NotAuthorized,
1692
        InvalidRequest {
1693
1694
        // call the CN as the MN to set the replication status
1695
        try {
1696
            this.cn = D1Client.getCN();
1697
            this.cn.setReplicationStatus(session, pid, nodeId,
1698
                    status, failure);
1699
1700
        } catch (InvalidToken e) {
1701 7091 leinfelder
        	String msg = "Could not set the replication status for " + pid.getValue() + " on the CN (InvalidToken): " + e.getMessage();
1702
            logMetacat.error(msg);
1703
        	throw new ServiceFailure("2151",
1704
                    msg);
1705 6795 cjones
1706
        } catch (NotFound e) {
1707 7091 leinfelder
        	String msg = "Could not set the replication status for " + pid.getValue() + " on the CN (NotFound): " + e.getMessage();
1708
            logMetacat.error(msg);
1709
        	throw new ServiceFailure("2151",
1710
                    msg);
1711 6795 cjones
1712
        }
1713
    }
1714 8591 leinfelder
1715
    private SystemMetadata makePublicIfNot(SystemMetadata sysmeta, Identifier pid) throws ServiceFailure, InvalidToken, NotFound, NotImplemented, InvalidRequest {
1716
    	// check if it is publicly readable
1717
		boolean isPublic = false;
1718
		Subject publicSubject = new Subject();
1719
		publicSubject.setValue(Constants.SUBJECT_PUBLIC);
1720
		Session publicSession = new Session();
1721
		publicSession.setSubject(publicSubject);
1722
		AccessRule publicRule = new AccessRule();
1723
		publicRule.addPermission(Permission.READ);
1724
		publicRule.addSubject(publicSubject);
1725
1726
		// see if we need to add the rule
1727
		try {
1728
			isPublic = this.isAuthorized(publicSession, pid, Permission.READ);
1729
		} catch (NotAuthorized na) {
1730
			// well, certainly not authorized for public read!
1731
		}
1732
		if (!isPublic) {
1733
			sysmeta.getAccessPolicy().addAllow(publicRule);
1734
		}
1735
1736
		return sysmeta;
1737
    }
1738 7099 leinfelder
1739
	@Override
1740 7441 leinfelder
	public Identifier generateIdentifier(Session session, String scheme, String fragment)
1741 7099 leinfelder
			throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented,
1742
			InvalidRequest {
1743 7448 leinfelder
1744 8210 leinfelder
		// check for null session
1745
        if (session == null) {
1746
          throw new InvalidToken("2190", "Session is required to generate an Identifier at this Node.");
1747
        }
1748
1749 7441 leinfelder
		Identifier identifier = new Identifier();
1750 7448 leinfelder
1751 7489 leinfelder
		// handle different schemes
1752
		if (scheme.equalsIgnoreCase(UUID_SCHEME)) {
1753
			// UUID
1754
			UUID uuid = UUID.randomUUID();
1755
            identifier.setValue(UUID_PREFIX + uuid.toString());
1756
		} else if (scheme.equalsIgnoreCase(DOI_SCHEME)) {
1757 7512 leinfelder
			// generate a DOI
1758 7448 leinfelder
			try {
1759 7512 leinfelder
				identifier = DOIService.getInstance().generateDOI();
1760 7448 leinfelder
			} catch (EZIDException e) {
1761 7512 leinfelder
				ServiceFailure sf = new ServiceFailure("2191", "Could not generate DOI: " + e.getMessage());
1762
				sf.initCause(e);
1763
				throw sf;
1764 7448 leinfelder
			}
1765 7489 leinfelder
		} else {
1766
			// default if we don't know the scheme
1767
			if (fragment != null) {
1768
				// for now, just autogen with fragment
1769
				String autogenId = DocumentUtil.generateDocumentId(fragment, 0);
1770
				identifier.setValue(autogenId);
1771
			} else {
1772
				// autogen with no fragment
1773
				String autogenId = DocumentUtil.generateDocumentId(0);
1774
				identifier.setValue(autogenId);
1775
			}
1776 7448 leinfelder
		}
1777
1778 7441 leinfelder
		// TODO: reserve the identifier with the CN. We can only do this when
1779
		// 1) the MN is part of a CN cluster
1780
		// 2) the request is from an authenticated user
1781
1782
		return identifier;
1783 7099 leinfelder
	}
1784 7144 leinfelder
1785 8810 leinfelder
1786 7144 leinfelder
1787
	@Override
1788 8810 leinfelder
	public QueryEngineDescription getQueryEngineDescription(Session session, String engine)
1789 7144 leinfelder
			throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented,
1790
			NotFound {
1791 7772 tao
	    if(engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) {
1792 8162 tao
	        if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) {
1793
                throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1794
            }
1795 7634 tao
	        QueryEngineDescription qed = new QueryEngineDescription();
1796 7772 tao
	        qed.setName(EnabledQueryEngines.PATHQUERYENGINE);
1797 7634 tao
	        qed.setQueryEngineVersion("1.0");
1798
	        qed.addAdditionalInfo("This is the traditional structured query for Metacat");
1799
	        Vector<String> pathsForIndexing = null;
1800
	        try {
1801
	            pathsForIndexing = SystemUtil.getPathsForIndexing();
1802
	        } catch (MetacatUtilException e) {
1803
	            logMetacat.warn("Could not get index paths", e);
1804
	        }
1805
	        for (String fieldName: pathsForIndexing) {
1806
	            QueryField field = new QueryField();
1807
	            field.addDescription("Indexed field for path '" + fieldName + "'");
1808
	            field.setName(fieldName);
1809
	            field.setReturnable(true);
1810
	            field.setSearchable(true);
1811
	            field.setSortable(false);
1812
	            // TODO: determine type and multivaluedness
1813
	            field.setType(String.class.getName());
1814
	            //field.setMultivalued(true);
1815
	            qed.addQueryField(field);
1816
	        }
1817
	        return qed;
1818 7772 tao
	    } else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) {
1819 7781 tao
	        if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) {
1820 7772 tao
                throw new NotImplemented("0000", "MNodeService.getQueryEngineDescription - the query engine "+engine +" hasn't been implemented or has been disabled.");
1821 7781 tao
            }
1822 7634 tao
	        try {
1823 7662 tao
	            QueryEngineDescription qed = MetacatSolrEngineDescriptionHandler.getInstance().getQueryEngineDescritpion();
1824 7634 tao
	            return qed;
1825
	        } catch (Exception e) {
1826
	            e.printStackTrace();
1827
	            throw new ServiceFailure("Solr server error", e.getMessage());
1828
	        }
1829
	    } else {
1830
	        throw new NotFound("404", "The Metacat member node can't find the query engine - "+engine);
1831
	    }
1832
1833 7417 leinfelder
	}
1834
1835
	@Override
1836 8810 leinfelder
	public QueryEngineList listQueryEngines(Session session) throws InvalidToken,
1837 7417 leinfelder
			ServiceFailure, NotAuthorized, NotImplemented {
1838
		QueryEngineList qel = new QueryEngineList();
1839 7781 tao
		//qel.addQueryEngine(EnabledQueryEngines.PATHQUERYENGINE);
1840
		//qel.addQueryEngine(EnabledQueryEngines.SOLRENGINE);
1841
		List<String> enables = EnabledQueryEngines.getInstance().getEnabled();
1842 7772 tao
		for(String name : enables) {
1843
		    qel.addQueryEngine(name);
1844 7781 tao
		}
1845 7417 leinfelder
		return qel;
1846
	}
1847
1848
	@Override
1849 8810 leinfelder
	public InputStream query(Session session, String engine, String query) throws InvalidToken,
1850 7417 leinfelder
			ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented,
1851
			NotFound {
1852 7648 tao
	    String user = Constants.SUBJECT_PUBLIC;
1853
        String[] groups= null;
1854 7680 tao
        Set<Subject> subjects = null;
1855 7648 tao
        if (session != null) {
1856
            user = session.getSubject().getValue();
1857 7680 tao
            subjects = AuthUtils.authorizedClientSubjects(session);
1858 7648 tao
            if (subjects != null) {
1859
                List<String> groupList = new ArrayList<String>();
1860
                for (Subject subject: subjects) {
1861
                    groupList.add(subject.getValue());
1862
                }
1863
                groups = groupList.toArray(new String[0]);
1864
            }
1865 7680 tao
        } else {
1866
            //add the public user subject to the set
1867
            Subject subject = new Subject();
1868
            subject.setValue(Constants.SUBJECT_PUBLIC);
1869 7757 leinfelder
            subjects = new HashSet<Subject>();
1870 7680 tao
            subjects.add(subject);
1871 7648 tao
        }
1872 7680 tao
        //System.out.println("====== user is "+user);
1873
        //System.out.println("====== groups are "+groups);
1874 7772 tao
		if (engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) {
1875 8162 tao
		    if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) {
1876
                throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1877
            }
1878 7417 leinfelder
			try {
1879
				DBQuery queryobj = new DBQuery();
1880 7648 tao
1881 7417 leinfelder
				String results = queryobj.performPathquery(query, user, groups);
1882 7757 leinfelder
				ContentTypeByteArrayInputStream ctbais = new ContentTypeByteArrayInputStream(results.getBytes(MetaCatServlet.DEFAULT_ENCODING));
1883
				ctbais.setContentType("text/xml");
1884
				return ctbais;
1885 7417 leinfelder
1886
			} catch (Exception e) {
1887 7757 leinfelder
				throw new ServiceFailure("Pathquery error", e.getMessage());
1888 7417 leinfelder
			}
1889
1890 7772 tao
		} else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) {
1891 7781 tao
		    if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) {
1892 7772 tao
		        throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1893 7781 tao
		    }
1894 7634 tao
		    logMetacat.info("The query is ==================================== \n"+query);
1895 7620 tao
		    try {
1896 7634 tao
1897 7680 tao
                return MetacatSolrIndex.getInstance().query(query, subjects);
1898 7620 tao
            } catch (Exception e) {
1899
                // TODO Auto-generated catch block
1900
                throw new ServiceFailure("Solr server error", e.getMessage());
1901
            }
1902 7417 leinfelder
		}
1903
		return null;
1904
	}
1905 7849 leinfelder
1906
	/**
1907
	 * Given an existing Science Metadata PID, this method mints a DOI
1908
	 * and updates the original object "publishing" the update with the DOI.
1909
	 * This includes updating the ORE map that describes the Science Metadata+data.
1910
	 * TODO: ensure all referenced objects allow public read
1911
	 *
1912
	 * @see https://projects.ecoinformatics.org/ecoinfo/issues/6014
1913
	 *
1914
	 * @param originalIdentifier
1915
	 * @param request
1916 7864 leinfelder
	 * @throws InvalidRequest
1917
	 * @throws NotImplemented
1918
	 * @throws NotAuthorized
1919
	 * @throws ServiceFailure
1920
	 * @throws InvalidToken
1921 7849 leinfelder
	 * @throws NotFound
1922 7864 leinfelder
	 * @throws InvalidSystemMetadata
1923
	 * @throws InsufficientResources
1924
	 * @throws UnsupportedType
1925
	 * @throws IdentifierNotUnique
1926 7849 leinfelder
	 */
1927 9387 walker
	public Identifier publish(Session session, Identifier originalIdentifier) throws InvalidToken,
1928
	ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, NotFound, IdentifierNotUnique,
1929
	UnsupportedType, InsufficientResources, InvalidSystemMetadata, IOException {
1930 7849 leinfelder
1931 9092 tao
	    String serviceFailureCode = "1030";
1932
	    Identifier sid = getPIDForSID(originalIdentifier, serviceFailureCode);
1933
	    if(sid != null) {
1934
	        originalIdentifier = sid;
1935
	    }
1936 8141 leinfelder
		// get the original SM
1937
		SystemMetadata originalSystemMetadata = this.getSystemMetadata(session, originalIdentifier);
1938
1939 8594 leinfelder
		// make copy of it using the marshaller to ensure DEEP copy
1940
		SystemMetadata sysmeta = null;
1941 8141 leinfelder
		try {
1942 8594 leinfelder
			ByteArrayOutputStream baos = new ByteArrayOutputStream();
1943
			TypeMarshaller.marshalTypeToOutputStream(originalSystemMetadata, baos);
1944
			sysmeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, new ByteArrayInputStream(baos.toByteArray()));
1945 8141 leinfelder
		} catch (Exception e) {
1946
			// report as service failure
1947
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1948
			sf.initCause(e);
1949
			throw sf;
1950
		}
1951
1952 7849 leinfelder
		// mint a DOI for the new revision
1953
		Identifier newIdentifier = this.generateIdentifier(session, MNodeService.DOI_SCHEME, null);
1954 8141 leinfelder
1955
		// set new metadata values
1956 7849 leinfelder
		sysmeta.setIdentifier(newIdentifier);
1957
		sysmeta.setObsoletes(originalIdentifier);
1958
		sysmeta.setObsoletedBy(null);
1959
1960 8591 leinfelder
		// ensure it is publicly readable
1961
		sysmeta = makePublicIfNot(sysmeta, originalIdentifier);
1962
1963 9387 walker
		//Get the bytes
1964
		InputStream inputStream = null;
1965
		boolean isScienceMetadata = isScienceMetadata(sysmeta);
1966
		//If it's a science metadata doc, we want to update the packageId first
1967
		if(isScienceMetadata){
1968
			InputStream originalObject = this.get(session, originalIdentifier);
1969
1970
			//Edit the science metadata with the new package Id (EML)
1971
			inputStream = editScienceMetadata(session, originalObject, originalIdentifier, newIdentifier);
1972
		}
1973
		else{
1974
			inputStream = this.get(session, originalIdentifier);
1975
		}
1976 7849 leinfelder
1977
		// update the object
1978 7864 leinfelder
		this.update(session, originalIdentifier, inputStream, newIdentifier, sysmeta);
1979 7849 leinfelder
1980 7864 leinfelder
		// update ORE that references the scimeta
1981 8190 leinfelder
		// first try the naive method, then check the SOLR index
1982 7849 leinfelder
		try {
1983 7864 leinfelder
			String localId = IdentifierManager.getInstance().getLocalId(originalIdentifier.getValue());
1984 7849 leinfelder
1985 7864 leinfelder
			Identifier potentialOreIdentifier = new Identifier();
1986
			potentialOreIdentifier.setValue(SystemMetadataFactory.RESOURCE_MAP_PREFIX + localId);
1987 7849 leinfelder
1988 7864 leinfelder
			InputStream oreInputStream = null;
1989
			try {
1990
				oreInputStream = this.get(session, potentialOreIdentifier);
1991
			} catch (NotFound nf) {
1992
				// this is probably okay for many sci meta data docs
1993
				logMetacat.warn("No potential ORE map found for: " + potentialOreIdentifier.getValue());
1994 8190 leinfelder
				// try the SOLR index
1995 8200 leinfelder
				List<Identifier> potentialOreIdentifiers = this.lookupOreFor(originalIdentifier, false);
1996 8190 leinfelder
				if (potentialOreIdentifiers != null) {
1997
					potentialOreIdentifier = potentialOreIdentifiers.get(0);
1998
					try {
1999
						oreInputStream = this.get(session, potentialOreIdentifier);
2000
					} catch (NotFound nf2) {
2001
						// this is probably okay for many sci meta data docs
2002
						logMetacat.warn("No potential ORE map found for: " + potentialOreIdentifier.getValue());
2003
					}
2004
				}
2005 7864 leinfelder
			}
2006
			if (oreInputStream != null) {
2007
				Identifier newOreIdentifier = MNodeService.getInstance(request).generateIdentifier(session, MNodeService.UUID_SCHEME, null);
2008
2009
				Map<Identifier, Map<Identifier, List<Identifier>>> resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap(oreInputStream);
2010
				Map<Identifier, List<Identifier>> sciMetaMap = resourceMapStructure.get(potentialOreIdentifier);
2011
				List<Identifier> dataIdentifiers = sciMetaMap.get(originalIdentifier);
2012 8591 leinfelder
2013 7864 leinfelder
				// reconstruct the ORE with the new identifiers
2014
				sciMetaMap.remove(originalIdentifier);
2015
				sciMetaMap.put(newIdentifier, dataIdentifiers);
2016
2017
				ResourceMap resourceMap = ResourceMapFactory.getInstance().createResourceMap(newOreIdentifier, sciMetaMap);
2018
				String resourceMapString = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap);
2019
2020
				// get the original ORE SM and update the values
2021 8141 leinfelder
				SystemMetadata originalOreSysMeta = this.getSystemMetadata(session, potentialOreIdentifier);
2022
				SystemMetadata oreSysMeta = new SystemMetadata();
2023
				try {
2024 8594 leinfelder
					ByteArrayOutputStream baos = new ByteArrayOutputStream();
2025
					TypeMarshaller.marshalTypeToOutputStream(originalOreSysMeta, baos);
2026
					oreSysMeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, new ByteArrayInputStream(baos.toByteArray()));
2027 8141 leinfelder
				} catch (Exception e) {
2028
					// report as service failure
2029
					ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2030
					sf.initCause(e);
2031
					throw sf;
2032
				}
2033
2034 7864 leinfelder
				oreSysMeta.setIdentifier(newOreIdentifier);
2035
				oreSysMeta.setObsoletes(potentialOreIdentifier);
2036
				oreSysMeta.setObsoletedBy(null);
2037
				oreSysMeta.setSize(BigInteger.valueOf(resourceMapString.getBytes("UTF-8").length));
2038
				oreSysMeta.setChecksum(ChecksumUtil.checksum(resourceMapString.getBytes("UTF-8"), oreSysMeta.getChecksum().getAlgorithm()));
2039
2040 8591 leinfelder
				// ensure ORE is publicly readable
2041 8594 leinfelder
				oreSysMeta = makePublicIfNot(oreSysMeta, potentialOreIdentifier);
2042 8591 leinfelder
2043
				// ensure all data objects allow public read
2044
				List<String> pidsToSync = new ArrayList<String>();
2045
				for (Identifier dataId: dataIdentifiers) {
2046
					SystemMetadata dataSysMeta = this.getSystemMetadata(session, dataId);
2047
					dataSysMeta = makePublicIfNot(dataSysMeta, dataId);
2048
					this.updateSystemMetadata(dataSysMeta);
2049
					pidsToSync.add(dataId.getValue());
2050
				}
2051
				SyncAccessPolicy sap = new SyncAccessPolicy();
2052
				try {
2053
					sap.sync(pidsToSync);
2054
				} catch (Exception e) {
2055
					// ignore
2056
					logMetacat.warn("Error attempting to sync access for data objects when publishing package");
2057
				}
2058
2059 7864 leinfelder
				// save the updated ORE
2060
				this.update(
2061
						session,
2062
						potentialOreIdentifier,
2063
						new ByteArrayInputStream(resourceMapString.getBytes("UTF-8")),
2064
						newOreIdentifier,
2065
						oreSysMeta);
2066
2067 8362 leinfelder
			} else {
2068
				// create a new ORE for them
2069
				// https://projects.ecoinformatics.org/ecoinfo/issues/6194
2070
				try {
2071
					// find the local id for the NEW package.
2072
					String newLocalId = IdentifierManager.getInstance().getLocalId(newIdentifier.getValue());
2073
2074
					@SuppressWarnings("unused")
2075
					SystemMetadata extraSysMeta = SystemMetadataFactory.createSystemMetadata(newLocalId, true, false);
2076 8591 leinfelder
					// should be done generating the ORE here, and the same permissions were used from the metadata object
2077 8362 leinfelder
2078
				} catch (Exception e) {
2079
					// oops, guess there was a problem - no package for you
2080
					logMetacat.error("Could not generate new ORE for published object: " + newIdentifier.getValue(), e);
2081
				}
2082 7864 leinfelder
			}
2083
		} catch (McdbDocNotFoundException e) {
2084
			// report as service failure
2085
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2086
			sf.initCause(e);
2087
			throw sf;
2088
		} catch (UnsupportedEncodingException e) {
2089
			// report as service failure
2090
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2091
			sf.initCause(e);
2092
			throw sf;
2093
		} catch (OREException e) {
2094
			// report as service failure
2095
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2096
			sf.initCause(e);
2097
			throw sf;
2098
		} catch (URISyntaxException e) {
2099
			// report as service failure
2100
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2101
			sf.initCause(e);
2102
			throw sf;
2103
		} catch (OREParserException e) {
2104
			// report as service failure
2105
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2106
			sf.initCause(e);
2107
			throw sf;
2108
		} catch (ORESerialiserException e) {
2109
			// report as service failure
2110
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2111
			sf.initCause(e);
2112
			throw sf;
2113
		} catch (NoSuchAlgorithmException e) {
2114
			// report as service failure
2115
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2116
			sf.initCause(e);
2117
			throw sf;
2118 9024 tao
		} catch (SQLException e) {
2119
            // report as service failure
2120
            ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2121
            sf.initCause(e);
2122
            throw sf;
2123
        }
2124 7849 leinfelder
2125
		return newIdentifier;
2126
	}
2127 7850 leinfelder
2128
	/**
2129 9387 walker
	   * Update a science metadata document with its new Identifier
2130
	   *
2131
	   * @param session - the Session object containing the credentials for the Subject
2132
	   * @param object - the InputStream for the XML object to be edited
2133
	   * @param pid - the Identifier of the XML object to be updated
2134
	   * @param newPid = the new Identifier to give to the modified XML doc
2135
	   *
2136
	   * @return newObject - The InputStream for the modified XML object
2137
	   *
2138
	   * @throws ServiceFailure
2139
	   * @throws IOException
2140
	   * @throws UnsupportedEncodingException
2141
	   * @throws InvalidToken
2142
	   * @throws NotAuthorized
2143
	   * @throws NotFound
2144
	   * @throws NotImplemented
2145
	   */
2146
	  public InputStream editScienceMetadata(Session session, InputStream object, Identifier pid, Identifier newPid)
2147
	  	throws ServiceFailure, IOException, UnsupportedEncodingException, InvalidToken, NotAuthorized, NotFound, NotImplemented {
2148
2149
		logMetacat.debug("D1NodeService.editScienceMetadata() called.");
2150
2151
		 InputStream newObject = null;
2152
2153
	    try{
2154
	    	//Get the root node of the XML document
2155
	    	byte[] xmlBytes  = IOUtils.toByteArray(object);
2156
	        String xmlStr = new String(xmlBytes, "UTF-8");
2157
2158
	    	Document doc = XMLUtilities.getXMLReaderAsDOMDocument(new StringReader(xmlStr));
2159
		    org.w3c.dom.Node docNode = doc.getDocumentElement();
2160
2161
		    //Get the system metadata for this object
2162
		    SystemMetadata sysMeta = null;
2163
		    try{
2164
		    	sysMeta = getSystemMetadata(session, pid);
2165
		    } catch(NotAuthorized e){
2166
		    	throw new ServiceFailure("1030", "D1NodeService.editScienceMetadata(): " +
2167
		    			"This session is not authorized to access the system metadata for " +
2168
		    			pid.getValue() + " : " + e.getMessage());
2169
		    } catch(NotFound e){
2170
		    	throw new ServiceFailure("1030", "D1NodeService.editScienceMetadata(): " +
2171
		    			"Could not find the system metadata for " +
2172
		    			pid.getValue() + " : " + e.getMessage());
2173
		    }
2174
2175
		    //Get the formatId
2176
	        ObjectFormatIdentifier objFormatId = sysMeta.getFormatId();
2177
	        String formatId = objFormatId.getValue();
2178
2179
	    	//For all EML formats
2180
	        if(formatId.indexOf("eml") == 0){
2181
	        	//Update or add the id attribute
2182
	    	    XMLUtilities.addAttributeNodeToDOMTree(docNode, XPATH_EML_ID, newPid.getValue());
2183
	        }
2184
2185
	        //The modified object InputStream
2186
		    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
2187
		    Source xmlSource = new DOMSource(docNode);
2188
		    Result outputTarget = new StreamResult(outputStream);
2189
		    TransformerFactory.newInstance().newTransformer().transform(xmlSource, outputTarget);
2190
		    newObject = new ByteArrayInputStream(outputStream.toByteArray());
2191
2192
	    } catch(TransformerException e) {
2193
	    	throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " +
2194
	                "Could not update the ID in the XML document for " +
2195
	                "pid " + pid.getValue() +" : " + e.getMessage());
2196
	    } catch(IOException e){
2197
	    	throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " +
2198
	                "Could not update the ID in the XML document for " +
2199
	                "pid " + pid.getValue() +" : " + e.getMessage());
2200
	    }
2201
2202
	    return newObject;
2203
	  }
2204
2205
	/**
2206 8190 leinfelder
	 * Determines if we already have registered an ORE map for this package
2207
	 * NOTE: uses a solr query to locate OREs for the object
2208
	 * @param guid of the EML/packaging object
2209
	 * @return list of resource map identifiers for the given pid
2210
	 */
2211 8200 leinfelder
	public List<Identifier> lookupOreFor(Identifier guid, boolean includeObsolete) {
2212 8190 leinfelder
		// Search for the ORE if we can find it
2213
		String pid = guid.getValue();
2214
		List<Identifier> retList = null;
2215
		try {
2216 8745 leinfelder
			String query = "fl=id,resourceMap&wt=xml&q=-obsoletedBy:[* TO *]+resourceMap:[* TO *]+id:\"" + pid + "\"";
2217 8200 leinfelder
			if (includeObsolete) {
2218 8745 leinfelder
				query = "fl=id,resourceMap&wt=xml&q=resourceMap:[* TO *]+id:\"" + pid + "\"";
2219 8200 leinfelder
			}
2220
2221 8810 leinfelder
			InputStream results = this.query(null, "solr", query);
2222 8190 leinfelder
			org.w3c.dom.Node rootNode = XMLUtilities.getXMLReaderAsDOMTreeRootNode(new InputStreamReader(results, "UTF-8"));
2223
			//String resultString = XMLUtilities.getDOMTreeAsString(rootNode);
2224
			org.w3c.dom.NodeList nodeList = XMLUtilities.getNodeListWithXPath(rootNode, "//arr[@name=\"resourceMap\"]/str");
2225
			if (nodeList != null && nodeList.getLength() > 0) {
2226
				retList = new ArrayList<Identifier>();
2227
				for (int i = 0; i < nodeList.getLength(); i++) {
2228
					String found = nodeList.item(i).getFirstChild().getNodeValue();
2229
					Identifier oreId = new Identifier();
2230
					oreId.setValue(found);
2231
					retList.add(oreId);
2232
				}
2233
			}
2234
		} catch (Exception e) {
2235
			logMetacat.error("Error checking for resourceMap[s] on pid " + pid + ". " + e.getMessage(), e);
2236
		}
2237
2238
		return retList;
2239
	}
2240
2241 8810 leinfelder
2242
	@Override
2243
	public InputStream getPackage(Session session, ObjectFormatIdentifier formatId,
2244
			Identifier pid) throws InvalidToken, ServiceFailure,
2245
			NotAuthorized, InvalidRequest, NotImplemented, NotFound {
2246 9290 tao
	    if(formatId == null) {
2247
	        throw new InvalidRequest("2873", "The format type can't be null in the getpackage method.");
2248 9305 tao
	    } else if(!formatId.getValue().equals("application/bagit-097")) {
2249 9290 tao
	        throw new NotImplemented("", "The format "+formatId.getValue()+" is not supported in the getpackage method");
2250
	    }
2251 9092 tao
	    String serviceFailureCode = "2871";
2252
	    Identifier sid = getPIDForSID(pid, serviceFailureCode);
2253
	    if(sid != null) {
2254
	        pid = sid;
2255
	    }
2256 7850 leinfelder
		InputStream bagInputStream = null;
2257
		BagFactory bagFactory = new BagFactory();
2258
		Bag bag = bagFactory.createBag();
2259
2260
		// track the temp files we use so we can delete them when finished
2261
		List<File> tempFiles = new ArrayList<File>();
2262
2263
		// the pids to include in the package
2264
		List<Identifier> packagePids = new ArrayList<Identifier>();
2265
2266 7855 leinfelder
		// catch non-D1 service errors and throw as ServiceFailures
2267
		try {
2268 8437 walker
			//Create a map of dataone ids and file names
2269
			Map<Identifier, String> fileNames = new HashMap<Identifier, String>();
2270 7855 leinfelder
2271 8800 leinfelder
			// track the pid-to-file mapping
2272
			StringBuffer pidMapping = new StringBuffer();
2273
2274 7855 leinfelder
			// find the package contents
2275
			SystemMetadata sysMeta = this.getSystemMetadata(session, pid);
2276 8025 leinfelder
			if (ObjectFormatCache.getInstance().getFormat(sysMeta.getFormatId()).getFormatType().equals("RESOURCE")) {
2277 8437 walker
				//Get the resource map as a map of Identifiers
2278 7855 leinfelder
				InputStream oreInputStream = this.get(session, pid);
2279
				Map<Identifier, Map<Identifier, List<Identifier>>> resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap(oreInputStream);
2280
				packagePids.addAll(resourceMapStructure.keySet());
2281 8437 walker
				//Loop through each object in this resource map
2282 7855 leinfelder
				for (Map<Identifier, List<Identifier>> entries: resourceMapStructure.values()) {
2283 8437 walker
					//Loop through each metadata object in this entry
2284
					Set<Identifier> metadataIdentifiers = entries.keySet();
2285
					for(Identifier metadataID: metadataIdentifiers){
2286
						try{
2287
							//Get the system metadata for this metadata object
2288
							SystemMetadata metadataSysMeta = this.getSystemMetadata(session, metadataID);
2289
2290 8800 leinfelder
							// include user-friendly metadata
2291
							if (ObjectFormatCache.getInstance().getFormat(metadataSysMeta.getFormatId()).getFormatType().equals("METADATA")) {
2292
								InputStream metadataStream = this.get(session, metadataID);
2293
2294
								try {
2295
									// transform
2296
						            String format = "default";
2297
2298
									DBTransform transformer = new DBTransform();
2299
						            String documentContent = IOUtils.toString(metadataStream, "UTF-8");
2300
						            String sourceType = metadataSysMeta.getFormatId().getValue();
2301
						            String targetType = "-//W3C//HTML//EN";
2302
						            ByteArrayOutputStream baos = new ByteArrayOutputStream();
2303
						            Writer writer = new OutputStreamWriter(baos , "UTF-8");
2304
						            // TODO: include more params?
2305
						            Hashtable<String, String[]> params = new Hashtable<String, String[]>();
2306
						            String localId = null;
2307
									try {
2308
										localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
2309
									} catch (McdbDocNotFoundException e) {
2310
										throw new NotFound("1020", e.getMessage());
2311
									}
2312
									params.put("qformat", new String[] {format});
2313
						            params.put("docid", new String[] {localId});
2314
						            params.put("pid", new String[] {pid.getValue()});
2315
						            params.put("displaymodule", new String[] {"printall"});
2316
2317
						            transformer.transformXMLDocument(
2318
						                    documentContent ,
2319
						                    sourceType,
2320
						                    targetType ,
2321
						                    format,
2322
						                    writer,
2323
						                    params,
2324
						                    null //sessionid
2325
						                    );
2326
2327
						            // finally, get the HTML back
2328
						            ContentTypeByteArrayInputStream resultInputStream = new ContentTypeByteArrayInputStream(baos.toByteArray());
2329
2330
						            // write to temp file with correct css path
2331
						            File tmpDir = File.createTempFile("package_", "_dir");
2332
						            tmpDir.delete();
2333
						            tmpDir.mkdir();
2334
						            File htmlFile = File.createTempFile("metadata", ".html", tmpDir);
2335
						            File cssDir = new File(tmpDir, format);
2336
						            cssDir.mkdir();
2337
						            File cssFile = new File(tmpDir, format + "/" + format + ".css");
2338
						            String pdfFileName = metadataID.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-METADATA.pdf";
2339
						            File pdfFile = new File(tmpDir, pdfFileName);
2340
						            //File pdfFile = File.createTempFile("metadata", ".pdf", tmpDir);
2341
2342
						            // put the CSS file in place for the html to find it
2343
						            String originalCssPath = SystemUtil.getContextDir() + "/style/skins/" + format + "/" + format + ".css";
2344
						            IOUtils.copy(new FileInputStream(originalCssPath), new FileOutputStream(cssFile));
2345
2346
						            // write the HTML file
2347
						            IOUtils.copy(resultInputStream, new FileOutputStream(htmlFile));
2348
2349
						            // convert to PDF
2350
						            HtmlToPdf.export(htmlFile.getAbsolutePath(), pdfFile.getAbsolutePath());
2351
2352
						            //add to the package
2353
						            bag.addFileToPayload(pdfFile);
2354
									pidMapping.append(metadataID.getValue() + " (pdf)" +  "\t" + "data/" + pdfFile.getName() + "\n");
2355
2356
						            // mark for clean up after we are done
2357
									htmlFile.delete();
2358
									cssFile.delete();
2359
									cssDir.delete();
2360
						            tempFiles.add(tmpDir);
2361
									tempFiles.add(pdfFile); // delete this first later on
2362
2363
								} catch (Exception e) {
2364
									logMetacat.warn("Could not transform metadata", e);
2365
								}
2366
							}
2367
2368
2369 8437 walker
							//If this is in eml format, extract the filename and GUID from each entity in its package
2370
							if (metadataSysMeta.getFormatId().getValue().startsWith("eml://")) {
2371
								//Get the package
2372
								DataPackageParserInterface parser = new Eml200DataPackageParser();
2373
								InputStream emlStream = this.get(session, metadataID);
2374
								parser.parse(emlStream);
2375
								DataPackage dataPackage = parser.getDataPackage();
2376
2377
								//Get all the entities in this package and loop through each to extract its ID and file name
2378
								Entity[] entities = dataPackage.getEntityList();
2379
								for(Entity entity: entities){
2380
									try{
2381
										//Get the file name from the metadata
2382
										String fileNameFromMetadata = entity.getName();
2383
2384
										//Get the ecogrid URL from the metadata
2385
										String ecogridIdentifier = entity.getEntityIdentifier();
2386
										//Parse the ecogrid URL to get the local id
2387
										String idFromMetadata = DocumentUtil.getAccessionNumberFromEcogridIdentifier(ecogridIdentifier);
2388
2389
										//Get the docid and rev pair
2390
										String docid = DocumentUtil.getDocIdFromString(idFromMetadata);
2391
										String rev = DocumentUtil.getRevisionStringFromString(idFromMetadata);
2392
2393
										//Get the GUID
2394
										String guid = IdentifierManager.getInstance().getGUID(docid, Integer.valueOf(rev));
2395
										Identifier dataIdentifier = new Identifier();
2396
										dataIdentifier.setValue(guid);
2397
2398
										//Add the GUID to our GUID & file name map
2399
										fileNames.put(dataIdentifier, fileNameFromMetadata);
2400
									}
2401
									catch(Exception e){
2402
										//Prevent just one entity error
2403
										e.printStackTrace();
2404
										logMetacat.debug(e.getMessage(), e);
2405
									}
2406
								}
2407
							}
2408
						}
2409
						catch(Exception e){
2410
							//Catch errors that would prevent package download
2411
							logMetacat.debug(e.toString());
2412
						}
2413
					}
2414 7855 leinfelder
					packagePids.addAll(entries.keySet());
2415
					for (List<Identifier> dataPids: entries.values()) {
2416
						packagePids.addAll(dataPids);
2417
					}
2418 7850 leinfelder
				}
2419 7855 leinfelder
			} else {
2420
				// just the lone pid in this package
2421 9493 tao
				//packagePids.add(pid);
2422
			    //throw an invalid request exception
2423 9494 tao
			    throw new InvalidRequest("2873", "The given pid "+pid.getValue()+" is not a package id (resource map id). Please use a package id instead.");
2424 7850 leinfelder
			}
2425 8436 walker
2426 8437 walker
			//Create a temp file, then delete it and make a directory with that name
2427
			File tempDir = File.createTempFile("temp", Long.toString(System.nanoTime()));
2428
			tempDir.delete();
2429
			tempDir = new File(tempDir.getPath() + "_dir");
2430
			tempDir.mkdir();
2431 8436 walker
			tempFiles.add(tempDir);
2432 8800 leinfelder
			File pidMappingFile = new File(tempDir, "pid-mapping.txt");
2433 8436 walker
2434 7855 leinfelder
			// loop through the package contents
2435
			for (Identifier entryPid: packagePids) {
2436 8436 walker
				//Get the system metadata for each item
2437 8437 walker
				SystemMetadata entrySysMeta = this.getSystemMetadata(session, entryPid);
2438 8436 walker
2439
				String objectFormatType = ObjectFormatCache.getInstance().getFormat(entrySysMeta.getFormatId()).getFormatType();
2440 8437 walker
				String fileName = null;
2441 8436 walker
2442 8437 walker
				//TODO: Be more specific of what characters to replace. Make sure periods arent replaced for the filename from metadata
2443
				//Our default file name is just the ID + format type (e.g. walker.1.1-DATA)
2444
				fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + objectFormatType;
2445
2446 9542 leinfelder
				if (fileNames.containsKey(entryPid)){
2447 8437 walker
					//Let's use the file name and extension from the metadata is we have it
2448
					fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + fileNames.get(entryPid).replaceAll("[^a-zA-Z0-9\\-\\.]", "_");
2449
				}
2450
2451 9541 leinfelder
				// ensure there is a file extension for the object
2452
				String extension = ObjectFormatInfo.instance().getExtension(entrySysMeta.getFormatId().getValue());
2453
				fileName += extension;
2454
2455 9542 leinfelder
				// if SM has the file name, ignore everything else and use that
2456
				if (entrySysMeta.getFileName() != null) {
2457
					fileName = entrySysMeta.getFileName().replaceAll("[^a-zA-Z0-9\\-\\.]", "_");
2458
				}
2459
2460 8436 walker
		        //Create a new file for this item and add to the list
2461 8437 walker
				File tempFile = new File(tempDir, fileName);
2462 7855 leinfelder
				tempFiles.add(tempFile);
2463 8436 walker
2464
				InputStream entryInputStream = this.get(session, entryPid);
2465 7855 leinfelder
				IOUtils.copy(entryInputStream, new FileOutputStream(tempFile));
2466
				bag.addFileToPayload(tempFile);
2467
				pidMapping.append(entryPid.getValue() + "\t" + "data/" + tempFile.getName() + "\n");
2468
			}
2469
2470
			//add the the pid to data file map
2471
			IOUtils.write(pidMapping.toString(), new FileOutputStream(pidMappingFile));
2472
			bag.addFileAsTag(pidMappingFile);
2473 8026 leinfelder
			tempFiles.add(pidMappingFile);
2474
2475 7855 leinfelder
			bag = bag.makeComplete();
2476 8026 leinfelder
2477 8436 walker
			///Now create the zip file
2478 8437 walker
			//Use the pid as the file name prefix, replacing all non-word characters
2479
			String zipName = pid.getValue().replaceAll("\\W", "_");
2480 8160 leinfelder
2481 8436 walker
			File bagFile = new File(tempDir, zipName+".zip");
2482
2483 7855 leinfelder
			bag.setFile(bagFile);
2484 7860 leinfelder
			ZipWriter zipWriter = new ZipWriter(bagFactory);
2485 7855 leinfelder
			bag.write(zipWriter, bagFile);
2486
			bagFile = bag.getFile();
2487 8026 leinfelder
			// use custom FIS that will delete the file when closed
2488
			bagInputStream = new DeleteOnCloseFileInputStream(bagFile);
2489
			// also mark for deletion on shutdown in case the stream is never closed
2490
			bagFile.deleteOnExit();
2491 8436 walker
			tempFiles.add(bagFile);
2492 7855 leinfelder
2493 8026 leinfelder
			// clean up other temp files
2494 8800 leinfelder
			for (int i=tempFiles.size()-1; i>=0; i--){
2495
				tempFiles.get(i).delete();
2496 7855 leinfelder
			}
2497 8436 walker
2498 7855 leinfelder
		} catch (IOException e) {
2499
			// report as service failure
2500 9538 tao
		    e.printStackTrace();
2501 7855 leinfelder
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2502
			sf.initCause(e);
2503
			throw sf;
2504
		} catch (OREException e) {
2505
			// report as service failure
2506 9538 tao
		    e.printStackTrace();
2507 7855 leinfelder
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2508
			sf.initCause(e);
2509
			throw sf;
2510
		} catch (URISyntaxException e) {
2511
			// report as service failure
2512 9538 tao
		    e.printStackTrace();
2513 7855 leinfelder
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2514
			sf.initCause(e);
2515
			throw sf;
2516
		} catch (OREParserException e) {
2517
			// report as service failure
2518 9538 tao
		    e.printStackTrace();
2519 7855 leinfelder
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2520
			sf.initCause(e);
2521
			throw sf;
2522 7850 leinfelder
		}
2523
2524
		return bagInputStream;
2525 8810 leinfelder
	}
2526 9353 tao
2527
	 /**
2528
	   * Archives an object, where the object is either a
2529
	   * data object or a science metadata object.
2530
	   *
2531
	   * @param session - the Session object containing the credentials for the Subject
2532
	   * @param pid - The object identifier to be archived
2533
	   *
2534
	   * @return pid - the identifier of the object used for the archiving
2535
	   *
2536
	   * @throws InvalidToken
2537
	   * @throws ServiceFailure
2538
	   * @throws NotAuthorized
2539
	   * @throws NotFound
2540
	   * @throws NotImplemented
2541
	   * @throws InvalidRequest
2542
	   */
2543
	  public Identifier archive(Session session, Identifier pid)
2544
	      throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
2545 9478 tao
	      if(isReadOnlyMode()) {
2546 9481 tao
	            throw new ServiceFailure("2912", ReadOnlyChecker.DATAONEERROR);
2547 9478 tao
	        }
2548 9353 tao
	      boolean allowed = false;
2549
	      // do we have a valid pid?
2550
	      if (pid == null || pid.getValue().trim().equals("")) {
2551
	          throw new ServiceFailure("1350", "The provided identifier was invalid.");
2552
	      }
2553
2554
	      String serviceFailureCode = "1350";
2555
	      Identifier sid = getPIDForSID(pid, serviceFailureCode);
2556
	      if(sid != null) {
2557
	          pid = sid;
2558
	      }
2559
	      // does the subject have archive (a D1 CHANGE_PERMISSION level) privileges on the pid?
2560
	      try {
2561
	            allowed = isAuthorized(session, pid, Permission.CHANGE_PERMISSION);
2562
	        } catch (InvalidRequest e) {
2563
	          throw new ServiceFailure("1350", e.getDescription());
2564
	        }
2565
2566
	      if (allowed) {
2567
	         try {
2568
	             HazelcastService.getInstance().getSystemMetadataMap().lock(pid);
2569
	             logMetacat.debug("MNodeService.archive - lock the identifier "+pid.getValue()+" in the system metadata map.");
2570
	             SystemMetadata sysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
2571 9354 tao
	             boolean needModifyDate = true;
2572 9373 tao
	             boolean logArchive = true;
2573
	             super.archiveObject(logArchive, session, pid, sysmeta, needModifyDate);
2574 9353 tao
	         } finally {
2575
	             HazelcastService.getInstance().getSystemMetadataMap().unlock(pid);
2576
	             logMetacat.debug("MNodeService.archive - unlock the identifier "+pid.getValue()+" in the system metadata map.");
2577
	         }
2578
2579
2580
	      } else {
2581
	          throw new NotAuthorized("1320", "The provided identity does not have " + "permission to archive the object on the Node.");
2582
	      }
2583
2584
	      return pid;
2585
	  }
2586 9190 tao
2587
	/**
2588
	 * Update the system metadata of the specified pid.
2589
	 */
2590 9177 tao
	@Override
2591
	public boolean updateSystemMetadata(Session session, Identifier pid,
2592
            SystemMetadata sysmeta) throws NotImplemented, NotAuthorized,
2593
            ServiceFailure, InvalidRequest, InvalidSystemMetadata, InvalidToken {
2594 9478 tao
2595
	  if(isReadOnlyMode()) {
2596 9481 tao
            throw new ServiceFailure("4868",  ReadOnlyChecker.DATAONEERROR);
2597 9478 tao
      }
2598 9177 tao
	 if(sysmeta == null) {
2599 9478 tao
	     throw  new InvalidRequest("4869", "The system metadata object should NOT be null in the updateSystemMetadata request.");
2600 9177 tao
	 }
2601 9190 tao
	 if(pid == null || pid.getValue() == null) {
2602 9478 tao
         throw new InvalidRequest("4869", "Please specify the id in the updateSystemMetadata request ") ;
2603 9190 tao
     }
2604
2605
     if (session == null) {
2606
         //TODO: many of the thrown exceptions do not use the correct error codes
2607
         //check these against the docs and correct them
2608
         throw new NotAuthorized("4861", "No Session - could not authorize for updating system metadata." +
2609
                 "  If you are not logged in, please do so and retry the request.");
2610
     } else {
2611
         try {
2612
             //Following session can do the change:
2613 9263 tao
           //- Authoritative Member Node (we can use isNodeAdmin since we checked isAuthoritativeNode )
2614 9190 tao
             //- Owner of object (coved by the userHasPermission method)
2615
             //- user subjects with the change permission
2616
             //Note: Coordinating Node can not because MN is authoritative
2617 9263 tao
             /*if(!isAuthoritativeNode(pid)) {
2618
                throw  new InvalidRequest("4863", "Client can only call updateSystemMetadata request on the authoritative memember node.");
2619
             }
2620 9190 tao
             if(!isNodeAdmin(session) && !userHasPermission(session, pid, Permission.CHANGE_PERMISSION)) {
2621
                 throw new NotAuthorized("4861", "The client -"+ session.getSubject().getValue()+ "is not authorized for updating the system metadata of the object "+pid.getValue());
2622 9263 tao
             }*/
2623
             if(!allowUpdating(session, pid, Permission.CHANGE_PERMISSION)) {
2624
                 throw new NotAuthorized("4861", "The client -"+ session.getSubject().getValue()+ "is not authorized for updating the system metadata of the object "+pid.getValue());
2625 9190 tao
             }
2626
         } catch (NotFound e) {
2627 9478 tao
             throw new InvalidRequest("4869", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2628 9997 tao
         } catch(ServiceFailure e) {
2629
             throw new ServiceFailure("4868", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2630
         } catch(NotAuthorized e) {
2631
             throw new NotAuthorized("4861", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2632
         } catch(NotImplemented e) {
2633
             throw new NotImplemented("4866","Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2634
         } catch(InvalidRequest e) {
2635
             throw new InvalidRequest("4869", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2636
         } catch(InvalidToken e) {
2637
             throw new InvalidToken("4957", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2638 9190 tao
         }
2639
2640
     }
2641 9335 tao
      //update the system metadata locally
2642
      boolean success = false;
2643
      try {
2644
          HazelcastService.getInstance().getSystemMetadataMap().lock(pid);
2645
          SystemMetadata currentSysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
2646
          if(currentSysmeta == null) {
2647 9478 tao
              throw  new InvalidRequest("4869", "We can't find the current system metadata on the member node for the id "+pid.getValue());
2648 9335 tao
          }
2649
          Date currentModiDate = currentSysmeta.getDateSysMetadataModified();
2650
          Date commingModiDate = sysmeta.getDateSysMetadataModified();
2651
          if(commingModiDate == null) {
2652 9478 tao
              throw  new InvalidRequest("4869", "The system metadata modification date can't be null.");
2653 9335 tao
          }
2654
          if(currentModiDate != null && commingModiDate.getTime() != currentModiDate.getTime()) {
2655 9478 tao
              throw new InvalidRequest("4869", "Your system metadata modification date is "+commingModiDate.toString()+
2656 9335 tao
                      ". It doesn't match our current system metadata modification date in the member node - "+currentModiDate.toString()+
2657
                      ". Please check if you have got the newest version of the system metadata before the modification.");
2658
          }
2659
          boolean needUpdateModificationDate = true;
2660 9354 tao
          boolean fromCN = false;
2661
          success = updateSystemMetadata(session, pid, sysmeta, needUpdateModificationDate, currentSysmeta, fromCN);
2662 9335 tao
      } finally {
2663
          HazelcastService.getInstance().getSystemMetadataMap().unlock(pid);
2664
      }
2665 9177 tao
2666
      if(success) {
2667 10102 tao
          this.cn = D1Client.getCN();
2668 9190 tao
          //TODO
2669 9177 tao
          //notify the cns the synchornize the new system metadata.
2670 10102 tao
          // run it in a thread to avoid connection timeout
2671
          Runnable runner = new Runnable() {
2672
              private CNode cNode = null;
2673
              private SystemMetadata sys = null;
2674
              private Identifier id = null;
2675
              @Override
2676
              public void run() {
2677
                  try {
2678
                      if(this.cNode == null)  {
2679
                          logMetacat.warn("MNodeService.updateSystemMetadata - can't get the instance of the CN. So can't call cn.synchronize to update the system metadata in CN.");
2680
                      } else if(id != null) {
2681
                          logMetacat.info("MNodeService.updateSystemMetadata - calling cn.synchornized in another thread for pid "+id.getValue());
2682
                          this.cNode.synchronize(null, id);
2683
                      } else {
2684 10104 tao
                          logMetacat.warn("MNodeService.updateSystemMetadata - the pid is null. So can't call cn.synchronize to update the system metadata in CN.");
2685 10102 tao
                      }
2686
                  } catch (BaseException e) {
2687
                      e.printStackTrace();
2688
                      logMetacat.error("It is a DataONEBaseException and its detail code is "+e.getDetail_code() +" and its code is "+e.getCode());
2689
                      logMetacat.error("Can't update the systemmetadata of pid "+id.getValue()+" in CNs through cn.synchronize method since "+e.getMessage(), e);
2690
                  } catch (Exception e) {
2691
                      e.printStackTrace();
2692
                      logMetacat.error("Can't update the systemmetadata of pid "+id.getValue()+" in CNs through cn.synchronize method since "+e.getMessage(), e);
2693
                  }
2694
2695
                  // attempt to re-register the identifier (it checks if it is a doi)
2696
                  try {
2697
                      logMetacat.info("MNodeSerice.updateSystemMetadata - register doi if the pid "+sys.getIdentifier().getValue()+" is a doi");
2698
                      DOIService.getInstance().registerDOI(sys);
2699
                  } catch (Exception e) {
2700
                    logMetacat.warn("Could not [re]register DOI: " + e.getMessage(), e);
2701
                  }
2702 9256 tao
              }
2703 10102 tao
              private Runnable init(CNode cn, SystemMetadata sys, Identifier id){
2704
                  this.cNode = cn;
2705
                  this.sys = sys;
2706
                  this.id = id;
2707
                  return this;
2708
2709
              }
2710
          }.init(cn, sysmeta, pid);
2711
          // submit the task, and that's it
2712 10104 tao
          if(executor != null) {
2713
              executor.submit(runner);
2714 10102 tao
          } else {
2715 10104 tao
              logMetacat.warn("MNodeSerivce.updateSystemMetadata - since the executor service for submitting the call of cn.synchronize() is null, the system metadata change of the id "+pid.getValue()+" can't go to cn through cn.synchronize.");
2716 9246 tao
          }
2717 9177 tao
      }
2718
      return success;
2719
    }
2720 9190 tao
2721
	/*
2722
     * Determine if the current node is the authoritative node for the given pid.
2723
     */
2724 9374 tao
    protected boolean isAuthoritativeNode(Identifier pid) throws InvalidRequest {
2725 9190 tao
        boolean isAuthoritativeNode = false;
2726
        if(pid != null && pid.getValue() != null) {
2727
            SystemMetadata sys = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
2728
            if(sys != null) {
2729
                NodeReference node = sys.getAuthoritativeMemberNode();
2730
                if(node != null) {
2731
                    String nodeValue = node.getValue();
2732
                    logMetacat.debug("The authoritative node for id "+pid.getValue()+" is "+nodeValue);
2733
                    //System.out.println("The authoritative node for id "+pid.getValue()+" is "+nodeValue);
2734
                    String currentNodeId = Settings.getConfiguration().getString("dataone.nodeId");
2735
                    logMetacat.debug("The node id in metacat.properties is "+currentNodeId);
2736
                    //System.out.println("The node id in metacat.properties is "+currentNodeId);
2737
                    if(currentNodeId != null && !currentNodeId.trim().equals("") && currentNodeId.equals(nodeValue)) {
2738 10021 tao
                        logMetacat.debug("They are matching, so the authoritative mn of the object "+pid.getValue()+" is the current node");
2739 9190 tao
                        //System.out.println("They are matching");
2740
                        isAuthoritativeNode = true;
2741
                    }
2742 9374 tao
                } else {
2743
                    throw new InvalidRequest("4869", "Coudn't find the authoritative member node in the system metadata associated with the pid "+pid.getValue());
2744 9190 tao
                }
2745 9374 tao
            } else {
2746
                throw new InvalidRequest("4869", "Coudn't find the system metadata associated with the pid "+pid.getValue());
2747 9190 tao
            }
2748 9374 tao
        } else {
2749
            throw new InvalidRequest("4869", "The request pid is null");
2750 9190 tao
        }
2751
        return isAuthoritativeNode;
2752
    }
2753 6795 cjones
2754 9263 tao
    /*
2755
     * Rules are:
2756
     * 1. If the session has an cn object, it is allowed.
2757
     * 2. If it is not a cn object, the client should have approperate permission and it should also happen on the authorative node.
2758 9700 cjones
     * 3. If it's the authoritative node, the MN Admin Subject is allowed.
2759 9263 tao
     */
2760 9997 tao
    private boolean allowUpdating(Session session, Identifier pid, Permission permission) throws NotAuthorized, NotFound, InvalidRequest, ServiceFailure, NotImplemented, InvalidToken {
2761 9263 tao
        boolean allow = false;
2762 9781 cjones
2763 9700 cjones
        if( isCNAdmin (session) ) {
2764 9263 tao
            allow = true;
2765 9700 cjones
2766 9263 tao
        } else {
2767 9700 cjones
            if( isAuthoritativeNode(pid) ) {
2768
2769 9781 cjones
            	// Check for admin authorization
2770 9700 cjones
            	try {
2771 9781 cjones
					allow = isNodeAdmin(session);
2772 9700 cjones
2773
				} catch (NotImplemented e) {
2774
					logMetacat.debug("Failed to authorize the Member Node Admin Subject: " + e.getMessage());
2775
2776
				} catch (ServiceFailure e) {
2777
					logMetacat.debug("Failed to authorize the Member Node Admin Subject: " + e.getMessage());
2778
2779
				}
2780
2781 9781 cjones
            	// Check for user authorization
2782
            	if ( !allow ) {
2783
                    allow = userHasPermission(session, pid, permission);
2784 9700 cjones
2785 9781 cjones
            	}
2786 9700 cjones
2787 9263 tao
            } else {
2788 10021 tao
                throw new NotAuthorized("4861", "Client can only call the request on the authoritative memember node of the object "+pid.getValue());
2789 9700 cjones
2790 9263 tao
            }
2791
        }
2792
        return allow;
2793
2794
    }
2795
2796 9478 tao
    /**
2797
     * Check if the metacat is in the read-only mode.
2798
     * @return true if it is; otherwise false.
2799
     */
2800
    protected boolean isReadOnlyMode() {
2801
        boolean readOnly = false;
2802
        ReadOnlyChecker checker = new ReadOnlyChecker();
2803
        readOnly = checker.isReadOnly();
2804
        return readOnly;
2805
    }
2806
2807 6179 cjones
}