Project

General

Profile

1 6179 cjones
/**
2
 *  '$RCSfile$'
3
 *  Copyright: 2000-2011 Regents of the University of California and the
4
 *              National Center for Ecological Analysis and Synthesis
5
 *
6
 *   '$Author:  $'
7
 *     '$Date:  $'
8
 *
9
 * This program is free software; you can redistribute it and/or modify
10
 * it under the terms of the GNU General Public License as published by
11
 * the Free Software Foundation; either version 2 of the License, or
12
 * (at your option) any later version.
13
 *
14
 * This program is distributed in the hope that it will be useful,
15
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 * GNU General Public License for more details.
18
 *
19
 * You should have received a copy of the GNU General Public License
20
 * along with this program; if not, write to the Free Software
21
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22
 */
23
24
package edu.ucsb.nceas.metacat.dataone;
25
26 7417 leinfelder
import java.io.ByteArrayInputStream;
27 7860 leinfelder
import java.io.ByteArrayOutputStream;
28 7850 leinfelder
import java.io.File;
29 8800 leinfelder
import java.io.FileInputStream;
30 7850 leinfelder
import java.io.FileOutputStream;
31 6228 cjones
import java.io.IOException;
32 6179 cjones
import java.io.InputStream;
33 8190 leinfelder
import java.io.InputStreamReader;
34 7860 leinfelder
import java.io.OutputStreamWriter;
35 9387 walker
import java.io.StringReader;
36 7849 leinfelder
import java.io.UnsupportedEncodingException;
37 7860 leinfelder
import java.io.Writer;
38 7021 leinfelder
import java.math.BigInteger;
39 7849 leinfelder
import java.net.URISyntaxException;
40 9352 walker
import java.nio.charset.Charset;
41 6228 cjones
import java.security.NoSuchAlgorithmException;
42 7860 leinfelder
import java.sql.SQLException;
43 7417 leinfelder
import java.util.ArrayList;
44 6525 leinfelder
import java.util.Calendar;
45 6179 cjones
import java.util.Date;
46 8437 walker
import java.util.HashMap;
47 7680 tao
import java.util.HashSet;
48 7860 leinfelder
import java.util.Hashtable;
49 6250 cjones
import java.util.List;
50 7849 leinfelder
import java.util.Map;
51 7417 leinfelder
import java.util.Set;
52 6389 leinfelder
import java.util.Timer;
53 7489 leinfelder
import java.util.UUID;
54 7418 leinfelder
import java.util.Vector;
55 10104 tao
import java.util.concurrent.ExecutorService;
56
import java.util.concurrent.Executors;
57 6179 cjones
58 6542 leinfelder
import javax.servlet.http.HttpServletRequest;
59 9387 walker
import javax.xml.transform.Result;
60
import javax.xml.transform.Source;
61
import javax.xml.transform.TransformerException;
62
import javax.xml.transform.TransformerFactory;
63
import javax.xml.transform.dom.DOMSource;
64
import javax.xml.transform.stream.StreamResult;
65 6542 leinfelder
66 6258 cjones
import org.apache.commons.io.IOUtils;
67 9997 tao
import org.apache.commons.lang.NotImplementedException;
68 6179 cjones
import org.apache.log4j.Logger;
69 8810 leinfelder
import org.dataone.client.v2.CNode;
70
import org.dataone.client.v2.itk.D1Client;
71
import org.dataone.client.v2.MNode;
72
import org.dataone.client.v2.formats.ObjectFormatCache;
73 6552 leinfelder
import org.dataone.client.auth.CertificateManager;
74 8810 leinfelder
import org.dataone.client.v2.formats.ObjectFormatInfo;
75 6552 leinfelder
import org.dataone.configuration.Settings;
76 7849 leinfelder
import org.dataone.ore.ResourceMapFactory;
77 6795 cjones
import org.dataone.service.exceptions.BaseException;
78 6179 cjones
import org.dataone.service.exceptions.IdentifierNotUnique;
79
import org.dataone.service.exceptions.InsufficientResources;
80
import org.dataone.service.exceptions.InvalidRequest;
81
import org.dataone.service.exceptions.InvalidSystemMetadata;
82
import org.dataone.service.exceptions.InvalidToken;
83
import org.dataone.service.exceptions.NotAuthorized;
84
import org.dataone.service.exceptions.NotFound;
85
import org.dataone.service.exceptions.NotImplemented;
86
import org.dataone.service.exceptions.ServiceFailure;
87 6185 leinfelder
import org.dataone.service.exceptions.SynchronizationFailed;
88 6179 cjones
import org.dataone.service.exceptions.UnsupportedType;
89 8810 leinfelder
import org.dataone.service.mn.tier1.v2.MNCore;
90
import org.dataone.service.mn.tier1.v2.MNRead;
91
import org.dataone.service.mn.tier2.v2.MNAuthorization;
92
import org.dataone.service.mn.tier3.v2.MNStorage;
93
import org.dataone.service.mn.tier4.v2.MNReplication;
94
import org.dataone.service.mn.v2.MNPackage;
95
import org.dataone.service.mn.v2.MNQuery;
96
import org.dataone.service.mn.v2.MNView;
97 8591 leinfelder
import org.dataone.service.types.v1.AccessRule;
98 6366 leinfelder
import org.dataone.service.types.v1.Checksum;
99 7144 leinfelder
import org.dataone.service.types.v1.DescribeResponse;
100 6366 leinfelder
import org.dataone.service.types.v1.Event;
101
import org.dataone.service.types.v1.Identifier;
102 8810 leinfelder
import org.dataone.service.types.v2.Log;
103
import org.dataone.service.types.v2.LogEntry;
104
import org.dataone.service.types.v2.OptionList;
105 6366 leinfelder
import org.dataone.service.types.v1.MonitorInfo;
106
import org.dataone.service.types.v1.MonitorList;
107 8810 leinfelder
import org.dataone.service.types.v2.Node;
108
import org.dataone.service.types.v2.NodeList;
109 6366 leinfelder
import org.dataone.service.types.v1.NodeReference;
110
import org.dataone.service.types.v1.NodeState;
111
import org.dataone.service.types.v1.NodeType;
112 8810 leinfelder
import org.dataone.service.types.v2.ObjectFormat;
113 9353 tao
import org.dataone.service.types.v1.Group;
114 6366 leinfelder
import org.dataone.service.types.v1.ObjectFormatIdentifier;
115
import org.dataone.service.types.v1.ObjectList;
116
import org.dataone.service.types.v1.Permission;
117
import org.dataone.service.types.v1.Ping;
118 9340 tao
import org.dataone.service.types.v1.Replica;
119 6528 cjones
import org.dataone.service.types.v1.ReplicationStatus;
120 6366 leinfelder
import org.dataone.service.types.v1.Schedule;
121
import org.dataone.service.types.v1.Service;
122
import org.dataone.service.types.v1.Services;
123
import org.dataone.service.types.v1.Session;
124
import org.dataone.service.types.v1.Subject;
125
import org.dataone.service.types.v1.Synchronization;
126 8810 leinfelder
import org.dataone.service.types.v2.SystemMetadata;
127 7417 leinfelder
import org.dataone.service.types.v1.util.AuthUtils;
128 6366 leinfelder
import org.dataone.service.types.v1.util.ChecksumUtil;
129 7417 leinfelder
import org.dataone.service.types.v1_1.QueryEngineDescription;
130
import org.dataone.service.types.v1_1.QueryEngineList;
131 7418 leinfelder
import org.dataone.service.types.v1_1.QueryField;
132 6476 jones
import org.dataone.service.util.Constants;
133 8594 leinfelder
import org.dataone.service.util.TypeMarshaller;
134 7849 leinfelder
import org.dspace.foresite.OREException;
135
import org.dspace.foresite.OREParserException;
136
import org.dspace.foresite.ORESerialiserException;
137
import org.dspace.foresite.ResourceMap;
138 8437 walker
import org.ecoinformatics.datamanager.parser.DataPackage;
139
import org.ecoinformatics.datamanager.parser.Entity;
140
import org.ecoinformatics.datamanager.parser.generic.DataPackageParserInterface;
141
import org.ecoinformatics.datamanager.parser.generic.Eml200DataPackageParser;
142 9387 walker
import org.w3c.dom.Document;
143 6179 cjones
144 7448 leinfelder
import edu.ucsb.nceas.ezid.EZIDException;
145 7417 leinfelder
import edu.ucsb.nceas.metacat.DBQuery;
146 7860 leinfelder
import edu.ucsb.nceas.metacat.DBTransform;
147 6234 cjones
import edu.ucsb.nceas.metacat.EventLog;
148 6230 cjones
import edu.ucsb.nceas.metacat.IdentifierManager;
149 6234 cjones
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
150 7417 leinfelder
import edu.ucsb.nceas.metacat.MetaCatServlet;
151 6389 leinfelder
import edu.ucsb.nceas.metacat.MetacatHandler;
152 9478 tao
import edu.ucsb.nceas.metacat.ReadOnlyChecker;
153 7772 tao
import edu.ucsb.nceas.metacat.common.query.EnabledQueryEngines;
154 7757 leinfelder
import edu.ucsb.nceas.metacat.common.query.stream.ContentTypeByteArrayInputStream;
155 6648 leinfelder
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;
156 7662 tao
import edu.ucsb.nceas.metacat.index.MetacatSolrEngineDescriptionHandler;
157 7620 tao
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;
158 10172 leinfelder
import edu.ucsb.nceas.metacat.mdq.MDQClient;
159 6340 cjones
import edu.ucsb.nceas.metacat.properties.PropertyService;
160 7418 leinfelder
import edu.ucsb.nceas.metacat.shared.MetacatUtilException;
161 8026 leinfelder
import edu.ucsb.nceas.metacat.util.DeleteOnCloseFileInputStream;
162 7441 leinfelder
import edu.ucsb.nceas.metacat.util.DocumentUtil;
163 8810 leinfelder
import edu.ucsb.nceas.metacat.util.SkinUtil;
164 6542 leinfelder
import edu.ucsb.nceas.metacat.util.SystemUtil;
165 6340 cjones
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
166 8190 leinfelder
import edu.ucsb.nceas.utilities.XMLUtilities;
167 8800 leinfelder
import edu.ucsb.nceas.utilities.export.HtmlToPdf;
168 7850 leinfelder
import gov.loc.repository.bagit.Bag;
169
import gov.loc.repository.bagit.BagFactory;
170
import gov.loc.repository.bagit.writer.impl.ZipWriter;
171 6230 cjones
172 6179 cjones
/**
173
 * Represents Metacat's implementation of the DataONE Member Node
174
 * service API. Methods implement the various MN* interfaces, and methods common
175
 * to both Member Node and Coordinating Node interfaces are found in the
176
 * D1NodeService base class.
177 6288 cjones
 *
178
 * Implements:
179
 * MNCore.ping()
180
 * MNCore.getLogRecords()
181
 * MNCore.getObjectStatistics()
182
 * MNCore.getOperationStatistics()
183
 * MNCore.getStatus()
184
 * MNCore.getCapabilities()
185
 * MNRead.get()
186
 * MNRead.getSystemMetadata()
187
 * MNRead.describe()
188
 * MNRead.getChecksum()
189
 * MNRead.listObjects()
190
 * MNRead.synchronizationFailed()
191
 * MNAuthorization.isAuthorized()
192
 * MNAuthorization.setAccessPolicy()
193
 * MNStorage.create()
194
 * MNStorage.update()
195
 * MNStorage.delete()
196 9177 tao
 * MNStorage.updateSystemMetadata()
197 6288 cjones
 * MNReplication.replicate()
198
 *
199 6179 cjones
 */
200 6599 cjones
public class MNodeService extends D1NodeService
201 8810 leinfelder
    implements MNAuthorization, MNCore, MNRead, MNReplication, MNStorage, MNQuery, MNView, MNPackage {
202 6179 cjones
203 7772 tao
    //private static final String PATHQUERY = "pathquery";
204 7849 leinfelder
	public static final String UUID_SCHEME = "UUID";
205
	public static final String DOI_SCHEME = "DOI";
206 7489 leinfelder
	private static final String UUID_PREFIX = "urn:uuid:";
207 9387 walker
208
	private static String XPATH_EML_ID = "/eml:eml/@packageId";
209 7418 leinfelder
210
	/* the logger instance */
211 6475 jones
    private Logger logMetacat = null;
212 6795 cjones
213
    /* A reference to a remote Memeber Node */
214 9295 tao
    //private MNode mn;
215 6795 cjones
216
    /* A reference to a Coordinating Node */
217
    private CNode cn;
218 10104 tao
219
    // shared executor
220
    private static ExecutorService executor = null;
221 6241 cjones
222 10104 tao
    static {
223
        // use a shared executor service with nThreads == one less than available processors
224
        int availableProcessors = Runtime.getRuntime().availableProcessors();
225
        int nThreads = availableProcessors * 1;
226
        nThreads--;
227
        nThreads = Math.max(1, nThreads);
228
        executor = Executors.newFixedThreadPool(nThreads);
229
    }
230 6795 cjones
231 10104 tao
232 6475 jones
    /**
233
     * Singleton accessor to get an instance of MNodeService.
234
     *
235
     * @return instance - the instance of MNodeService
236
     */
237 6542 leinfelder
    public static MNodeService getInstance(HttpServletRequest request) {
238
        return new MNodeService(request);
239 6179 cjones
    }
240
241 6475 jones
    /**
242
     * Constructor, private for singleton access
243
     */
244 6542 leinfelder
    private MNodeService(HttpServletRequest request) {
245
        super(request);
246 6475 jones
        logMetacat = Logger.getLogger(MNodeService.class);
247 6552 leinfelder
248
        // set the Member Node certificate file location
249
        CertificateManager.getInstance().setCertificateLocation(Settings.getConfiguration().getString("D1Client.certificate.file"));
250 6310 cjones
    }
251 6475 jones
252
    /**
253
     * Deletes an object from the Member Node, where the object is either a
254
     * data object or a science metadata object.
255
     *
256
     * @param session - the Session object containing the credentials for the Subject
257
     * @param pid - The object identifier to be deleted
258
     *
259
     * @return pid - the identifier of the object used for the deletion
260
     *
261
     * @throws InvalidToken
262
     * @throws ServiceFailure
263
     * @throws NotAuthorized
264
     * @throws NotFound
265
     * @throws NotImplemented
266
     * @throws InvalidRequest
267
     */
268
    @Override
269
    public Identifier delete(Session session, Identifier pid)
270 6610 cjones
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
271 6475 jones
272 9478 tao
        if(isReadOnlyMode()) {
273 9481 tao
            throw new ServiceFailure("2902", ReadOnlyChecker.DATAONEERROR);
274 9478 tao
        }
275 7162 leinfelder
    	// only admin of  the MN or the CN is allowed a full delete
276
        boolean allowed = false;
277 7330 leinfelder
        allowed = isAdminAuthorized(session);
278 8360 tao
279 9050 tao
        String serviceFailureCode = "2902";
280
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
281
        if(sid != null) {
282
            pid = sid;
283
        }
284
285 8360 tao
        //check if it is the authoritative member node
286
        if(!allowed) {
287
            allowed = isAuthoritativeMNodeAdmin(session, pid);
288
        }
289
290 7162 leinfelder
        if (!allowed) {
291 7245 cjones
            throw new NotAuthorized("1320", "The provided identity does not have " + "permission to delete objects on the Node.");
292 7162 leinfelder
        }
293
294 7077 leinfelder
    	// defer to superclass implementation
295
        return super.delete(session, pid);
296 6250 cjones
    }
297
298 6475 jones
    /**
299
     * Updates an existing object by creating a new object identified by
300
     * newPid on the Member Node which explicitly obsoletes the object
301
     * identified by pid through appropriate changes to the SystemMetadata
302
     * of pid and newPid
303
     *
304
     * @param session - the Session object containing the credentials for the Subject
305
     * @param pid - The identifier of the object to be updated
306
     * @param object - the new object bytes
307
     * @param sysmeta - the new system metadata describing the object
308
     *
309
     * @return newPid - the identifier of the new object
310
     *
311
     * @throws InvalidToken
312
     * @throws ServiceFailure
313
     * @throws NotAuthorized
314
     * @throws NotFound
315
     * @throws NotImplemented
316
     * @throws IdentifierNotUnique
317
     * @throws UnsupportedType
318
     * @throws InsufficientResources
319
     * @throws InvalidSystemMetadata
320
     * @throws InvalidRequest
321
     */
322
    @Override
323 6575 cjones
    public Identifier update(Session session, Identifier pid, InputStream object,
324
        Identifier newPid, SystemMetadata sysmeta)
325
        throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique,
326
        UnsupportedType, InsufficientResources, NotFound,
327
        InvalidSystemMetadata, NotImplemented, InvalidRequest {
328 9781 cjones
329 10118 tao
        long startTime = System.currentTimeMillis();
330
331 9781 cjones
        if(isReadOnlyMode()) {
332
            throw new ServiceFailure("1310", ReadOnlyChecker.DATAONEERROR);
333
        }
334
335
        //transform a sid to a pid if it is applicable
336
        String serviceFailureCode = "1310";
337
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
338
        if(sid != null) {
339
            pid = sid;
340
        }
341
342
        String localId = null;
343
        boolean allowed = false;
344
        boolean isScienceMetadata = false;
345
346
        if (session == null) {
347
        	throw new InvalidToken("1210", "No session has been provided");
348
        }
349
        Subject subject = session.getSubject();
350
351
        // verify the pid is valid format
352
        if (!isValidIdentifier(pid)) {
353
        	throw new InvalidRequest("1202", "The provided identifier is invalid.");
354
        }
355
356
        // verify the new pid is valid format
357
        if (!isValidIdentifier(newPid)) {
358
            throw new InvalidRequest("1202", "The provided identifier is invalid.");
359
        }
360
361
        // make sure that the newPid doesn't exists
362
        boolean idExists = true;
363 9600 tao
        try {
364 9781 cjones
            idExists = IdentifierManager.getInstance().identifierExists(newPid.getValue());
365
        } catch (SQLException e) {
366
            throw new ServiceFailure("1310",
367
                                    "The requested identifier " + newPid.getValue() +
368
                                    " couldn't be determined if it is unique since : "+e.getMessage());
369
        }
370
        if (idExists) {
371
                throw new IdentifierNotUnique("1220",
372
                          "The requested identifier " + newPid.getValue() +
373
                          " is already used by another object and" +
374
                          "therefore can not be used for this object. Clients should choose" +
375
                          "a new identifier that is unique and retry the operation or " +
376
                          "use CN.reserveIdentifier() to reserve one.");
377 9044 tao
378 9781 cjones
        }
379
380
381
382
        // check for the existing identifier
383
        try {
384
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
385 6575 cjones
386 9781 cjones
        } catch (McdbDocNotFoundException e) {
387
            throw new InvalidRequest("1202", "The object with the provided " +
388
                "identifier was not found.");
389 6575 cjones
390 9781 cjones
        } catch (SQLException ee) {
391
            throw new ServiceFailure("1310", "The object with the provided " +
392
                    "identifier "+pid.getValue()+" can't be identified since - "+ee.getMessage());
393
        }
394
395 10118 tao
        long end =System.currentTimeMillis();
396
        logMetacat.debug("MNodeService.update - the time spending on checking the validation of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end- startTime)+ " milli seconds.");
397
398 9781 cjones
        // set the originating node
399
        NodeReference originMemberNode = this.getCapabilities().getIdentifier();
400
        sysmeta.setOriginMemberNode(originMemberNode);
401
402
        // set the submitter to match the certificate
403
        sysmeta.setSubmitter(subject);
404
        // set the dates
405
        Date now = Calendar.getInstance().getTime();
406
        sysmeta.setDateSysMetadataModified(now);
407
        sysmeta.setDateUploaded(now);
408
409
        // make sure serial version is set to something
410
        BigInteger serialVersion = sysmeta.getSerialVersion();
411
        if (serialVersion == null) {
412
        	sysmeta.setSerialVersion(BigInteger.ZERO);
413
        }
414 10118 tao
        long startTime2 = System.currentTimeMillis();
415 9781 cjones
        // does the subject have WRITE ( == update) priveleges on the pid?
416
        //allowed = isAuthorized(session, pid, Permission.WRITE);
417
        //CN having the permission is allowed; user with the write permission and calling on the authoritative node is allowed.
418 9997 tao
        try {
419
            allowed = allowUpdating(session, pid, Permission.WRITE);
420
        }   catch (NotFound e) {
421
            throw new NotFound("1280", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
422
        } catch(ServiceFailure e) {
423
            throw new ServiceFailure("1310", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
424
        } catch(NotAuthorized e) {
425
            throw new NotAuthorized("1200", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
426
        } catch(NotImplemented e) {
427
            throw new NotImplemented("1201","Can't determine if the client has the permission to update he object with id "+pid.getValue()+" since "+e.getDescription());
428
        } catch(InvalidRequest e) {
429
            throw new InvalidRequest("1202", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
430
        } catch(InvalidToken e) {
431
            throw new InvalidToken("1210", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription());
432
        }
433
434 10118 tao
        end =System.currentTimeMillis();
435
        logMetacat.debug("MNodeService.update - the time spending on checking if the user has the permission to update the old pid "+pid.getValue()+" with the new pid "+newPid.getValue()+" is "+(end- startTime2)+ " milli seconds.");
436
437 9781 cjones
        if (allowed) {
438 10118 tao
            long startTime3 = System.currentTimeMillis();
439 9781 cjones
        	// check quality of SM
440
        	if (sysmeta.getObsoletedBy() != null) {
441
        		throw new InvalidSystemMetadata("1300", "Cannot include obsoletedBy when updating object");
442
        	}
443
        	if (sysmeta.getObsoletes() != null && !sysmeta.getObsoletes().getValue().equals(pid.getValue())) {
444
        		throw new InvalidSystemMetadata("1300", "The identifier provided in obsoletes does not match old Identifier");
445
        	}
446
447
            // get the existing system metadata for the object
448
            SystemMetadata existingSysMeta = getSystemMetadata(session, pid);
449
            //System.out.println("the archive is "+existingSysMeta.getArchived());
450
            //Base on documentation, we can't update an archived object:
451
            //The update operation MUST fail with Exceptions.InvalidRequest on objects that have the Types.SystemMetadata.archived property set to true.
452
            if(existingSysMeta.getArchived() != null && existingSysMeta.getArchived()) {
453
                throw new InvalidRequest("1202","An archived object"+pid.getValue()+" can't be updated");
454 9200 tao
            }
455 9781 cjones
456
            // check for previous update
457
            // see: https://redmine.dataone.org/issues/3336
458
            Identifier existingObsoletedBy = existingSysMeta.getObsoletedBy();
459
            if (existingObsoletedBy != null) {
460
            	throw new InvalidRequest("1202",
461
            			"The previous identifier has already been made obsolete by: " + existingObsoletedBy.getValue());
462 7400 leinfelder
            }
463 10118 tao
            end =System.currentTimeMillis();
464
            logMetacat.debug("MNodeService.update - the time spending on checking the quality of the system metadata of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end- startTime3)+ " milli seconds.");
465
466 9781 cjones
            //check the sid in the system metadata. If it exists, it should be non-exist or match the old sid in the previous system metadata.
467
            Identifier sidInSys = sysmeta.getSeriesId();
468
            if(sidInSys != null) {
469
                if (!isValidIdentifier(sidInSys)) {
470
                    throw new InvalidSystemMetadata("1300", "The provided series id in the system metadata is invalid.");
471 9046 tao
                }
472 9781 cjones
                Identifier previousSid = existingSysMeta.getSeriesId();
473
                if(previousSid != null) {
474
                    // there is a previous sid, if the new sid doesn't match it, the new sid should be non-existing.
475
                    if(!sidInSys.getValue().equals(previousSid.getValue())) {
476 9044 tao
                        try {
477
                            idExists = IdentifierManager.getInstance().identifierExists(sidInSys.getValue());
478
                        } catch (SQLException e) {
479
                            throw new ServiceFailure("1310",
480
                                                    "The requested identifier " + sidInSys.getValue() +
481
                                                    " couldn't be determined if it is unique since : "+e.getMessage());
482
                        }
483
                        if(idExists) {
484 9781 cjones
                            throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata doesn't match the previous series id "
485
                                                            +previousSid.getValue()+", so it should NOT exist. However, it was used by another object.");
486 9044 tao
                        }
487
                    }
488 9781 cjones
                } else {
489
                    // there is no previous sid, the new sid should be non-existing.
490
                    try {
491
                        idExists = IdentifierManager.getInstance().identifierExists(sidInSys.getValue());
492
                    } catch (SQLException e) {
493
                        throw new ServiceFailure("1310",
494
                                                "The requested identifier " + sidInSys.getValue() +
495
                                                " couldn't be determined if it is unique since : "+e.getMessage());
496 9601 tao
                    }
497 9781 cjones
                    if(idExists) {
498
                        throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata should NOT exist since the previous series id is null."
499
                                                        +"However, it was used by another object.");
500
                    }
501 9601 tao
                }
502 9781 cjones
                //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario)
503
                if(sidInSys.getValue().equals(newPid.getValue())) {
504
                    throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata shouldn't have the same value of the pid.");
505
                }
506
            }
507 10118 tao
508
            long end2 =System.currentTimeMillis();
509
            logMetacat.debug("MNodeService.update - the time spending on checking the sid validation of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end2- end)+ " milli seconds.");
510
511 9781 cjones
512
            isScienceMetadata = isScienceMetadata(sysmeta);
513
514
            // do we have XML metadata or a data object?
515
            if (isScienceMetadata) {
516
517
                // update the science metadata XML document
518
                // TODO: handle non-XML metadata/data documents (like netCDF)
519
                // TODO: don't put objects into memory using stream to string
520
                //String objectAsXML = "";
521
                try {
522
                    //objectAsXML = IOUtils.toString(object, "UTF-8");
523 9783 leinfelder
                	String formatId = null;
524
                	if(sysmeta.getFormatId() != null) {
525
                	    formatId = sysmeta.getFormatId().getValue();
526
                	}
527
                    localId = insertOrUpdateDocument(object, "UTF-8", pid, session, "update", formatId);
528 9781 cjones
529
                    // register the newPid and the generated localId
530
                    if (newPid != null) {
531
                        IdentifierManager.getInstance().createMapping(newPid.getValue(), localId);
532 9044 tao
                    }
533 9781 cjones
534
                } catch (IOException e) {
535 10129 tao
                    String msg = "The Node is unable to create the object: "+pid.getValue() + "There was a problem converting the object to XML";
536
                    logMetacat.error(msg, e);
537 9781 cjones
                    throw new ServiceFailure("1310", msg + ": " + e.getMessage());
538
539 10129 tao
                }  catch (PropertyNotFoundException e) {
540
                    String msg = "The Node is unable to create the object. " +pid.getValue()+ " since the properties are not configured well "+e.getMessage();
541
                    logMetacat.error(msg, e);
542
                    throw new ServiceFailure("1310", msg);
543 9044 tao
                }
544 9781 cjones
545 6475 jones
            } else {
546 9781 cjones
547
                // update the data object
548
                localId = insertDataObject(object, newPid, session);
549
550 6475 jones
            }
551 9781 cjones
552 10118 tao
            long end3 =System.currentTimeMillis();
553
            logMetacat.debug("MNodeService.update - the time spending on saving the object with the new pid "+newPid.getValue()+" is "+(end3- end2)+ " milli seconds.");
554
555 9781 cjones
            // add the newPid to the obsoletedBy list for the existing sysmeta
556
            existingSysMeta.setObsoletedBy(newPid);
557
            //increase version
558
            BigInteger current = existingSysMeta.getSerialVersion();
559
            //System.out.println("the current version is "+current);
560
            current = current.add(BigInteger.ONE);
561
            //System.out.println("the new current version is "+current);
562
            existingSysMeta.setSerialVersion(current);
563
            // then update the existing system metadata
564
            updateSystemMetadata(existingSysMeta);
565
566
            // prep the new system metadata, add pid to the affected lists
567
            sysmeta.setObsoletes(pid);
568
            //sysmeta.addDerivedFrom(pid);
569
570
            // and insert the new system metadata
571
            insertSystemMetadata(sysmeta);
572
573
            // log the update event
574
            EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), subject.getValue(), localId, Event.UPDATE.toString());
575
576 10172 leinfelder
            // queue for QC reporting
577
            MDQClient.submit(sysmeta);
578
579 10118 tao
            long end4 =System.currentTimeMillis();
580
            logMetacat.debug("MNodeService.update - the time spending on updating/saving system metadata  of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" and saving the log information is "+(end4- end3)+ " milli seconds.");
581
582 9781 cjones
            // attempt to register the identifier - it checks if it is a doi
583
            try {
584
    			DOIService.getInstance().registerDOI(sysmeta);
585
    		} catch (Exception e) {
586
                throw new ServiceFailure("1190", "Could not register DOI: " + e.getMessage());
587
    		}
588 10118 tao
            long end5 =System.currentTimeMillis();
589
            logMetacat.debug("MNodeService.update - the time spending on registering the doi (if it is doi ) of the new pid "+newPid.getValue()+" is "+(end5- end4)+ " milli seconds.");
590
591 9781 cjones
        } else {
592
            throw new NotAuthorized("1200", "The provided identity does not have " + "permission to UPDATE the object identified by " + pid.getValue()
593
                    + " on the Member Node.");
594 9600 tao
        }
595 9781 cjones
596 10118 tao
        long end6 =System.currentTimeMillis();
597
        logMetacat.debug("MNodeService.update - the total time of updating the old pid " +pid.getValue() +" whth the new pid "+newPid.getValue()+" is "+(end6- startTime)+ " milli seconds.");
598 6475 jones
        return newPid;
599 6250 cjones
    }
600 6254 cjones
601 6475 jones
    public Identifier create(Session session, Identifier pid, InputStream object, SystemMetadata sysmeta) throws InvalidToken, ServiceFailure, NotAuthorized,
602
            IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest {
603 9781 cjones
604
        if(isReadOnlyMode()) {
605
            throw new ServiceFailure("1190", ReadOnlyChecker.DATAONEERROR);
606
        }
607
        // check for null session
608
        if (session == null) {
609
          throw new InvalidToken("1110", "Session is required to WRITE to the Node.");
610
        }
611
        // verify the pid is valid format
612
        if (!isValidIdentifier(pid)) {
613
            throw new InvalidRequest("1102", "The provided identifier is invalid.");
614
        }
615
        // set the submitter to match the certificate
616
        sysmeta.setSubmitter(session.getSubject());
617
        // set the originating node
618
        NodeReference originMemberNode = this.getCapabilities().getIdentifier();
619
        sysmeta.setOriginMemberNode(originMemberNode);
620
621
        // if no authoritative MN, set it to the same
622
        if (sysmeta.getAuthoritativeMemberNode() == null) {
623
        	sysmeta.setAuthoritativeMemberNode(originMemberNode);
624
        }
625
626
        sysmeta.setArchived(false);
627
628
        // set the dates
629
        Date now = Calendar.getInstance().getTime();
630
        sysmeta.setDateSysMetadataModified(now);
631
        sysmeta.setDateUploaded(now);
632
633
        // set the serial version
634
        sysmeta.setSerialVersion(BigInteger.ZERO);
635
636
        // check that we are not attempting to subvert versioning
637
        if (sysmeta.getObsoletes() != null && sysmeta.getObsoletes().getValue() != null) {
638
            throw new InvalidSystemMetadata("1180",
639
              "The supplied system metadata is invalid. " +
640
              "The obsoletes field cannot have a value when creating entries.");
641
        }
642
643
        if (sysmeta.getObsoletedBy() != null && sysmeta.getObsoletedBy().getValue() != null) {
644
            throw new InvalidSystemMetadata("1180",
645
              "The supplied system metadata is invalid. " +
646
              "The obsoletedBy field cannot have a value when creating entries.");
647
        }
648
649
        // verify the sid in the system metadata
650
        Identifier sid = sysmeta.getSeriesId();
651
        boolean idExists = false;
652
        if(sid != null) {
653
            if (!isValidIdentifier(sid)) {
654
                throw new InvalidSystemMetadata("1180", "The provided series id is invalid.");
655 9068 tao
            }
656 9781 cjones
            try {
657
                idExists = IdentifierManager.getInstance().identifierExists(sid.getValue());
658
            } catch (SQLException e) {
659
                throw new ServiceFailure("1190",
660
                                        "The series identifier " + sid.getValue() +
661
                                        " in the system metadata couldn't be determined if it is unique since : "+e.getMessage());
662 9068 tao
            }
663 9781 cjones
            if (idExists) {
664
                    throw new InvalidSystemMetadata("1180",
665
                              "The series identifier " + sid.getValue() +
666
                              " is already used by another object and" +
667
                              "therefore can not be used for this object. Clients should choose" +
668
                              "a new identifier that is unique and retry the operation or " +
669
                              "use CN.reserveIdentifier() to reserve one.");
670
671 9068 tao
            }
672 9781 cjones
            //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario )
673
            if(sid.getValue().equals(pid.getValue())) {
674
                throw new InvalidSystemMetadata("1180", "The series id "+sid.getValue()+" in the system metadata shouldn't have the same value of the pid.");
675 9068 tao
            }
676 9600 tao
        }
677 9781 cjones
678
        // call the shared impl
679
        Identifier resultPid = super.create(session, pid, object, sysmeta);
680
681
        // attempt to register the identifier - it checks if it is a doi
682
        try {
683
			DOIService.getInstance().registerDOI(sysmeta);
684
		} catch (Exception e) {
685
			ServiceFailure sf = new ServiceFailure("1190", "Could not register DOI: " + e.getMessage());
686
			sf.initCause(e);
687
            throw sf;
688
		}
689
690 10172 leinfelder
        // queue for QC reporting
691
        MDQClient.submit(sysmeta);
692
693 7507 leinfelder
        // return
694
		return resultPid ;
695 6475 jones
    }
696 6250 cjones
697 6475 jones
    /**
698
     * Called by a Coordinating Node to request that the Member Node create a
699
     * copy of the specified object by retrieving it from another Member
700
     * Node and storing it locally so that it can be made accessible to
701
     * the DataONE system.
702
     *
703
     * @param session - the Session object containing the credentials for the Subject
704
     * @param sysmeta - Copy of the CN held system metadata for the object
705
     * @param sourceNode - A reference to node from which the content should be
706
     *                     retrieved. The reference should be resolved by
707
     *                     checking the CN node registry.
708
     *
709
     * @return true if the replication succeeds
710
     *
711
     * @throws ServiceFailure
712
     * @throws NotAuthorized
713
     * @throws NotImplemented
714
     * @throws UnsupportedType
715
     * @throws InsufficientResources
716
     * @throws InvalidRequest
717
     */
718
    @Override
719 6786 cjones
    public boolean replicate(Session session, SystemMetadata sysmeta,
720
            NodeReference sourceNode) throws NotImplemented, ServiceFailure,
721
            NotAuthorized, InvalidRequest, InsufficientResources,
722
            UnsupportedType {
723 9480 tao
        /*if(isReadOnlyMode()) {
724 9478 tao
            throw new InvalidRequest("2153", "The Metacat member node is on the read-only mode and your request can't be fulfiled. Please try again later.");
725 9480 tao
        }*/
726 6786 cjones
727 6875 cjones
        if (session != null && sysmeta != null && sourceNode != null) {
728
            logMetacat.info("MNodeService.replicate() called with parameters: \n" +
729
                            "\tSession.Subject      = "                           +
730
                            session.getSubject().getValue() + "\n"                +
731 7082 cjones
                            "\tidentifier           = "                           +
732
                            sysmeta.getIdentifier().getValue()                    +
733 6875 cjones
                            "\n" + "\tSource NodeReference ="                     +
734
                            sourceNode.getValue());
735
        }
736 6475 jones
        boolean result = false;
737 6786 cjones
        String nodeIdStr = null;
738 6651 cjones
        NodeReference nodeId = null;
739 6786 cjones
740 6795 cjones
        // get the referenced object
741
        Identifier pid = sysmeta.getIdentifier();
742 9068 tao
        // verify the pid is valid format
743
        if (!isValidIdentifier(pid)) {
744
            throw new InvalidRequest("2153", "The provided identifier in the system metadata is invalid.");
745
        }
746 6795 cjones
747
        // get from the membernode
748
        // TODO: switch credentials for the server retrieval?
749
        this.cn = D1Client.getCN();
750
        InputStream object = null;
751
        Session thisNodeSession = null;
752
        SystemMetadata localSystemMetadata = null;
753
        BaseException failure = null;
754 6818 cjones
        String localId = null;
755
756 6795 cjones
        // TODO: check credentials
757
        // cannot be called by public
758 7063 leinfelder
        if (session == null || session.getSubject() == null) {
759 7082 cjones
            String msg = "No session was provided to replicate identifier " +
760
            sysmeta.getIdentifier().getValue();
761 8414 tao
            logMetacat.error(msg);
762 7192 cjones
            throw new NotAuthorized("2152", msg);
763
764 6795 cjones
        }
765
766
767 6651 cjones
        // get the local node id
768
        try {
769 7030 cjones
            nodeIdStr = PropertyService.getProperty("dataone.nodeId");
770 6651 cjones
            nodeId = new NodeReference();
771
            nodeId.setValue(nodeIdStr);
772 6786 cjones
773 6651 cjones
        } catch (PropertyNotFoundException e1) {
774 7030 cjones
            String msg = "Couldn't get dataone.nodeId property: " + e1.getMessage();
775 6795 cjones
            failure = new ServiceFailure("2151", msg);
776 8414 tao
            //setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
777 6795 cjones
            logMetacat.error(msg);
778 8414 tao
            //return true;
779
            throw new ServiceFailure("2151", msg);
780 6786 cjones
781 6651 cjones
        }
782 6795 cjones
783 6475 jones
        try {
784 6786 cjones
            try {
785 9544 tao
                // do we already have a replica?
786 6822 cjones
                try {
787 9544 tao
                    localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
788
                    // if we have a local id, get the local object
789
                    try {
790
                        object = MetacatHandler.read(localId);
791
                    } catch (Exception e) {
792
                        // NOTE: we may already know about this ID because it could be a data file described by a metadata file
793
                        // https://redmine.dataone.org/issues/2572
794
                        // TODO: fix this so that we don't prevent ourselves from getting replicas
795
796
                        // let the CN know that the replication failed
797
                        logMetacat.warn("Object content not found on this node despite having localId: " + localId);
798
                        String msg = "Can't read the object bytes properly, replica is invalid.";
799
                        ServiceFailure serviceFailure = new ServiceFailure("2151", msg);
800
                        setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, serviceFailure);
801
                        logMetacat.warn(msg);
802
                        throw serviceFailure;
803
804
                    }
805
806
                } catch (McdbDocNotFoundException e) {
807
                    logMetacat.info("No replica found. Continuing.");
808 6822 cjones
809 9544 tao
                } catch (SQLException ee) {
810
                    throw new ServiceFailure("2151", "Couldn't identify the local id of the object with the specified identifier "
811
                                            +pid.getValue()+" since - "+ee.getMessage());
812 6822 cjones
                }
813 9544 tao
814
                // no local replica, get a replica
815
                if ( object == null ) {
816
                    /*boolean success = true;
817
                    try {
818
                        //use the v2 ping api to connect the source node
819
                        mn.ping();
820
                    } catch (Exception e) {
821
                        success = false;
822
                    }*/
823
                    D1NodeVersionChecker checker = new D1NodeVersionChecker(sourceNode);
824
                    String nodeVersion = checker.getVersion("MNRead");
825
                    if(nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V1)) {
826
                        //The source node is a v1 node, we use the v1 api
827
                        org.dataone.client.v1.MNode mNodeV1 =  org.dataone.client.v1.itk.D1Client.getMN(sourceNode);
828
                        object = mNodeV1.getReplica(thisNodeSession, pid);
829
                    } else if (nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V2)){
830
                     // session should be null to use the default certificate
831
                        // location set in the Certificate manager
832
                        MNode mn = D1Client.getMN(sourceNode);
833
                        object = mn.getReplica(thisNodeSession, pid);
834
                    } else {
835
                        throw new ServiceFailure("2151", "The version of MNRead service is "+nodeVersion+" in the source node "+sourceNode.getValue()+" and it is supported. Please check the information in the cn");
836
                    }
837
838
                    logMetacat.info("MNodeService.getReplica() called for identifier "
839
                                    + pid.getValue());
840 6822 cjones
841 9544 tao
                }
842
843
            } catch (InvalidToken e) {
844
                String msg = "Could not retrieve object to replicate (InvalidToken): "+ e.getMessage();
845
                failure = new ServiceFailure("2151", msg);
846
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
847
                logMetacat.error(msg);
848
                throw new ServiceFailure("2151", msg);
849
850
            } catch (NotFound e) {
851
                String msg = "Could not retrieve object to replicate (NotFound): "+ e.getMessage();
852
                failure = new ServiceFailure("2151", msg);
853
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
854
                logMetacat.error(msg);
855
                throw new ServiceFailure("2151", msg);
856
857
            } catch (NotAuthorized e) {
858
                String msg = "Could not retrieve object to replicate (NotAuthorized): "+ e.getMessage();
859
                failure = new ServiceFailure("2151", msg);
860
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
861
                logMetacat.error(msg);
862
                throw new ServiceFailure("2151", msg);
863
            } catch (NotImplemented e) {
864
                String msg = "Could not retrieve object to replicate (mn.getReplica NotImplemented): "+ e.getMessage();
865
                failure = new ServiceFailure("2151", msg);
866
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
867
                logMetacat.error(msg);
868
                throw new ServiceFailure("2151", msg);
869
            } catch (ServiceFailure e) {
870
                String msg = "Could not retrieve object to replicate (ServiceFailure): "+ e.getMessage();
871
                failure = new ServiceFailure("2151", msg);
872
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
873
                logMetacat.error(msg);
874
                throw new ServiceFailure("2151", msg);
875
            } catch (InsufficientResources e) {
876
                String msg = "Could not retrieve object to replicate (InsufficientResources): "+ e.getMessage();
877
                failure = new ServiceFailure("2151", msg);
878
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
879
                logMetacat.error(msg);
880
                throw new ServiceFailure("2151", msg);
881 6819 cjones
            }
882 9544 tao
883
            // verify checksum on the object, if supported
884
            if (object.markSupported()) {
885
                Checksum givenChecksum = sysmeta.getChecksum();
886
                Checksum computedChecksum = null;
887 9288 tao
                try {
888 9544 tao
                    computedChecksum = ChecksumUtil.checksum(object, givenChecksum.getAlgorithm());
889
                    object.reset();
890
891 9288 tao
                } catch (Exception e) {
892 9544 tao
                    String msg = "Error computing checksum on replica: " + e.getMessage();
893
                    logMetacat.error(msg);
894
                    ServiceFailure sf = new ServiceFailure("2151", msg);
895
                    sf.initCause(e);
896
                    setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, sf);
897
                    throw sf;
898 9287 tao
                }
899 9544 tao
                if (!givenChecksum.getValue().equals(computedChecksum.getValue())) {
900
                    logMetacat.error("Given    checksum for " + pid.getValue() +
901
                        "is " + givenChecksum.getValue());
902
                    logMetacat.error("Computed checksum for " + pid.getValue() +
903
                        "is " + computedChecksum.getValue());
904
                    String msg = "Computed checksum does not match declared checksum";
905
                    failure = new ServiceFailure("2151", msg);
906
                    setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
907
                    throw new ServiceFailure("2151", msg);
908
                }
909 6786 cjones
            }
910
911 9544 tao
            // add it to local store
912
            Identifier retPid;
913 6786 cjones
            try {
914 9544 tao
                // skip the MN.create -- this mutates the system metadata and we don't want it to
915
                if ( localId == null ) {
916
                    // TODO: this will fail if we already "know" about the identifier
917
                    // FIXME: see https://redmine.dataone.org/issues/2572
918
                    retPid = super.create(session, pid, object, sysmeta);
919
                    result = (retPid.getValue().equals(pid.getValue()));
920
                }
921
922 6786 cjones
            } catch (Exception e) {
923 9544 tao
                String msg = "Could not save object to local store (" + e.getClass().getName() + "): " + e.getMessage();
924 8414 tao
                failure = new ServiceFailure("2151", msg);
925
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
926 9544 tao
                logMetacat.error(msg);
927 8414 tao
                throw new ServiceFailure("2151", msg);
928 9544 tao
929 6786 cjones
            }
930 9544 tao
        } finally {
931
            IOUtils.closeQuietly(object);
932 6693 leinfelder
        }
933 9544 tao
934 6786 cjones
935 6795 cjones
        // finish by setting the replication status
936
        setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.COMPLETED, null);
937 6475 jones
        return result;
938
939 6250 cjones
    }
940 9287 tao
941
    /*
942
     * If the given node supports v2 replication.
943
     */
944
    private boolean supportV2Replication(Node node) throws InvalidRequest {
945
        return supportVersionReplication(node, "v2");
946
    }
947
948
    /*
949
     * If the given node support the the given version replication. Return true if it does.
950
     */
951
    private boolean supportVersionReplication(Node node, String version) throws InvalidRequest{
952
        boolean support = false;
953
        if(node == null) {
954
            throw new InvalidRequest("2153", "There is no capacity information about the node in the replicate.");
955
        } else {
956
            Services services = node.getServices();
957
            if(services == null) {
958
                throw new InvalidRequest("2153", "Can't get replica from a node which doesn't have the replicate service.");
959
            } else {
960
               List<Service> list = services.getServiceList();
961
               if(list == null) {
962
                   throw new InvalidRequest("2153", "Can't get replica from a node which doesn't have the replicate service.");
963
               } else {
964
                   for(Service service : list) {
965
                       if(service != null && service.getName() != null && service.getName().equals("MNReplication") &&
966
                               service.getVersion() != null && service.getVersion().equalsIgnoreCase(version) && service.getAvailable() == true ) {
967
                           support = true;
968
969
                       }
970
                   }
971
               }
972
            }
973
        }
974
        return support;
975
    }
976 6179 cjones
977 6475 jones
    /**
978
     * Return the object identified by the given object identifier
979
     *
980
     * @param session - the Session object containing the credentials for the Subject
981
     * @param pid - the object identifier for the given object
982
     *
983
     * @return inputStream - the input stream of the given object
984
     *
985
     * @throws InvalidToken
986
     * @throws ServiceFailure
987
     * @throws NotAuthorized
988
     * @throws InvalidRequest
989
     * @throws NotImplemented
990
     */
991
    @Override
992 6610 cjones
    public InputStream get(Session session, Identifier pid)
993
    throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
994 6258 cjones
995 6475 jones
        return super.get(session, pid);
996 6258 cjones
997 6259 cjones
    }
998 6258 cjones
999 6475 jones
    /**
1000
     * Returns a Checksum for the specified object using an accepted hashing algorithm
1001
     *
1002
     * @param session - the Session object containing the credentials for the Subject
1003
     * @param pid - the object identifier for the given object
1004
     * @param algorithm -  the name of an algorithm that will be used to compute
1005
     *                     a checksum of the bytes of the object
1006
     *
1007
     * @return checksum - the checksum of the given object
1008
     *
1009
     * @throws InvalidToken
1010
     * @throws ServiceFailure
1011
     * @throws NotAuthorized
1012
     * @throws NotFound
1013
     * @throws InvalidRequest
1014
     * @throws NotImplemented
1015
     */
1016
    @Override
1017 6610 cjones
    public Checksum getChecksum(Session session, Identifier pid, String algorithm)
1018
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
1019
        InvalidRequest, NotImplemented {
1020 6258 cjones
1021 6475 jones
        Checksum checksum = null;
1022 9040 tao
        String serviceFailure = "1410";
1023
        String notFound = "1420";
1024
        //Checkum only handles the pid, not sid
1025
        checkV1SystemMetaPidExist(pid, serviceFailure, "The checksum for the object specified by "+pid.getValue()+" couldn't be returned ",  notFound,
1026
                "The object specified by "+pid.getValue()+" does not exist at this node.");
1027 6475 jones
        InputStream inputStream = get(session, pid);
1028
1029 6259 cjones
        try {
1030 6475 jones
            checksum = ChecksumUtil.checksum(inputStream, algorithm);
1031
1032
        } catch (NoSuchAlgorithmException e) {
1033
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned due to an internal error: "
1034
                    + e.getMessage());
1035 6259 cjones
        } catch (IOException e) {
1036 6475 jones
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned due to an internal error: "
1037
                    + e.getMessage());
1038 6259 cjones
        }
1039 6382 cjones
1040 6475 jones
        if (checksum == null) {
1041
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned.");
1042
        }
1043 6258 cjones
1044 6475 jones
        return checksum;
1045 6259 cjones
    }
1046 6179 cjones
1047 6475 jones
    /**
1048
     * Return the system metadata for a given object
1049
     *
1050
     * @param session - the Session object containing the credentials for the Subject
1051
     * @param pid - the object identifier for the given object
1052
     *
1053
     * @return inputStream - the input stream of the given system metadata object
1054
     *
1055
     * @throws InvalidToken
1056
     * @throws ServiceFailure
1057
     * @throws NotAuthorized
1058
     * @throws NotFound
1059
     * @throws InvalidRequest
1060
     * @throws NotImplemented
1061
     */
1062
    @Override
1063 6610 cjones
    public SystemMetadata getSystemMetadata(Session session, Identifier pid)
1064
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
1065
        NotImplemented {
1066 6341 leinfelder
1067 6475 jones
        return super.getSystemMetadata(session, pid);
1068
    }
1069 6341 leinfelder
1070 6475 jones
    /**
1071
     * Retrieve the list of objects present on the MN that match the calling parameters
1072
     *
1073
     * @param session - the Session object containing the credentials for the Subject
1074
     * @param startTime - Specifies the beginning of the time range from which
1075
     *                    to return object (>=)
1076
     * @param endTime - Specifies the beginning of the time range from which
1077
     *                  to return object (>=)
1078
     * @param objectFormat - Restrict results to the specified object format
1079
     * @param replicaStatus - Indicates if replicated objects should be returned in the list
1080
     * @param start - The zero-based index of the first value, relative to the
1081
     *                first record of the resultset that matches the parameters.
1082
     * @param count - The maximum number of entries that should be returned in
1083
     *                the response. The Member Node may return less entries
1084
     *                than specified in this value.
1085
     *
1086
     * @return objectList - the list of objects matching the criteria
1087
     *
1088
     * @throws InvalidToken
1089
     * @throws ServiceFailure
1090
     * @throws NotAuthorized
1091
     * @throws InvalidRequest
1092
     * @throws NotImplemented
1093
     */
1094
    @Override
1095 8810 leinfelder
    public ObjectList listObjects(Session session, Date startTime, Date endTime, ObjectFormatIdentifier objectFormatId, Identifier identifier, Boolean replicaStatus, Integer start,
1096 6475 jones
            Integer count) throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken {
1097 9249 tao
        NodeReference nodeId = null;
1098
        if(!replicaStatus) {
1099
            //not include those objects whose authoritative node is not this mn
1100
            nodeId = new NodeReference();
1101
            try {
1102
                String currentNodeId = PropertyService.getInstance().getProperty("dataone.nodeId"); // return only pids for which this mn
1103
                nodeId.setValue(currentNodeId);
1104
            } catch(Exception e) {
1105
                throw new ServiceFailure("1580", e.getMessage());
1106
            }
1107
        }
1108
        return super.listObjects(session, startTime, endTime, objectFormatId, identifier, nodeId, start, count);
1109 6229 cjones
    }
1110 6179 cjones
1111 6475 jones
    /**
1112 6476 jones
     * Return a description of the node's capabilities and services.
1113 6475 jones
     *
1114
     * @return node - the technical capabilities of the Member Node
1115
     *
1116
     * @throws ServiceFailure
1117
     * @throws NotAuthorized
1118
     * @throws InvalidRequest
1119
     * @throws NotImplemented
1120
     */
1121
    @Override
1122 6610 cjones
    public Node getCapabilities()
1123
        throws NotImplemented, ServiceFailure {
1124 6179 cjones
1125 6475 jones
        String nodeName = null;
1126
        String nodeId = null;
1127 6492 jones
        String subject = null;
1128 6938 cjones
        String contactSubject = null;
1129 6475 jones
        String nodeDesc = null;
1130 6476 jones
        String nodeTypeString = null;
1131
        NodeType nodeType = null;
1132 9253 tao
        List<String> mnCoreServiceVersions = null;
1133
        List<String> mnReadServiceVersions = null;
1134
        List<String> mnAuthorizationServiceVersions = null;
1135
        List<String> mnStorageServiceVersions = null;
1136
        List<String> mnReplicationServiceVersions = null;
1137 6179 cjones
1138 6475 jones
        boolean nodeSynchronize = false;
1139
        boolean nodeReplicate = false;
1140 9253 tao
        List<String> mnCoreServiceAvailables = null;
1141
        List<String> mnReadServiceAvailables = null;
1142
        List<String> mnAuthorizationServiceAvailables = null;
1143
        List<String> mnStorageServiceAvailables = null;
1144
        List<String> mnReplicationServiceAvailables = null;
1145 6179 cjones
1146 6475 jones
        try {
1147
            // get the properties of the node based on configuration information
1148 9253 tao
            nodeName = Settings.getConfiguration().getString("dataone.nodeName");
1149
            nodeId = Settings.getConfiguration().getString("dataone.nodeId");
1150
            subject = Settings.getConfiguration().getString("dataone.subject");
1151
            contactSubject = Settings.getConfiguration().getString("dataone.contactSubject");
1152
            nodeDesc = Settings.getConfiguration().getString("dataone.nodeDescription");
1153
            nodeTypeString = Settings.getConfiguration().getString("dataone.nodeType");
1154 6476 jones
            nodeType = NodeType.convert(nodeTypeString);
1155 9253 tao
            nodeSynchronize = new Boolean(Settings.getConfiguration().getString("dataone.nodeSynchronize")).booleanValue();
1156
            nodeReplicate = new Boolean(Settings.getConfiguration().getString("dataone.nodeReplicate")).booleanValue();
1157 6475 jones
1158 6476 jones
            // Set the properties of the node based on configuration information and
1159
            // calls to current status methods
1160 9809 tao
            String serviceName = SystemUtil.getContextURL() + "/" + PropertyService.getProperty("dataone.serviceName");
1161 6476 jones
            Node node = new Node();
1162 6542 leinfelder
            node.setBaseURL(serviceName + "/" + nodeTypeString);
1163 6476 jones
            node.setDescription(nodeDesc);
1164 6475 jones
1165 6476 jones
            // set the node's health information
1166
            node.setState(NodeState.UP);
1167
1168
            // set the ping response to the current value
1169
            Ping canPing = new Ping();
1170
            canPing.setSuccess(false);
1171
            try {
1172 6803 leinfelder
            	Date pingDate = ping();
1173
                canPing.setSuccess(pingDate != null);
1174
            } catch (BaseException e) {
1175 6476 jones
                e.printStackTrace();
1176 6803 leinfelder
                // guess it can't be pinged
1177 6476 jones
            }
1178 6610 cjones
1179 6476 jones
            node.setPing(canPing);
1180 6475 jones
1181 6476 jones
            NodeReference identifier = new NodeReference();
1182
            identifier.setValue(nodeId);
1183
            node.setIdentifier(identifier);
1184 6492 jones
            Subject s = new Subject();
1185
            s.setValue(subject);
1186
            node.addSubject(s);
1187 6938 cjones
            Subject contact = new Subject();
1188
            contact.setValue(contactSubject);
1189
            node.addContactSubject(contact);
1190 6476 jones
            node.setName(nodeName);
1191
            node.setReplicate(nodeReplicate);
1192
            node.setSynchronize(nodeSynchronize);
1193 6475 jones
1194 6476 jones
            // services: MNAuthorization, MNCore, MNRead, MNReplication, MNStorage
1195
            Services services = new Services();
1196 6475 jones
1197 9253 tao
            mnCoreServiceVersions = Settings.getConfiguration().getList("dataone.mnCore.serviceVersion");
1198
            mnCoreServiceAvailables = Settings.getConfiguration().getList("dataone.mnCore.serviceAvailable");
1199
            if(mnCoreServiceVersions != null && mnCoreServiceAvailables != null && mnCoreServiceVersions.size() == mnCoreServiceAvailables.size()) {
1200
                for(int i=0; i<mnCoreServiceVersions.size(); i++) {
1201
                    String version = mnCoreServiceVersions.get(i);
1202
                    boolean available = new Boolean(mnCoreServiceAvailables.get(i)).booleanValue();
1203
                    Service sMNCore = new Service();
1204
                    sMNCore.setName("MNCore");
1205
                    sMNCore.setVersion(version);
1206
                    sMNCore.setAvailable(available);
1207
                    services.addService(sMNCore);
1208
                }
1209
            }
1210
1211
            mnReadServiceVersions = Settings.getConfiguration().getList("dataone.mnRead.serviceVersion");
1212
            mnReadServiceAvailables = Settings.getConfiguration().getList("dataone.mnRead.serviceAvailable");
1213
            if(mnReadServiceVersions != null && mnReadServiceAvailables != null && mnReadServiceVersions.size()==mnReadServiceAvailables.size()) {
1214
                for(int i=0; i<mnReadServiceVersions.size(); i++) {
1215
                    String version = mnReadServiceVersions.get(i);
1216
                    boolean available = new Boolean(mnReadServiceAvailables.get(i)).booleanValue();
1217
                    Service sMNRead = new Service();
1218
                    sMNRead.setName("MNRead");
1219
                    sMNRead.setVersion(version);
1220
                    sMNRead.setAvailable(available);
1221
                    services.addService(sMNRead);
1222
                }
1223
            }
1224
1225
            mnAuthorizationServiceVersions = Settings.getConfiguration().getList("dataone.mnAuthorization.serviceVersion");
1226
            mnAuthorizationServiceAvailables = Settings.getConfiguration().getList("dataone.mnAuthorization.serviceAvailable");
1227
            if(mnAuthorizationServiceVersions != null && mnAuthorizationServiceAvailables != null && mnAuthorizationServiceVersions.size()==mnAuthorizationServiceAvailables.size()) {
1228
                for(int i=0; i<mnAuthorizationServiceVersions.size(); i++) {
1229
                    String version = mnAuthorizationServiceVersions.get(i);
1230
                    boolean available = new Boolean(mnAuthorizationServiceAvailables.get(i)).booleanValue();
1231
                    Service sMNAuthorization = new Service();
1232
                    sMNAuthorization.setName("MNAuthorization");
1233
                    sMNAuthorization.setVersion(version);
1234
                    sMNAuthorization.setAvailable(available);
1235
                    services.addService(sMNAuthorization);
1236
                }
1237
            }
1238
1239
            mnStorageServiceVersions = Settings.getConfiguration().getList("dataone.mnStorage.serviceVersion");
1240
            mnStorageServiceAvailables = Settings.getConfiguration().getList("dataone.mnStorage.serviceAvailable");
1241
            if(mnStorageServiceVersions != null && mnStorageServiceAvailables != null && mnStorageServiceVersions.size() == mnStorageServiceAvailables.size()) {
1242
                for(int i=0; i<mnStorageServiceVersions.size(); i++) {
1243
                    String version = mnStorageServiceVersions.get(i);
1244
                    boolean available = new Boolean(mnStorageServiceAvailables.get(i)).booleanValue();
1245
                    Service sMNStorage = new Service();
1246
                    sMNStorage.setName("MNStorage");
1247
                    sMNStorage.setVersion(version);
1248
                    sMNStorage.setAvailable(available);
1249
                    services.addService(sMNStorage);
1250
                }
1251
            }
1252
1253
            mnReplicationServiceVersions = Settings.getConfiguration().getList("dataone.mnReplication.serviceVersion");
1254
            mnReplicationServiceAvailables = Settings.getConfiguration().getList("dataone.mnReplication.serviceAvailable");
1255
            if(mnReplicationServiceVersions != null && mnReplicationServiceAvailables != null && mnReplicationServiceVersions.size() == mnReplicationServiceAvailables.size()) {
1256
                for (int i=0; i<mnReplicationServiceVersions.size(); i++) {
1257
                    String version = mnReplicationServiceVersions.get(i);
1258
                    boolean available = new Boolean(mnReplicationServiceAvailables.get(i)).booleanValue();
1259
                    Service sMNReplication = new Service();
1260
                    sMNReplication.setName("MNReplication");
1261
                    sMNReplication.setVersion(version);
1262
                    sMNReplication.setAvailable(available);
1263
                    services.addService(sMNReplication);
1264
                }
1265
            }
1266
1267 6476 jones
            node.setServices(services);
1268 6475 jones
1269 6476 jones
            // Set the schedule for synchronization
1270
            Synchronization synchronization = new Synchronization();
1271
            Schedule schedule = new Schedule();
1272
            Date now = new Date();
1273 6689 leinfelder
            schedule.setYear(PropertyService.getProperty("dataone.nodeSynchronization.schedule.year"));
1274
            schedule.setMon(PropertyService.getProperty("dataone.nodeSynchronization.schedule.mon"));
1275
            schedule.setMday(PropertyService.getProperty("dataone.nodeSynchronization.schedule.mday"));
1276
            schedule.setWday(PropertyService.getProperty("dataone.nodeSynchronization.schedule.wday"));
1277
            schedule.setHour(PropertyService.getProperty("dataone.nodeSynchronization.schedule.hour"));
1278
            schedule.setMin(PropertyService.getProperty("dataone.nodeSynchronization.schedule.min"));
1279
            schedule.setSec(PropertyService.getProperty("dataone.nodeSynchronization.schedule.sec"));
1280 6476 jones
            synchronization.setSchedule(schedule);
1281
            synchronization.setLastHarvested(now);
1282
            synchronization.setLastCompleteHarvest(now);
1283
            node.setSynchronization(synchronization);
1284 6475 jones
1285 6476 jones
            node.setType(nodeType);
1286
            return node;
1287 6475 jones
1288 6476 jones
        } catch (PropertyNotFoundException pnfe) {
1289
            String msg = "MNodeService.getCapabilities(): " + "property not found: " + pnfe.getMessage();
1290
            logMetacat.error(msg);
1291
            throw new ServiceFailure("2162", msg);
1292
        }
1293 6228 cjones
    }
1294 6179 cjones
1295 8810 leinfelder
1296 6179 cjones
1297 6475 jones
    /**
1298
     * A callback method used by a CN to indicate to a MN that it cannot
1299
     * complete synchronization of the science metadata identified by pid.  Log
1300
     * the event in the metacat event log.
1301
     *
1302
     * @param session
1303
     * @param syncFailed
1304
     *
1305
     * @throws ServiceFailure
1306
     * @throws NotAuthorized
1307
     * @throws NotImplemented
1308
     */
1309
    @Override
1310 6991 leinfelder
    public boolean synchronizationFailed(Session session, SynchronizationFailed syncFailed)
1311 6610 cjones
        throws NotImplemented, ServiceFailure, NotAuthorized {
1312 6179 cjones
1313 6475 jones
        String localId;
1314 7075 cjones
        Identifier pid;
1315
        if ( syncFailed.getPid() != null ) {
1316
            pid = new Identifier();
1317
            pid.setValue(syncFailed.getPid());
1318
            boolean allowed;
1319
1320
            //are we allowed? only CNs
1321
            try {
1322 7142 leinfelder
                allowed = isAdminAuthorized(session);
1323 7075 cjones
                if ( !allowed ){
1324
                    throw new NotAuthorized("2162",
1325
                            "Not allowed to call synchronizationFailed() on this node.");
1326
                }
1327
            } catch (InvalidToken e) {
1328
                throw new NotAuthorized("2162",
1329
                        "Not allowed to call synchronizationFailed() on this node.");
1330 6331 leinfelder
1331 7075 cjones
            }
1332
1333
        } else {
1334
            throw new ServiceFailure("2161", "The identifier cannot be null.");
1335
1336
        }
1337
1338 6475 jones
        try {
1339 7075 cjones
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1340 6475 jones
        } catch (McdbDocNotFoundException e) {
1341 7075 cjones
            throw new ServiceFailure("2161", "The identifier specified by " +
1342
                    syncFailed.getPid() + " was not found on this node.");
1343 6179 cjones
1344 9024 tao
        } catch (SQLException e) {
1345
            throw new ServiceFailure("2161", "Couldn't identify the local id of the identifier specified by " +
1346
                    syncFailed.getPid() + " since "+e.getMessage());
1347 6475 jones
        }
1348
        // TODO: update the CN URL below when the CNRead.SynchronizationFailed
1349
        // method is changed to include the URL as a parameter
1350 9380 rnahf
        logMetacat.warn("Synchronization for the object identified by " +
1351 7075 cjones
                pid.getValue() + " failed from " + syncFailed.getNodeId() +
1352 9380 rnahf
                " with message: " + syncFailed.getDescription() +
1353
                ". Logging the event to the Metacat EventLog as a 'syncFailed' event.");
1354 6475 jones
        // TODO: use the event type enum when the SYNCHRONIZATION_FAILED event is added
1355 6532 leinfelder
        String principal = Constants.SUBJECT_PUBLIC;
1356 6506 leinfelder
        if (session != null && session.getSubject() != null) {
1357 6575 cjones
          principal = session.getSubject().getValue();
1358 6506 leinfelder
        }
1359
        try {
1360 6575 cjones
          EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), principal, localId, "synchronization_failed");
1361 6506 leinfelder
        } catch (Exception e) {
1362 7075 cjones
            throw new ServiceFailure("2161", "Could not log the error for: " + pid.getValue());
1363 6991 leinfelder
        }
1364 6475 jones
        //EventLog.getInstance().log("CN URL WILL GO HERE",
1365
        //  session.getSubject().getValue(), localId, Event.SYNCHRONIZATION_FAILED);
1366 6991 leinfelder
        return true;
1367 6179 cjones
1368 6260 cjones
    }
1369
1370 6475 jones
    /**
1371
     * Essentially a get() but with different logging behavior
1372
     */
1373
    @Override
1374 6540 cjones
    public InputStream getReplica(Session session, Identifier pid)
1375 9314 tao
        throws NotAuthorized, NotImplemented, ServiceFailure, InvalidToken, NotFound {
1376 6179 cjones
1377 6540 cjones
        logMetacat.info("MNodeService.getReplica() called.");
1378
1379 6653 leinfelder
        // cannot be called by public
1380
        if (session == null) {
1381
        	throw new InvalidToken("2183", "No session was provided.");
1382
        }
1383
1384 6631 cjones
        logMetacat.info("MNodeService.getReplica() called with parameters: \n" +
1385
             "\tSession.Subject      = " + session.getSubject().getValue() + "\n" +
1386
             "\tIdentifier           = " + pid.getValue());
1387
1388 6475 jones
        InputStream inputStream = null; // bytes to be returned
1389
        handler = new MetacatHandler(new Timer());
1390
        boolean allowed = false;
1391
        String localId; // the metacat docid for the pid
1392 6179 cjones
1393 6475 jones
        // get the local docid from Metacat
1394
        try {
1395
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1396
        } catch (McdbDocNotFoundException e) {
1397 9314 tao
            throw new NotFound("2185", "The object specified by " +
1398 6610 cjones
                    pid.getValue() + " does not exist at this node.");
1399
1400 9024 tao
        } catch (SQLException e) {
1401
            throw new ServiceFailure("2181", "The local id of the object specified by " +
1402
                    pid.getValue() + " couldn't be identified since "+e.getMessage());
1403 6475 jones
        }
1404 6234 cjones
1405 6552 leinfelder
        Subject targetNodeSubject = session.getSubject();
1406 6185 leinfelder
1407 6552 leinfelder
        // check for authorization to replicate, null session to act as this source MN
1408 6610 cjones
        try {
1409 6777 leinfelder
            allowed = D1Client.getCN().isNodeAuthorized(null, targetNodeSubject, pid);
1410 6610 cjones
        } catch (InvalidToken e1) {
1411
            throw new ServiceFailure("2181", "Could not determine if node is authorized: "
1412
                + e1.getMessage());
1413
1414
        } catch (NotFound e1) {
1415 9314 tao
            throw new NotFound("2185", "Could not find the object "+pid.getValue()+" in this node - "
1416 6610 cjones
                    + e1.getMessage());
1417 6384 cjones
1418 6610 cjones
        } catch (InvalidRequest e1) {
1419
            throw new ServiceFailure("2181", "Could not determine if node is authorized: "
1420
                    + e1.getMessage());
1421
1422
        }
1423
1424 6540 cjones
        logMetacat.info("Called D1Client.isNodeAuthorized(). Allowed = " + allowed +
1425
            " for identifier " + pid.getValue());
1426
1427 6475 jones
        // if the person is authorized, perform the read
1428
        if (allowed) {
1429
            try {
1430 6986 jones
                inputStream = MetacatHandler.read(localId);
1431 6475 jones
            } catch (Exception e) {
1432 9314 tao
                throw new ServiceFailure("2181", "The object specified by " +
1433 6610 cjones
                    pid.getValue() + "could not be returned due to error: " + e.getMessage());
1434 6475 jones
            }
1435 9314 tao
        } else {
1436
            throw new NotAuthorized("2182", "The pid "+pid.getValue()+" is not authorized to be read by the client.");
1437 6475 jones
        }
1438 6384 cjones
1439 6475 jones
        // if we fail to set the input stream
1440
        if (inputStream == null) {
1441 6610 cjones
            throw new ServiceFailure("2181", "The object specified by " +
1442 9314 tao
                pid.getValue() + " can't be returned from the node.");
1443 6475 jones
        }
1444
1445
        // log the replica event
1446
        String principal = null;
1447
        if (session.getSubject() != null) {
1448
            principal = session.getSubject().getValue();
1449
        }
1450 6576 cjones
        EventLog.getInstance().log(request.getRemoteAddr(),
1451
            request.getHeader("User-Agent"), principal, localId, "replicate");
1452 6475 jones
1453
        return inputStream;
1454
    }
1455 9340 tao
1456 6573 cjones
    /**
1457 6600 cjones
     * A method to notify the Member Node that the authoritative copy of
1458 6599 cjones
     * system metadata on the Coordinating Nodes has changed.
1459 9340 tao
     *
1460 6599 cjones
     * @param session   Session information that contains the identity of the
1461
     *                  calling user as retrieved from the X.509 certificate
1462
     *                  which must be traceable to the CILogon service.
1463
     * @param serialVersion   The serialVersion of the system metadata
1464
     * @param dateSysMetaLastModified  The time stamp for when the system metadata was changed
1465
     * @throws NotImplemented
1466
     * @throws ServiceFailure
1467
     * @throws NotAuthorized
1468
     * @throws InvalidRequest
1469
     * @throws InvalidToken
1470
     */
1471 6991 leinfelder
    public boolean systemMetadataChanged(Session session, Identifier pid,
1472 6599 cjones
        long serialVersion, Date dateSysMetaLastModified)
1473
        throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest,
1474
        InvalidToken {
1475 9340 tao
        boolean needCheckAuthoriativeNode = true;
1476
        return systemMetadataChanged(needCheckAuthoriativeNode, session, pid,serialVersion, dateSysMetaLastModified);
1477
    }
1478
1479
    /**
1480
     * A method to notify the Member Node that the authoritative copy of
1481
     * system metadata on the Coordinating Nodes has changed.
1482
     * @param needCheckAuthoriativeNode  this is for the dataone version 2. In the
1483
     * version 2, there are two scenarios:
1484
     * 1. If the node is the authoritative node, it only accepts serial version and replica list.
1485
     * 2. If the node is a replica, it accepts everything.
1486
     * For the v1, api, the parameter should be false.
1487
     * @param session   Session information that contains the identity of the
1488
     *                  calling user as retrieved from the X.509 certificate
1489
     *                  which must be traceable to the CILogon service.
1490
     * @param serialVersion   The serialVersion of the system metadata
1491
     * @param dateSysMetaLastModified  The time stamp for when the system metadata was changed
1492
     * @throws NotImplemented
1493
     * @throws ServiceFailure
1494
     * @throws NotAuthorized
1495
     * @throws InvalidRequest
1496
     * @throws InvalidToken
1497
     */
1498
    public boolean systemMetadataChanged(boolean needCheckAuthoriativeNode, Session session, Identifier pid,
1499
        long serialVersion, Date dateSysMetaLastModified)
1500
        throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest,
1501
        InvalidToken {
1502 6599 cjones
1503 9479 tao
        /*if(isReadOnlyMode()) {
1504 9478 tao
            throw new InvalidRequest("1334", "The Metacat member node is on the read-only mode and your request can't be fulfiled. Please try again later.");
1505 9479 tao
        }*/
1506 7600 cjones
        // cannot be called by public
1507
        if (session == null) {
1508 9051 tao
        	throw new InvalidToken("1332", "No session was provided.");
1509 7600 cjones
        }
1510
1511 9051 tao
        String serviceFailureCode = "1333";
1512
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
1513
        if(sid != null) {
1514
            pid = sid;
1515
        }
1516
1517 6600 cjones
        SystemMetadata currentLocalSysMeta = null;
1518
        SystemMetadata newSysMeta = null;
1519
        CNode cn = D1Client.getCN();
1520
        NodeList nodeList = null;
1521
        Subject callingSubject = null;
1522
        boolean allowed = false;
1523
1524
        // are we allowed to call this?
1525
        callingSubject = session.getSubject();
1526
        nodeList = cn.listNodes();
1527
1528
        for(Node node : nodeList.getNodeList()) {
1529
            // must be a CN
1530
            if ( node.getType().equals(NodeType.CN)) {
1531
               List<Subject> subjectList = node.getSubjectList();
1532
               // the calling subject must be in the subject list
1533
               if ( subjectList.contains(callingSubject)) {
1534
                   allowed = true;
1535
1536
               }
1537
1538
            }
1539
        }
1540
1541
        if (!allowed ) {
1542
            String msg = "The subject identified by " + callingSubject.getValue() +
1543
              " is not authorized to call this service.";
1544
            throw new NotAuthorized("1331", msg);
1545
1546
        }
1547
        try {
1548 9340 tao
            HazelcastService.getInstance().getSystemMetadataMap().lock(pid);
1549 6600 cjones
1550 9340 tao
            // compare what we have locally to what is sent in the change notification
1551 6600 cjones
            try {
1552 9340 tao
                currentLocalSysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
1553
1554
            } catch (RuntimeException e) {
1555
                String msg = "SystemMetadata for pid " + pid.getValue() +
1556
                  " couldn't be updated because it couldn't be found locally: " +
1557
                  e.getMessage();
1558 6692 leinfelder
                logMetacat.error(msg);
1559 9340 tao
                ServiceFailure sf = new ServiceFailure("1333", msg);
1560 6692 leinfelder
                sf.initCause(e);
1561 9340 tao
                throw sf;
1562 6600 cjones
            }
1563 6692 leinfelder
1564 9340 tao
            if(currentLocalSysMeta == null) {
1565
                throw new InvalidRequest("1334", "We can't find the system metadata in the node for the id "+pid.getValue());
1566
            }
1567 9382 tao
            if (currentLocalSysMeta.getSerialVersion().longValue() <= serialVersion ) {
1568 9340 tao
                try {
1569
                    newSysMeta = cn.getSystemMetadata(null, pid);
1570
                } catch (NotFound e) {
1571
                    // huh? you just said you had it
1572
                	String msg = "On updating the local copy of system metadata " +
1573
                    "for pid " + pid.getValue() +", the CN reports it is not found." +
1574
                    " The error message was: " + e.getMessage();
1575
                    logMetacat.error(msg);
1576
                    //ServiceFailure sf = new ServiceFailure("1333", msg);
1577
                    InvalidRequest sf = new InvalidRequest("1334", msg);
1578
                    sf.initCause(e);
1579
                    throw sf;
1580 9056 tao
                }
1581 9340 tao
1582
                //check about the sid in the system metadata
1583
                Identifier newSID = newSysMeta.getSeriesId();
1584
                if(newSID != null) {
1585
                    if (!isValidIdentifier(newSID)) {
1586
                        throw new InvalidRequest("1334", "The series identifier in the new system metadata is invalid.");
1587
                    }
1588
                    Identifier currentSID = currentLocalSysMeta.getSeriesId();
1589
                    if( currentSID != null && currentSID.getValue() != null) {
1590
                        if(!newSID.getValue().equals(currentSID.getValue())) {
1591
                            //newSID doesn't match the currentSID. The newSID shouldn't be used.
1592
                            try {
1593
                                if(IdentifierManager.getInstance().identifierExists(newSID.getValue())) {
1594
                                    throw new InvalidRequest("1334", "The series identifier "+newSID.getValue()+" in the new system metadata has been used by another object.");
1595
                                }
1596
                            } catch (SQLException sql) {
1597
                                throw new ServiceFailure("1333", "Couldn't determine if the SID "+newSID.getValue()+" in the system metadata exists in the node since "+sql.getMessage());
1598
                            }
1599
1600
                        }
1601
                    } else {
1602
                        //newSID shouldn't be used
1603 9056 tao
                        try {
1604
                            if(IdentifierManager.getInstance().identifierExists(newSID.getValue())) {
1605
                                throw new InvalidRequest("1334", "The series identifier "+newSID.getValue()+" in the new system metadata has been used by another object.");
1606
                            }
1607
                        } catch (SQLException sql) {
1608
                            throw new ServiceFailure("1333", "Couldn't determine if the SID "+newSID.getValue()+" in the system metadata exists in the node since "+sql.getMessage());
1609
                        }
1610
                    }
1611 9340 tao
                }
1612
                // update the local copy of system metadata for the pid
1613
                try {
1614
                    if(needCheckAuthoriativeNode) {
1615
                        //this is for the v2 api.
1616
                        if(isAuthoritativeNode(pid)) {
1617
                            //this is the authoritative node, so we only accept replica and serial version
1618 9375 tao
                            logMetacat.debug("MNodeService.systemMetadataChanged - this is the authoritative node for the pid "+pid.getValue());
1619 9340 tao
                            List<Replica> replicas = newSysMeta.getReplicaList();
1620
                            newSysMeta = currentLocalSysMeta;
1621
                            newSysMeta.setSerialVersion(new BigInteger((new Long(serialVersion)).toString()));
1622
                            newSysMeta.setReplicaList(replicas);
1623 9375 tao
                        } else {
1624
                            //we need to archive the object in the replica node
1625
                            logMetacat.debug("MNodeService.systemMetadataChanged - this is NOT the authoritative node for the pid "+pid.getValue());
1626
                            logMetacat.debug("MNodeService.systemMetadataChanged - the new value of archive is "+newSysMeta.getArchived()+" for the pid "+pid.getValue());
1627
                            logMetacat.debug("MNodeService.systemMetadataChanged - the local value of archive is "+currentLocalSysMeta.getArchived()+" for the pid "+pid.getValue());
1628
                            if (newSysMeta.getArchived() != null && newSysMeta.getArchived() == true  &&
1629
                                    ((currentLocalSysMeta.getArchived() != null && currentLocalSysMeta.getArchived() == false ) || currentLocalSysMeta.getArchived() == null)){
1630
                                logMetacat.debug("MNodeService.systemMetadataChanged - start to archive object "+pid.getValue());
1631
                                boolean logArchive = false;
1632
                                boolean needUpdateModificationDate = false;
1633
                                try {
1634
                                    archiveObject(logArchive, session, pid, newSysMeta, needUpdateModificationDate);
1635
                                } catch (NotFound e) {
1636
                                    throw new InvalidRequest("1334", "Can't find the pid "+pid.getValue()+" for archive.");
1637
                                }
1638
1639 9384 tao
                            } else if((newSysMeta.getArchived() == null || newSysMeta.getArchived() == false) && (currentLocalSysMeta.getArchived() != null && currentLocalSysMeta.getArchived() == true )) {
1640
                                throw new InvalidRequest("1334", "The pid "+pid.getValue()+" has been archived and it can't be reset to false.");
1641 9375 tao
                            }
1642 9056 tao
                        }
1643
                    }
1644 9340 tao
                    HazelcastService.getInstance().getSystemMetadataMap().put(newSysMeta.getIdentifier(), newSysMeta);
1645
                    logMetacat.info("Updated local copy of system metadata for pid " +
1646
                        pid.getValue() + " after change notification from the CN.");
1647
1648
                    // TODO: consider inspecting the change for archive
1649
                    // see: https://projects.ecoinformatics.org/ecoinfo/issues/6417
1650
    //                if (newSysMeta.getArchived() != null && newSysMeta.getArchived().booleanValue()) {
1651
    //                	try {
1652
    //						this.archive(session, newSysMeta.getIdentifier());
1653
    //					} catch (NotFound e) {
1654
    //						// do we care? nothing to do about it now
1655
    //						logMetacat.error(e.getMessage(), e);
1656
    //					}
1657
    //                }
1658
1659
                } catch (RuntimeException e) {
1660
                    String msg = "SystemMetadata for pid " + pid.getValue() +
1661
                      " couldn't be updated: " +
1662
                      e.getMessage();
1663
                    logMetacat.error(msg);
1664
                    ServiceFailure sf = new ServiceFailure("1333", msg);
1665
                    sf.initCause(e);
1666
                    throw sf;
1667 9056 tao
                }
1668 6600 cjones
1669 9375 tao
                try {
1670
                    String localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1671
                    EventLog.getInstance().log(request.getRemoteAddr(),
1672
                            request.getHeader("User-Agent"), session.getSubject().getValue(),
1673
                            localId, "updateSystemMetadata");
1674
                } catch (Exception e) {
1675
                    // do nothing, no localId to log with
1676
                    logMetacat.warn("MNodeService.systemMetadataChanged - Could not log 'updateSystemMetadata' event because no localId was found for pid: " + pid.getValue());
1677
                }
1678
1679 9340 tao
1680 6600 cjones
            }
1681 9340 tao
        } finally {
1682
            HazelcastService.getInstance().getSystemMetadataMap().unlock(pid);
1683
        }
1684
1685 9382 tao
        if (currentLocalSysMeta.getSerialVersion().longValue() <= serialVersion ) {
1686 9248 leinfelder
            // attempt to re-register the identifier (it checks if it is a doi)
1687
            try {
1688 9340 tao
                DOIService.getInstance().registerDOI(newSysMeta);
1689
            } catch (Exception e) {
1690
                logMetacat.warn("Could not [re]register DOI: " + e.getMessage(), e);
1691
            }
1692 9248 leinfelder
1693 8464 leinfelder
            // submit for indexing
1694
            try {
1695 9340 tao
                MetacatSolrIndex.getInstance().submit(newSysMeta.getIdentifier(), newSysMeta, null, true);
1696
            } catch (Exception e) {
1697 8464 leinfelder
                logMetacat.error("Could not submit changed systemMetadata for indexing, pid: " + newSysMeta.getIdentifier().getValue(), e);
1698 9340 tao
            }
1699 6600 cjones
        }
1700
1701 6991 leinfelder
        return true;
1702
1703 6599 cjones
    }
1704
1705 6795 cjones
    /*
1706
     * Set the replication status for the object on the Coordinating Node
1707
     *
1708
     * @param session - the session for the this target node
1709
     * @param pid - the identifier of the object being updated
1710
     * @param nodeId - the identifier of this target node
1711
     * @param status - the replication status to set
1712
     * @param failure - the exception to include, if any
1713
     */
1714
    private void setReplicationStatus(Session session, Identifier pid,
1715
        NodeReference nodeId, ReplicationStatus status, BaseException failure)
1716
        throws ServiceFailure, NotImplemented, NotAuthorized,
1717
        InvalidRequest {
1718
1719
        // call the CN as the MN to set the replication status
1720
        try {
1721
            this.cn = D1Client.getCN();
1722
            this.cn.setReplicationStatus(session, pid, nodeId,
1723
                    status, failure);
1724
1725
        } catch (InvalidToken e) {
1726 7091 leinfelder
        	String msg = "Could not set the replication status for " + pid.getValue() + " on the CN (InvalidToken): " + e.getMessage();
1727
            logMetacat.error(msg);
1728
        	throw new ServiceFailure("2151",
1729
                    msg);
1730 6795 cjones
1731
        } catch (NotFound e) {
1732 7091 leinfelder
        	String msg = "Could not set the replication status for " + pid.getValue() + " on the CN (NotFound): " + e.getMessage();
1733
            logMetacat.error(msg);
1734
        	throw new ServiceFailure("2151",
1735
                    msg);
1736 6795 cjones
1737
        }
1738
    }
1739 8591 leinfelder
1740
    private SystemMetadata makePublicIfNot(SystemMetadata sysmeta, Identifier pid) throws ServiceFailure, InvalidToken, NotFound, NotImplemented, InvalidRequest {
1741
    	// check if it is publicly readable
1742
		boolean isPublic = false;
1743
		Subject publicSubject = new Subject();
1744
		publicSubject.setValue(Constants.SUBJECT_PUBLIC);
1745
		Session publicSession = new Session();
1746
		publicSession.setSubject(publicSubject);
1747
		AccessRule publicRule = new AccessRule();
1748
		publicRule.addPermission(Permission.READ);
1749
		publicRule.addSubject(publicSubject);
1750
1751
		// see if we need to add the rule
1752
		try {
1753
			isPublic = this.isAuthorized(publicSession, pid, Permission.READ);
1754
		} catch (NotAuthorized na) {
1755
			// well, certainly not authorized for public read!
1756
		}
1757
		if (!isPublic) {
1758
			sysmeta.getAccessPolicy().addAllow(publicRule);
1759
		}
1760
1761
		return sysmeta;
1762
    }
1763 7099 leinfelder
1764
	@Override
1765 7441 leinfelder
	public Identifier generateIdentifier(Session session, String scheme, String fragment)
1766 7099 leinfelder
			throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented,
1767
			InvalidRequest {
1768 7448 leinfelder
1769 8210 leinfelder
		// check for null session
1770
        if (session == null) {
1771
          throw new InvalidToken("2190", "Session is required to generate an Identifier at this Node.");
1772
        }
1773
1774 7441 leinfelder
		Identifier identifier = new Identifier();
1775 7448 leinfelder
1776 7489 leinfelder
		// handle different schemes
1777
		if (scheme.equalsIgnoreCase(UUID_SCHEME)) {
1778
			// UUID
1779
			UUID uuid = UUID.randomUUID();
1780
            identifier.setValue(UUID_PREFIX + uuid.toString());
1781
		} else if (scheme.equalsIgnoreCase(DOI_SCHEME)) {
1782 7512 leinfelder
			// generate a DOI
1783 7448 leinfelder
			try {
1784 7512 leinfelder
				identifier = DOIService.getInstance().generateDOI();
1785 7448 leinfelder
			} catch (EZIDException e) {
1786 7512 leinfelder
				ServiceFailure sf = new ServiceFailure("2191", "Could not generate DOI: " + e.getMessage());
1787
				sf.initCause(e);
1788
				throw sf;
1789 7448 leinfelder
			}
1790 7489 leinfelder
		} else {
1791
			// default if we don't know the scheme
1792
			if (fragment != null) {
1793
				// for now, just autogen with fragment
1794
				String autogenId = DocumentUtil.generateDocumentId(fragment, 0);
1795
				identifier.setValue(autogenId);
1796
			} else {
1797
				// autogen with no fragment
1798
				String autogenId = DocumentUtil.generateDocumentId(0);
1799
				identifier.setValue(autogenId);
1800
			}
1801 7448 leinfelder
		}
1802
1803 7441 leinfelder
		// TODO: reserve the identifier with the CN. We can only do this when
1804
		// 1) the MN is part of a CN cluster
1805
		// 2) the request is from an authenticated user
1806
1807
		return identifier;
1808 7099 leinfelder
	}
1809 7144 leinfelder
1810 8810 leinfelder
1811 7144 leinfelder
1812
	@Override
1813 8810 leinfelder
	public QueryEngineDescription getQueryEngineDescription(Session session, String engine)
1814 7144 leinfelder
			throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented,
1815
			NotFound {
1816 7772 tao
	    if(engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) {
1817 8162 tao
	        if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) {
1818
                throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1819
            }
1820 7634 tao
	        QueryEngineDescription qed = new QueryEngineDescription();
1821 7772 tao
	        qed.setName(EnabledQueryEngines.PATHQUERYENGINE);
1822 7634 tao
	        qed.setQueryEngineVersion("1.0");
1823
	        qed.addAdditionalInfo("This is the traditional structured query for Metacat");
1824
	        Vector<String> pathsForIndexing = null;
1825
	        try {
1826
	            pathsForIndexing = SystemUtil.getPathsForIndexing();
1827
	        } catch (MetacatUtilException e) {
1828
	            logMetacat.warn("Could not get index paths", e);
1829
	        }
1830
	        for (String fieldName: pathsForIndexing) {
1831
	            QueryField field = new QueryField();
1832
	            field.addDescription("Indexed field for path '" + fieldName + "'");
1833
	            field.setName(fieldName);
1834
	            field.setReturnable(true);
1835
	            field.setSearchable(true);
1836
	            field.setSortable(false);
1837
	            // TODO: determine type and multivaluedness
1838
	            field.setType(String.class.getName());
1839
	            //field.setMultivalued(true);
1840
	            qed.addQueryField(field);
1841
	        }
1842
	        return qed;
1843 7772 tao
	    } else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) {
1844 7781 tao
	        if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) {
1845 7772 tao
                throw new NotImplemented("0000", "MNodeService.getQueryEngineDescription - the query engine "+engine +" hasn't been implemented or has been disabled.");
1846 7781 tao
            }
1847 7634 tao
	        try {
1848 7662 tao
	            QueryEngineDescription qed = MetacatSolrEngineDescriptionHandler.getInstance().getQueryEngineDescritpion();
1849 7634 tao
	            return qed;
1850
	        } catch (Exception e) {
1851
	            e.printStackTrace();
1852
	            throw new ServiceFailure("Solr server error", e.getMessage());
1853
	        }
1854
	    } else {
1855
	        throw new NotFound("404", "The Metacat member node can't find the query engine - "+engine);
1856
	    }
1857
1858 7417 leinfelder
	}
1859
1860
	@Override
1861 8810 leinfelder
	public QueryEngineList listQueryEngines(Session session) throws InvalidToken,
1862 7417 leinfelder
			ServiceFailure, NotAuthorized, NotImplemented {
1863
		QueryEngineList qel = new QueryEngineList();
1864 7781 tao
		//qel.addQueryEngine(EnabledQueryEngines.PATHQUERYENGINE);
1865
		//qel.addQueryEngine(EnabledQueryEngines.SOLRENGINE);
1866
		List<String> enables = EnabledQueryEngines.getInstance().getEnabled();
1867 7772 tao
		for(String name : enables) {
1868
		    qel.addQueryEngine(name);
1869 7781 tao
		}
1870 7417 leinfelder
		return qel;
1871
	}
1872
1873
	@Override
1874 8810 leinfelder
	public InputStream query(Session session, String engine, String query) throws InvalidToken,
1875 7417 leinfelder
			ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented,
1876
			NotFound {
1877 7648 tao
	    String user = Constants.SUBJECT_PUBLIC;
1878
        String[] groups= null;
1879 7680 tao
        Set<Subject> subjects = null;
1880 7648 tao
        if (session != null) {
1881
            user = session.getSubject().getValue();
1882 7680 tao
            subjects = AuthUtils.authorizedClientSubjects(session);
1883 7648 tao
            if (subjects != null) {
1884
                List<String> groupList = new ArrayList<String>();
1885
                for (Subject subject: subjects) {
1886
                    groupList.add(subject.getValue());
1887
                }
1888
                groups = groupList.toArray(new String[0]);
1889
            }
1890 7680 tao
        } else {
1891
            //add the public user subject to the set
1892
            Subject subject = new Subject();
1893
            subject.setValue(Constants.SUBJECT_PUBLIC);
1894 7757 leinfelder
            subjects = new HashSet<Subject>();
1895 7680 tao
            subjects.add(subject);
1896 7648 tao
        }
1897 7680 tao
        //System.out.println("====== user is "+user);
1898
        //System.out.println("====== groups are "+groups);
1899 7772 tao
		if (engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) {
1900 8162 tao
		    if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) {
1901
                throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1902
            }
1903 7417 leinfelder
			try {
1904
				DBQuery queryobj = new DBQuery();
1905 7648 tao
1906 7417 leinfelder
				String results = queryobj.performPathquery(query, user, groups);
1907 7757 leinfelder
				ContentTypeByteArrayInputStream ctbais = new ContentTypeByteArrayInputStream(results.getBytes(MetaCatServlet.DEFAULT_ENCODING));
1908
				ctbais.setContentType("text/xml");
1909
				return ctbais;
1910 7417 leinfelder
1911
			} catch (Exception e) {
1912 7757 leinfelder
				throw new ServiceFailure("Pathquery error", e.getMessage());
1913 7417 leinfelder
			}
1914
1915 7772 tao
		} else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) {
1916 7781 tao
		    if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) {
1917 7772 tao
		        throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1918 7781 tao
		    }
1919 7634 tao
		    logMetacat.info("The query is ==================================== \n"+query);
1920 7620 tao
		    try {
1921 7634 tao
1922 7680 tao
                return MetacatSolrIndex.getInstance().query(query, subjects);
1923 7620 tao
            } catch (Exception e) {
1924
                // TODO Auto-generated catch block
1925
                throw new ServiceFailure("Solr server error", e.getMessage());
1926
            }
1927 7417 leinfelder
		}
1928
		return null;
1929
	}
1930 7849 leinfelder
1931
	/**
1932
	 * Given an existing Science Metadata PID, this method mints a DOI
1933
	 * and updates the original object "publishing" the update with the DOI.
1934
	 * This includes updating the ORE map that describes the Science Metadata+data.
1935
	 * TODO: ensure all referenced objects allow public read
1936
	 *
1937
	 * @see https://projects.ecoinformatics.org/ecoinfo/issues/6014
1938
	 *
1939
	 * @param originalIdentifier
1940
	 * @param request
1941 7864 leinfelder
	 * @throws InvalidRequest
1942
	 * @throws NotImplemented
1943
	 * @throws NotAuthorized
1944
	 * @throws ServiceFailure
1945
	 * @throws InvalidToken
1946 7849 leinfelder
	 * @throws NotFound
1947 7864 leinfelder
	 * @throws InvalidSystemMetadata
1948
	 * @throws InsufficientResources
1949
	 * @throws UnsupportedType
1950
	 * @throws IdentifierNotUnique
1951 7849 leinfelder
	 */
1952 9387 walker
	public Identifier publish(Session session, Identifier originalIdentifier) throws InvalidToken,
1953
	ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, NotFound, IdentifierNotUnique,
1954
	UnsupportedType, InsufficientResources, InvalidSystemMetadata, IOException {
1955 7849 leinfelder
1956 9092 tao
	    String serviceFailureCode = "1030";
1957
	    Identifier sid = getPIDForSID(originalIdentifier, serviceFailureCode);
1958
	    if(sid != null) {
1959
	        originalIdentifier = sid;
1960
	    }
1961 8141 leinfelder
		// get the original SM
1962
		SystemMetadata originalSystemMetadata = this.getSystemMetadata(session, originalIdentifier);
1963
1964 8594 leinfelder
		// make copy of it using the marshaller to ensure DEEP copy
1965
		SystemMetadata sysmeta = null;
1966 8141 leinfelder
		try {
1967 8594 leinfelder
			ByteArrayOutputStream baos = new ByteArrayOutputStream();
1968
			TypeMarshaller.marshalTypeToOutputStream(originalSystemMetadata, baos);
1969
			sysmeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, new ByteArrayInputStream(baos.toByteArray()));
1970 8141 leinfelder
		} catch (Exception e) {
1971
			// report as service failure
1972
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1973
			sf.initCause(e);
1974
			throw sf;
1975
		}
1976
1977 7849 leinfelder
		// mint a DOI for the new revision
1978
		Identifier newIdentifier = this.generateIdentifier(session, MNodeService.DOI_SCHEME, null);
1979 8141 leinfelder
1980
		// set new metadata values
1981 7849 leinfelder
		sysmeta.setIdentifier(newIdentifier);
1982
		sysmeta.setObsoletes(originalIdentifier);
1983
		sysmeta.setObsoletedBy(null);
1984
1985 8591 leinfelder
		// ensure it is publicly readable
1986
		sysmeta = makePublicIfNot(sysmeta, originalIdentifier);
1987
1988 9387 walker
		//Get the bytes
1989
		InputStream inputStream = null;
1990
		boolean isScienceMetadata = isScienceMetadata(sysmeta);
1991
		//If it's a science metadata doc, we want to update the packageId first
1992
		if(isScienceMetadata){
1993
			InputStream originalObject = this.get(session, originalIdentifier);
1994
1995
			//Edit the science metadata with the new package Id (EML)
1996
			inputStream = editScienceMetadata(session, originalObject, originalIdentifier, newIdentifier);
1997
		}
1998
		else{
1999
			inputStream = this.get(session, originalIdentifier);
2000
		}
2001 7849 leinfelder
2002
		// update the object
2003 7864 leinfelder
		this.update(session, originalIdentifier, inputStream, newIdentifier, sysmeta);
2004 7849 leinfelder
2005 7864 leinfelder
		// update ORE that references the scimeta
2006 8190 leinfelder
		// first try the naive method, then check the SOLR index
2007 7849 leinfelder
		try {
2008 7864 leinfelder
			String localId = IdentifierManager.getInstance().getLocalId(originalIdentifier.getValue());
2009 7849 leinfelder
2010 7864 leinfelder
			Identifier potentialOreIdentifier = new Identifier();
2011
			potentialOreIdentifier.setValue(SystemMetadataFactory.RESOURCE_MAP_PREFIX + localId);
2012 7849 leinfelder
2013 7864 leinfelder
			InputStream oreInputStream = null;
2014
			try {
2015
				oreInputStream = this.get(session, potentialOreIdentifier);
2016
			} catch (NotFound nf) {
2017
				// this is probably okay for many sci meta data docs
2018
				logMetacat.warn("No potential ORE map found for: " + potentialOreIdentifier.getValue());
2019 8190 leinfelder
				// try the SOLR index
2020 8200 leinfelder
				List<Identifier> potentialOreIdentifiers = this.lookupOreFor(originalIdentifier, false);
2021 8190 leinfelder
				if (potentialOreIdentifiers != null) {
2022
					potentialOreIdentifier = potentialOreIdentifiers.get(0);
2023
					try {
2024
						oreInputStream = this.get(session, potentialOreIdentifier);
2025
					} catch (NotFound nf2) {
2026
						// this is probably okay for many sci meta data docs
2027
						logMetacat.warn("No potential ORE map found for: " + potentialOreIdentifier.getValue());
2028
					}
2029
				}
2030 7864 leinfelder
			}
2031
			if (oreInputStream != null) {
2032
				Identifier newOreIdentifier = MNodeService.getInstance(request).generateIdentifier(session, MNodeService.UUID_SCHEME, null);
2033
2034
				Map<Identifier, Map<Identifier, List<Identifier>>> resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap(oreInputStream);
2035
				Map<Identifier, List<Identifier>> sciMetaMap = resourceMapStructure.get(potentialOreIdentifier);
2036
				List<Identifier> dataIdentifiers = sciMetaMap.get(originalIdentifier);
2037 8591 leinfelder
2038 7864 leinfelder
				// reconstruct the ORE with the new identifiers
2039
				sciMetaMap.remove(originalIdentifier);
2040
				sciMetaMap.put(newIdentifier, dataIdentifiers);
2041
2042
				ResourceMap resourceMap = ResourceMapFactory.getInstance().createResourceMap(newOreIdentifier, sciMetaMap);
2043
				String resourceMapString = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap);
2044
2045
				// get the original ORE SM and update the values
2046 8141 leinfelder
				SystemMetadata originalOreSysMeta = this.getSystemMetadata(session, potentialOreIdentifier);
2047
				SystemMetadata oreSysMeta = new SystemMetadata();
2048
				try {
2049 8594 leinfelder
					ByteArrayOutputStream baos = new ByteArrayOutputStream();
2050
					TypeMarshaller.marshalTypeToOutputStream(originalOreSysMeta, baos);
2051
					oreSysMeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, new ByteArrayInputStream(baos.toByteArray()));
2052 8141 leinfelder
				} catch (Exception e) {
2053
					// report as service failure
2054
					ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2055
					sf.initCause(e);
2056
					throw sf;
2057
				}
2058
2059 7864 leinfelder
				oreSysMeta.setIdentifier(newOreIdentifier);
2060
				oreSysMeta.setObsoletes(potentialOreIdentifier);
2061
				oreSysMeta.setObsoletedBy(null);
2062
				oreSysMeta.setSize(BigInteger.valueOf(resourceMapString.getBytes("UTF-8").length));
2063
				oreSysMeta.setChecksum(ChecksumUtil.checksum(resourceMapString.getBytes("UTF-8"), oreSysMeta.getChecksum().getAlgorithm()));
2064
2065 8591 leinfelder
				// ensure ORE is publicly readable
2066 8594 leinfelder
				oreSysMeta = makePublicIfNot(oreSysMeta, potentialOreIdentifier);
2067 8591 leinfelder
2068
				// ensure all data objects allow public read
2069
				List<String> pidsToSync = new ArrayList<String>();
2070
				for (Identifier dataId: dataIdentifiers) {
2071
					SystemMetadata dataSysMeta = this.getSystemMetadata(session, dataId);
2072
					dataSysMeta = makePublicIfNot(dataSysMeta, dataId);
2073
					this.updateSystemMetadata(dataSysMeta);
2074
					pidsToSync.add(dataId.getValue());
2075
				}
2076
				SyncAccessPolicy sap = new SyncAccessPolicy();
2077
				try {
2078
					sap.sync(pidsToSync);
2079
				} catch (Exception e) {
2080
					// ignore
2081
					logMetacat.warn("Error attempting to sync access for data objects when publishing package");
2082
				}
2083
2084 7864 leinfelder
				// save the updated ORE
2085
				this.update(
2086
						session,
2087
						potentialOreIdentifier,
2088
						new ByteArrayInputStream(resourceMapString.getBytes("UTF-8")),
2089
						newOreIdentifier,
2090
						oreSysMeta);
2091
2092 8362 leinfelder
			} else {
2093
				// create a new ORE for them
2094
				// https://projects.ecoinformatics.org/ecoinfo/issues/6194
2095
				try {
2096
					// find the local id for the NEW package.
2097
					String newLocalId = IdentifierManager.getInstance().getLocalId(newIdentifier.getValue());
2098
2099
					@SuppressWarnings("unused")
2100
					SystemMetadata extraSysMeta = SystemMetadataFactory.createSystemMetadata(newLocalId, true, false);
2101 8591 leinfelder
					// should be done generating the ORE here, and the same permissions were used from the metadata object
2102 8362 leinfelder
2103
				} catch (Exception e) {
2104
					// oops, guess there was a problem - no package for you
2105
					logMetacat.error("Could not generate new ORE for published object: " + newIdentifier.getValue(), e);
2106
				}
2107 7864 leinfelder
			}
2108
		} catch (McdbDocNotFoundException e) {
2109
			// report as service failure
2110
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2111
			sf.initCause(e);
2112
			throw sf;
2113
		} catch (UnsupportedEncodingException e) {
2114
			// report as service failure
2115
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2116
			sf.initCause(e);
2117
			throw sf;
2118
		} catch (OREException e) {
2119
			// report as service failure
2120
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2121
			sf.initCause(e);
2122
			throw sf;
2123
		} catch (URISyntaxException e) {
2124
			// report as service failure
2125
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2126
			sf.initCause(e);
2127
			throw sf;
2128
		} catch (OREParserException e) {
2129
			// report as service failure
2130
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2131
			sf.initCause(e);
2132
			throw sf;
2133
		} catch (ORESerialiserException e) {
2134
			// report as service failure
2135
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2136
			sf.initCause(e);
2137
			throw sf;
2138
		} catch (NoSuchAlgorithmException e) {
2139
			// report as service failure
2140
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2141
			sf.initCause(e);
2142
			throw sf;
2143 9024 tao
		} catch (SQLException e) {
2144
            // report as service failure
2145
            ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2146
            sf.initCause(e);
2147
            throw sf;
2148
        }
2149 7849 leinfelder
2150
		return newIdentifier;
2151
	}
2152 7850 leinfelder
2153
	/**
2154 9387 walker
	   * Update a science metadata document with its new Identifier
2155
	   *
2156
	   * @param session - the Session object containing the credentials for the Subject
2157
	   * @param object - the InputStream for the XML object to be edited
2158
	   * @param pid - the Identifier of the XML object to be updated
2159
	   * @param newPid = the new Identifier to give to the modified XML doc
2160
	   *
2161
	   * @return newObject - The InputStream for the modified XML object
2162
	   *
2163
	   * @throws ServiceFailure
2164
	   * @throws IOException
2165
	   * @throws UnsupportedEncodingException
2166
	   * @throws InvalidToken
2167
	   * @throws NotAuthorized
2168
	   * @throws NotFound
2169
	   * @throws NotImplemented
2170
	   */
2171
	  public InputStream editScienceMetadata(Session session, InputStream object, Identifier pid, Identifier newPid)
2172
	  	throws ServiceFailure, IOException, UnsupportedEncodingException, InvalidToken, NotAuthorized, NotFound, NotImplemented {
2173
2174
		logMetacat.debug("D1NodeService.editScienceMetadata() called.");
2175
2176
		 InputStream newObject = null;
2177
2178
	    try{
2179
	    	//Get the root node of the XML document
2180
	    	byte[] xmlBytes  = IOUtils.toByteArray(object);
2181
	        String xmlStr = new String(xmlBytes, "UTF-8");
2182
2183
	    	Document doc = XMLUtilities.getXMLReaderAsDOMDocument(new StringReader(xmlStr));
2184
		    org.w3c.dom.Node docNode = doc.getDocumentElement();
2185
2186
		    //Get the system metadata for this object
2187
		    SystemMetadata sysMeta = null;
2188
		    try{
2189
		    	sysMeta = getSystemMetadata(session, pid);
2190
		    } catch(NotAuthorized e){
2191
		    	throw new ServiceFailure("1030", "D1NodeService.editScienceMetadata(): " +
2192
		    			"This session is not authorized to access the system metadata for " +
2193
		    			pid.getValue() + " : " + e.getMessage());
2194
		    } catch(NotFound e){
2195
		    	throw new ServiceFailure("1030", "D1NodeService.editScienceMetadata(): " +
2196
		    			"Could not find the system metadata for " +
2197
		    			pid.getValue() + " : " + e.getMessage());
2198
		    }
2199
2200
		    //Get the formatId
2201
	        ObjectFormatIdentifier objFormatId = sysMeta.getFormatId();
2202
	        String formatId = objFormatId.getValue();
2203
2204
	    	//For all EML formats
2205
	        if(formatId.indexOf("eml") == 0){
2206
	        	//Update or add the id attribute
2207
	    	    XMLUtilities.addAttributeNodeToDOMTree(docNode, XPATH_EML_ID, newPid.getValue());
2208
	        }
2209
2210
	        //The modified object InputStream
2211
		    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
2212
		    Source xmlSource = new DOMSource(docNode);
2213
		    Result outputTarget = new StreamResult(outputStream);
2214
		    TransformerFactory.newInstance().newTransformer().transform(xmlSource, outputTarget);
2215
		    newObject = new ByteArrayInputStream(outputStream.toByteArray());
2216
2217
	    } catch(TransformerException e) {
2218
	    	throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " +
2219
	                "Could not update the ID in the XML document for " +
2220
	                "pid " + pid.getValue() +" : " + e.getMessage());
2221
	    } catch(IOException e){
2222
	    	throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " +
2223
	                "Could not update the ID in the XML document for " +
2224
	                "pid " + pid.getValue() +" : " + e.getMessage());
2225
	    }
2226
2227
	    return newObject;
2228
	  }
2229
2230
	/**
2231 8190 leinfelder
	 * Determines if we already have registered an ORE map for this package
2232
	 * NOTE: uses a solr query to locate OREs for the object
2233
	 * @param guid of the EML/packaging object
2234
	 * @return list of resource map identifiers for the given pid
2235
	 */
2236 8200 leinfelder
	public List<Identifier> lookupOreFor(Identifier guid, boolean includeObsolete) {
2237 8190 leinfelder
		// Search for the ORE if we can find it
2238
		String pid = guid.getValue();
2239
		List<Identifier> retList = null;
2240
		try {
2241 8745 leinfelder
			String query = "fl=id,resourceMap&wt=xml&q=-obsoletedBy:[* TO *]+resourceMap:[* TO *]+id:\"" + pid + "\"";
2242 8200 leinfelder
			if (includeObsolete) {
2243 8745 leinfelder
				query = "fl=id,resourceMap&wt=xml&q=resourceMap:[* TO *]+id:\"" + pid + "\"";
2244 8200 leinfelder
			}
2245
2246 8810 leinfelder
			InputStream results = this.query(null, "solr", query);
2247 8190 leinfelder
			org.w3c.dom.Node rootNode = XMLUtilities.getXMLReaderAsDOMTreeRootNode(new InputStreamReader(results, "UTF-8"));
2248
			//String resultString = XMLUtilities.getDOMTreeAsString(rootNode);
2249
			org.w3c.dom.NodeList nodeList = XMLUtilities.getNodeListWithXPath(rootNode, "//arr[@name=\"resourceMap\"]/str");
2250
			if (nodeList != null && nodeList.getLength() > 0) {
2251
				retList = new ArrayList<Identifier>();
2252
				for (int i = 0; i < nodeList.getLength(); i++) {
2253
					String found = nodeList.item(i).getFirstChild().getNodeValue();
2254
					Identifier oreId = new Identifier();
2255
					oreId.setValue(found);
2256
					retList.add(oreId);
2257
				}
2258
			}
2259
		} catch (Exception e) {
2260
			logMetacat.error("Error checking for resourceMap[s] on pid " + pid + ". " + e.getMessage(), e);
2261
		}
2262
2263
		return retList;
2264
	}
2265
2266 8810 leinfelder
2267
	@Override
2268
	public InputStream getPackage(Session session, ObjectFormatIdentifier formatId,
2269
			Identifier pid) throws InvalidToken, ServiceFailure,
2270
			NotAuthorized, InvalidRequest, NotImplemented, NotFound {
2271 9290 tao
	    if(formatId == null) {
2272
	        throw new InvalidRequest("2873", "The format type can't be null in the getpackage method.");
2273 9305 tao
	    } else if(!formatId.getValue().equals("application/bagit-097")) {
2274 9290 tao
	        throw new NotImplemented("", "The format "+formatId.getValue()+" is not supported in the getpackage method");
2275
	    }
2276 9092 tao
	    String serviceFailureCode = "2871";
2277
	    Identifier sid = getPIDForSID(pid, serviceFailureCode);
2278
	    if(sid != null) {
2279
	        pid = sid;
2280
	    }
2281 7850 leinfelder
		InputStream bagInputStream = null;
2282
		BagFactory bagFactory = new BagFactory();
2283
		Bag bag = bagFactory.createBag();
2284
2285
		// track the temp files we use so we can delete them when finished
2286
		List<File> tempFiles = new ArrayList<File>();
2287
2288
		// the pids to include in the package
2289
		List<Identifier> packagePids = new ArrayList<Identifier>();
2290
2291 7855 leinfelder
		// catch non-D1 service errors and throw as ServiceFailures
2292
		try {
2293 8437 walker
			//Create a map of dataone ids and file names
2294
			Map<Identifier, String> fileNames = new HashMap<Identifier, String>();
2295 7855 leinfelder
2296 8800 leinfelder
			// track the pid-to-file mapping
2297
			StringBuffer pidMapping = new StringBuffer();
2298
2299 7855 leinfelder
			// find the package contents
2300
			SystemMetadata sysMeta = this.getSystemMetadata(session, pid);
2301 8025 leinfelder
			if (ObjectFormatCache.getInstance().getFormat(sysMeta.getFormatId()).getFormatType().equals("RESOURCE")) {
2302 8437 walker
				//Get the resource map as a map of Identifiers
2303 7855 leinfelder
				InputStream oreInputStream = this.get(session, pid);
2304
				Map<Identifier, Map<Identifier, List<Identifier>>> resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap(oreInputStream);
2305
				packagePids.addAll(resourceMapStructure.keySet());
2306 8437 walker
				//Loop through each object in this resource map
2307 7855 leinfelder
				for (Map<Identifier, List<Identifier>> entries: resourceMapStructure.values()) {
2308 8437 walker
					//Loop through each metadata object in this entry
2309
					Set<Identifier> metadataIdentifiers = entries.keySet();
2310
					for(Identifier metadataID: metadataIdentifiers){
2311
						try{
2312
							//Get the system metadata for this metadata object
2313
							SystemMetadata metadataSysMeta = this.getSystemMetadata(session, metadataID);
2314
2315 8800 leinfelder
							// include user-friendly metadata
2316
							if (ObjectFormatCache.getInstance().getFormat(metadataSysMeta.getFormatId()).getFormatType().equals("METADATA")) {
2317
								InputStream metadataStream = this.get(session, metadataID);
2318
2319
								try {
2320
									// transform
2321
						            String format = "default";
2322
2323
									DBTransform transformer = new DBTransform();
2324
						            String documentContent = IOUtils.toString(metadataStream, "UTF-8");
2325
						            String sourceType = metadataSysMeta.getFormatId().getValue();
2326
						            String targetType = "-//W3C//HTML//EN";
2327
						            ByteArrayOutputStream baos = new ByteArrayOutputStream();
2328
						            Writer writer = new OutputStreamWriter(baos , "UTF-8");
2329
						            // TODO: include more params?
2330
						            Hashtable<String, String[]> params = new Hashtable<String, String[]>();
2331
						            String localId = null;
2332
									try {
2333
										localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
2334
									} catch (McdbDocNotFoundException e) {
2335
										throw new NotFound("1020", e.getMessage());
2336
									}
2337
									params.put("qformat", new String[] {format});
2338
						            params.put("docid", new String[] {localId});
2339
						            params.put("pid", new String[] {pid.getValue()});
2340
						            params.put("displaymodule", new String[] {"printall"});
2341
2342
						            transformer.transformXMLDocument(
2343
						                    documentContent ,
2344
						                    sourceType,
2345
						                    targetType ,
2346
						                    format,
2347
						                    writer,
2348
						                    params,
2349
						                    null //sessionid
2350
						                    );
2351
2352
						            // finally, get the HTML back
2353
						            ContentTypeByteArrayInputStream resultInputStream = new ContentTypeByteArrayInputStream(baos.toByteArray());
2354
2355
						            // write to temp file with correct css path
2356
						            File tmpDir = File.createTempFile("package_", "_dir");
2357
						            tmpDir.delete();
2358
						            tmpDir.mkdir();
2359
						            File htmlFile = File.createTempFile("metadata", ".html", tmpDir);
2360
						            File cssDir = new File(tmpDir, format);
2361
						            cssDir.mkdir();
2362
						            File cssFile = new File(tmpDir, format + "/" + format + ".css");
2363
						            String pdfFileName = metadataID.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-METADATA.pdf";
2364
						            File pdfFile = new File(tmpDir, pdfFileName);
2365
						            //File pdfFile = File.createTempFile("metadata", ".pdf", tmpDir);
2366
2367
						            // put the CSS file in place for the html to find it
2368
						            String originalCssPath = SystemUtil.getContextDir() + "/style/skins/" + format + "/" + format + ".css";
2369
						            IOUtils.copy(new FileInputStream(originalCssPath), new FileOutputStream(cssFile));
2370
2371
						            // write the HTML file
2372
						            IOUtils.copy(resultInputStream, new FileOutputStream(htmlFile));
2373
2374
						            // convert to PDF
2375
						            HtmlToPdf.export(htmlFile.getAbsolutePath(), pdfFile.getAbsolutePath());
2376
2377
						            //add to the package
2378
						            bag.addFileToPayload(pdfFile);
2379
									pidMapping.append(metadataID.getValue() + " (pdf)" +  "\t" + "data/" + pdfFile.getName() + "\n");
2380
2381
						            // mark for clean up after we are done
2382
									htmlFile.delete();
2383
									cssFile.delete();
2384
									cssDir.delete();
2385
						            tempFiles.add(tmpDir);
2386
									tempFiles.add(pdfFile); // delete this first later on
2387
2388
								} catch (Exception e) {
2389
									logMetacat.warn("Could not transform metadata", e);
2390
								}
2391
							}
2392
2393
2394 8437 walker
							//If this is in eml format, extract the filename and GUID from each entity in its package
2395
							if (metadataSysMeta.getFormatId().getValue().startsWith("eml://")) {
2396
								//Get the package
2397
								DataPackageParserInterface parser = new Eml200DataPackageParser();
2398
								InputStream emlStream = this.get(session, metadataID);
2399
								parser.parse(emlStream);
2400
								DataPackage dataPackage = parser.getDataPackage();
2401
2402
								//Get all the entities in this package and loop through each to extract its ID and file name
2403
								Entity[] entities = dataPackage.getEntityList();
2404
								for(Entity entity: entities){
2405
									try{
2406
										//Get the file name from the metadata
2407
										String fileNameFromMetadata = entity.getName();
2408
2409
										//Get the ecogrid URL from the metadata
2410
										String ecogridIdentifier = entity.getEntityIdentifier();
2411
										//Parse the ecogrid URL to get the local id
2412
										String idFromMetadata = DocumentUtil.getAccessionNumberFromEcogridIdentifier(ecogridIdentifier);
2413
2414
										//Get the docid and rev pair
2415
										String docid = DocumentUtil.getDocIdFromString(idFromMetadata);
2416
										String rev = DocumentUtil.getRevisionStringFromString(idFromMetadata);
2417
2418
										//Get the GUID
2419
										String guid = IdentifierManager.getInstance().getGUID(docid, Integer.valueOf(rev));
2420
										Identifier dataIdentifier = new Identifier();
2421
										dataIdentifier.setValue(guid);
2422
2423
										//Add the GUID to our GUID & file name map
2424
										fileNames.put(dataIdentifier, fileNameFromMetadata);
2425
									}
2426
									catch(Exception e){
2427
										//Prevent just one entity error
2428
										e.printStackTrace();
2429
										logMetacat.debug(e.getMessage(), e);
2430
									}
2431
								}
2432
							}
2433
						}
2434
						catch(Exception e){
2435
							//Catch errors that would prevent package download
2436
							logMetacat.debug(e.toString());
2437
						}
2438
					}
2439 7855 leinfelder
					packagePids.addAll(entries.keySet());
2440
					for (List<Identifier> dataPids: entries.values()) {
2441
						packagePids.addAll(dataPids);
2442
					}
2443 7850 leinfelder
				}
2444 7855 leinfelder
			} else {
2445
				// just the lone pid in this package
2446 9493 tao
				//packagePids.add(pid);
2447
			    //throw an invalid request exception
2448 9494 tao
			    throw new InvalidRequest("2873", "The given pid "+pid.getValue()+" is not a package id (resource map id). Please use a package id instead.");
2449 7850 leinfelder
			}
2450 8436 walker
2451 8437 walker
			//Create a temp file, then delete it and make a directory with that name
2452
			File tempDir = File.createTempFile("temp", Long.toString(System.nanoTime()));
2453
			tempDir.delete();
2454
			tempDir = new File(tempDir.getPath() + "_dir");
2455
			tempDir.mkdir();
2456 8436 walker
			tempFiles.add(tempDir);
2457 8800 leinfelder
			File pidMappingFile = new File(tempDir, "pid-mapping.txt");
2458 8436 walker
2459 7855 leinfelder
			// loop through the package contents
2460
			for (Identifier entryPid: packagePids) {
2461 8436 walker
				//Get the system metadata for each item
2462 8437 walker
				SystemMetadata entrySysMeta = this.getSystemMetadata(session, entryPid);
2463 8436 walker
2464
				String objectFormatType = ObjectFormatCache.getInstance().getFormat(entrySysMeta.getFormatId()).getFormatType();
2465 8437 walker
				String fileName = null;
2466 8436 walker
2467 8437 walker
				//TODO: Be more specific of what characters to replace. Make sure periods arent replaced for the filename from metadata
2468
				//Our default file name is just the ID + format type (e.g. walker.1.1-DATA)
2469
				fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + objectFormatType;
2470
2471 9542 leinfelder
				if (fileNames.containsKey(entryPid)){
2472 8437 walker
					//Let's use the file name and extension from the metadata is we have it
2473
					fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + fileNames.get(entryPid).replaceAll("[^a-zA-Z0-9\\-\\.]", "_");
2474
				}
2475
2476 9541 leinfelder
				// ensure there is a file extension for the object
2477
				String extension = ObjectFormatInfo.instance().getExtension(entrySysMeta.getFormatId().getValue());
2478
				fileName += extension;
2479
2480 9542 leinfelder
				// if SM has the file name, ignore everything else and use that
2481
				if (entrySysMeta.getFileName() != null) {
2482
					fileName = entrySysMeta.getFileName().replaceAll("[^a-zA-Z0-9\\-\\.]", "_");
2483
				}
2484
2485 8436 walker
		        //Create a new file for this item and add to the list
2486 8437 walker
				File tempFile = new File(tempDir, fileName);
2487 7855 leinfelder
				tempFiles.add(tempFile);
2488 8436 walker
2489
				InputStream entryInputStream = this.get(session, entryPid);
2490 7855 leinfelder
				IOUtils.copy(entryInputStream, new FileOutputStream(tempFile));
2491
				bag.addFileToPayload(tempFile);
2492
				pidMapping.append(entryPid.getValue() + "\t" + "data/" + tempFile.getName() + "\n");
2493
			}
2494
2495
			//add the the pid to data file map
2496
			IOUtils.write(pidMapping.toString(), new FileOutputStream(pidMappingFile));
2497
			bag.addFileAsTag(pidMappingFile);
2498 8026 leinfelder
			tempFiles.add(pidMappingFile);
2499
2500 7855 leinfelder
			bag = bag.makeComplete();
2501 8026 leinfelder
2502 8436 walker
			///Now create the zip file
2503 8437 walker
			//Use the pid as the file name prefix, replacing all non-word characters
2504
			String zipName = pid.getValue().replaceAll("\\W", "_");
2505 8160 leinfelder
2506 8436 walker
			File bagFile = new File(tempDir, zipName+".zip");
2507
2508 7855 leinfelder
			bag.setFile(bagFile);
2509 7860 leinfelder
			ZipWriter zipWriter = new ZipWriter(bagFactory);
2510 7855 leinfelder
			bag.write(zipWriter, bagFile);
2511
			bagFile = bag.getFile();
2512 8026 leinfelder
			// use custom FIS that will delete the file when closed
2513
			bagInputStream = new DeleteOnCloseFileInputStream(bagFile);
2514
			// also mark for deletion on shutdown in case the stream is never closed
2515
			bagFile.deleteOnExit();
2516 8436 walker
			tempFiles.add(bagFile);
2517 7855 leinfelder
2518 8026 leinfelder
			// clean up other temp files
2519 8800 leinfelder
			for (int i=tempFiles.size()-1; i>=0; i--){
2520
				tempFiles.get(i).delete();
2521 7855 leinfelder
			}
2522 8436 walker
2523 7855 leinfelder
		} catch (IOException e) {
2524
			// report as service failure
2525 9538 tao
		    e.printStackTrace();
2526 7855 leinfelder
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2527
			sf.initCause(e);
2528
			throw sf;
2529
		} catch (OREException e) {
2530
			// report as service failure
2531 9538 tao
		    e.printStackTrace();
2532 7855 leinfelder
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2533
			sf.initCause(e);
2534
			throw sf;
2535
		} catch (URISyntaxException e) {
2536
			// report as service failure
2537 9538 tao
		    e.printStackTrace();
2538 7855 leinfelder
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2539
			sf.initCause(e);
2540
			throw sf;
2541
		} catch (OREParserException e) {
2542
			// report as service failure
2543 9538 tao
		    e.printStackTrace();
2544 7855 leinfelder
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2545
			sf.initCause(e);
2546
			throw sf;
2547 7850 leinfelder
		}
2548
2549
		return bagInputStream;
2550 8810 leinfelder
	}
2551 9353 tao
2552
	 /**
2553
	   * Archives an object, where the object is either a
2554
	   * data object or a science metadata object.
2555
	   *
2556
	   * @param session - the Session object containing the credentials for the Subject
2557
	   * @param pid - The object identifier to be archived
2558
	   *
2559
	   * @return pid - the identifier of the object used for the archiving
2560
	   *
2561
	   * @throws InvalidToken
2562
	   * @throws ServiceFailure
2563
	   * @throws NotAuthorized
2564
	   * @throws NotFound
2565
	   * @throws NotImplemented
2566
	   * @throws InvalidRequest
2567
	   */
2568
	  public Identifier archive(Session session, Identifier pid)
2569
	      throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
2570 9478 tao
	      if(isReadOnlyMode()) {
2571 9481 tao
	            throw new ServiceFailure("2912", ReadOnlyChecker.DATAONEERROR);
2572 9478 tao
	        }
2573 9353 tao
	      boolean allowed = false;
2574
	      // do we have a valid pid?
2575
	      if (pid == null || pid.getValue().trim().equals("")) {
2576
	          throw new ServiceFailure("1350", "The provided identifier was invalid.");
2577
	      }
2578
2579
	      String serviceFailureCode = "1350";
2580
	      Identifier sid = getPIDForSID(pid, serviceFailureCode);
2581
	      if(sid != null) {
2582
	          pid = sid;
2583
	      }
2584
	      // does the subject have archive (a D1 CHANGE_PERMISSION level) privileges on the pid?
2585
	      try {
2586
	            allowed = isAuthorized(session, pid, Permission.CHANGE_PERMISSION);
2587
	        } catch (InvalidRequest e) {
2588
	          throw new ServiceFailure("1350", e.getDescription());
2589
	        }
2590
2591
	      if (allowed) {
2592
	         try {
2593
	             HazelcastService.getInstance().getSystemMetadataMap().lock(pid);
2594
	             logMetacat.debug("MNodeService.archive - lock the identifier "+pid.getValue()+" in the system metadata map.");
2595
	             SystemMetadata sysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
2596 9354 tao
	             boolean needModifyDate = true;
2597 9373 tao
	             boolean logArchive = true;
2598
	             super.archiveObject(logArchive, session, pid, sysmeta, needModifyDate);
2599 9353 tao
	         } finally {
2600
	             HazelcastService.getInstance().getSystemMetadataMap().unlock(pid);
2601
	             logMetacat.debug("MNodeService.archive - unlock the identifier "+pid.getValue()+" in the system metadata map.");
2602
	         }
2603
2604
2605
	      } else {
2606
	          throw new NotAuthorized("1320", "The provided identity does not have " + "permission to archive the object on the Node.");
2607
	      }
2608
2609
	      return pid;
2610
	  }
2611 9190 tao
2612
	/**
2613
	 * Update the system metadata of the specified pid.
2614
	 */
2615 9177 tao
	@Override
2616
	public boolean updateSystemMetadata(Session session, Identifier pid,
2617
            SystemMetadata sysmeta) throws NotImplemented, NotAuthorized,
2618
            ServiceFailure, InvalidRequest, InvalidSystemMetadata, InvalidToken {
2619 9478 tao
2620
	  if(isReadOnlyMode()) {
2621 9481 tao
            throw new ServiceFailure("4868",  ReadOnlyChecker.DATAONEERROR);
2622 9478 tao
      }
2623 9177 tao
	 if(sysmeta == null) {
2624 9478 tao
	     throw  new InvalidRequest("4869", "The system metadata object should NOT be null in the updateSystemMetadata request.");
2625 9177 tao
	 }
2626 9190 tao
	 if(pid == null || pid.getValue() == null) {
2627 9478 tao
         throw new InvalidRequest("4869", "Please specify the id in the updateSystemMetadata request ") ;
2628 9190 tao
     }
2629
2630
     if (session == null) {
2631
         //TODO: many of the thrown exceptions do not use the correct error codes
2632
         //check these against the docs and correct them
2633
         throw new NotAuthorized("4861", "No Session - could not authorize for updating system metadata." +
2634
                 "  If you are not logged in, please do so and retry the request.");
2635
     } else {
2636
         try {
2637
             //Following session can do the change:
2638 9263 tao
           //- Authoritative Member Node (we can use isNodeAdmin since we checked isAuthoritativeNode )
2639 9190 tao
             //- Owner of object (coved by the userHasPermission method)
2640
             //- user subjects with the change permission
2641
             //Note: Coordinating Node can not because MN is authoritative
2642 9263 tao
             /*if(!isAuthoritativeNode(pid)) {
2643
                throw  new InvalidRequest("4863", "Client can only call updateSystemMetadata request on the authoritative memember node.");
2644
             }
2645 9190 tao
             if(!isNodeAdmin(session) && !userHasPermission(session, pid, Permission.CHANGE_PERMISSION)) {
2646
                 throw new NotAuthorized("4861", "The client -"+ session.getSubject().getValue()+ "is not authorized for updating the system metadata of the object "+pid.getValue());
2647 9263 tao
             }*/
2648
             if(!allowUpdating(session, pid, Permission.CHANGE_PERMISSION)) {
2649
                 throw new NotAuthorized("4861", "The client -"+ session.getSubject().getValue()+ "is not authorized for updating the system metadata of the object "+pid.getValue());
2650 9190 tao
             }
2651
         } catch (NotFound e) {
2652 9478 tao
             throw new InvalidRequest("4869", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2653 9997 tao
         } catch(ServiceFailure e) {
2654
             throw new ServiceFailure("4868", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2655
         } catch(NotAuthorized e) {
2656
             throw new NotAuthorized("4861", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2657
         } catch(NotImplemented e) {
2658
             throw new NotImplemented("4866","Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2659
         } catch(InvalidRequest e) {
2660
             throw new InvalidRequest("4869", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2661
         } catch(InvalidToken e) {
2662
             throw new InvalidToken("4957", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2663 9190 tao
         }
2664
2665
     }
2666 9335 tao
      //update the system metadata locally
2667
      boolean success = false;
2668
      try {
2669
          HazelcastService.getInstance().getSystemMetadataMap().lock(pid);
2670
          SystemMetadata currentSysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
2671
          if(currentSysmeta == null) {
2672 9478 tao
              throw  new InvalidRequest("4869", "We can't find the current system metadata on the member node for the id "+pid.getValue());
2673 9335 tao
          }
2674
          Date currentModiDate = currentSysmeta.getDateSysMetadataModified();
2675
          Date commingModiDate = sysmeta.getDateSysMetadataModified();
2676
          if(commingModiDate == null) {
2677 9478 tao
              throw  new InvalidRequest("4869", "The system metadata modification date can't be null.");
2678 9335 tao
          }
2679
          if(currentModiDate != null && commingModiDate.getTime() != currentModiDate.getTime()) {
2680 9478 tao
              throw new InvalidRequest("4869", "Your system metadata modification date is "+commingModiDate.toString()+
2681 9335 tao
                      ". It doesn't match our current system metadata modification date in the member node - "+currentModiDate.toString()+
2682
                      ". Please check if you have got the newest version of the system metadata before the modification.");
2683
          }
2684
          boolean needUpdateModificationDate = true;
2685 9354 tao
          boolean fromCN = false;
2686
          success = updateSystemMetadata(session, pid, sysmeta, needUpdateModificationDate, currentSysmeta, fromCN);
2687 9335 tao
      } finally {
2688
          HazelcastService.getInstance().getSystemMetadataMap().unlock(pid);
2689
      }
2690 9177 tao
2691
      if(success) {
2692 10102 tao
          this.cn = D1Client.getCN();
2693 9190 tao
          //TODO
2694 9177 tao
          //notify the cns the synchornize the new system metadata.
2695 10102 tao
          // run it in a thread to avoid connection timeout
2696
          Runnable runner = new Runnable() {
2697
              private CNode cNode = null;
2698
              private SystemMetadata sys = null;
2699
              private Identifier id = null;
2700
              @Override
2701
              public void run() {
2702
                  try {
2703
                      if(this.cNode == null)  {
2704
                          logMetacat.warn("MNodeService.updateSystemMetadata - can't get the instance of the CN. So can't call cn.synchronize to update the system metadata in CN.");
2705
                      } else if(id != null) {
2706
                          logMetacat.info("MNodeService.updateSystemMetadata - calling cn.synchornized in another thread for pid "+id.getValue());
2707
                          this.cNode.synchronize(null, id);
2708
                      } else {
2709 10104 tao
                          logMetacat.warn("MNodeService.updateSystemMetadata - the pid is null. So can't call cn.synchronize to update the system metadata in CN.");
2710 10102 tao
                      }
2711
                  } catch (BaseException e) {
2712
                      e.printStackTrace();
2713
                      logMetacat.error("It is a DataONEBaseException and its detail code is "+e.getDetail_code() +" and its code is "+e.getCode());
2714
                      logMetacat.error("Can't update the systemmetadata of pid "+id.getValue()+" in CNs through cn.synchronize method since "+e.getMessage(), e);
2715
                  } catch (Exception e) {
2716
                      e.printStackTrace();
2717
                      logMetacat.error("Can't update the systemmetadata of pid "+id.getValue()+" in CNs through cn.synchronize method since "+e.getMessage(), e);
2718
                  }
2719
2720
                  // attempt to re-register the identifier (it checks if it is a doi)
2721
                  try {
2722
                      logMetacat.info("MNodeSerice.updateSystemMetadata - register doi if the pid "+sys.getIdentifier().getValue()+" is a doi");
2723
                      DOIService.getInstance().registerDOI(sys);
2724
                  } catch (Exception e) {
2725
                    logMetacat.warn("Could not [re]register DOI: " + e.getMessage(), e);
2726
                  }
2727 9256 tao
              }
2728 10102 tao
              private Runnable init(CNode cn, SystemMetadata sys, Identifier id){
2729
                  this.cNode = cn;
2730
                  this.sys = sys;
2731
                  this.id = id;
2732
                  return this;
2733
2734
              }
2735
          }.init(cn, sysmeta, pid);
2736
          // submit the task, and that's it
2737 10104 tao
          if(executor != null) {
2738
              executor.submit(runner);
2739 10102 tao
          } else {
2740 10104 tao
              logMetacat.warn("MNodeSerivce.updateSystemMetadata - since the executor service for submitting the call of cn.synchronize() is null, the system metadata change of the id "+pid.getValue()+" can't go to cn through cn.synchronize.");
2741 9246 tao
          }
2742 9177 tao
      }
2743
      return success;
2744
    }
2745 9190 tao
2746
	/*
2747
     * Determine if the current node is the authoritative node for the given pid.
2748
     */
2749 9374 tao
    protected boolean isAuthoritativeNode(Identifier pid) throws InvalidRequest {
2750 9190 tao
        boolean isAuthoritativeNode = false;
2751
        if(pid != null && pid.getValue() != null) {
2752
            SystemMetadata sys = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
2753
            if(sys != null) {
2754
                NodeReference node = sys.getAuthoritativeMemberNode();
2755
                if(node != null) {
2756
                    String nodeValue = node.getValue();
2757
                    logMetacat.debug("The authoritative node for id "+pid.getValue()+" is "+nodeValue);
2758
                    //System.out.println("The authoritative node for id "+pid.getValue()+" is "+nodeValue);
2759
                    String currentNodeId = Settings.getConfiguration().getString("dataone.nodeId");
2760
                    logMetacat.debug("The node id in metacat.properties is "+currentNodeId);
2761
                    //System.out.println("The node id in metacat.properties is "+currentNodeId);
2762
                    if(currentNodeId != null && !currentNodeId.trim().equals("") && currentNodeId.equals(nodeValue)) {
2763 10021 tao
                        logMetacat.debug("They are matching, so the authoritative mn of the object "+pid.getValue()+" is the current node");
2764 9190 tao
                        //System.out.println("They are matching");
2765
                        isAuthoritativeNode = true;
2766
                    }
2767 9374 tao
                } else {
2768
                    throw new InvalidRequest("4869", "Coudn't find the authoritative member node in the system metadata associated with the pid "+pid.getValue());
2769 9190 tao
                }
2770 9374 tao
            } else {
2771
                throw new InvalidRequest("4869", "Coudn't find the system metadata associated with the pid "+pid.getValue());
2772 9190 tao
            }
2773 9374 tao
        } else {
2774
            throw new InvalidRequest("4869", "The request pid is null");
2775 9190 tao
        }
2776
        return isAuthoritativeNode;
2777
    }
2778 6795 cjones
2779 9263 tao
    /*
2780
     * Rules are:
2781
     * 1. If the session has an cn object, it is allowed.
2782
     * 2. If it is not a cn object, the client should have approperate permission and it should also happen on the authorative node.
2783 9700 cjones
     * 3. If it's the authoritative node, the MN Admin Subject is allowed.
2784 9263 tao
     */
2785 9997 tao
    private boolean allowUpdating(Session session, Identifier pid, Permission permission) throws NotAuthorized, NotFound, InvalidRequest, ServiceFailure, NotImplemented, InvalidToken {
2786 9263 tao
        boolean allow = false;
2787 9781 cjones
2788 9700 cjones
        if( isCNAdmin (session) ) {
2789 9263 tao
            allow = true;
2790 9700 cjones
2791 9263 tao
        } else {
2792 9700 cjones
            if( isAuthoritativeNode(pid) ) {
2793
2794 9781 cjones
            	// Check for admin authorization
2795 9700 cjones
            	try {
2796 9781 cjones
					allow = isNodeAdmin(session);
2797 9700 cjones
2798
				} catch (NotImplemented e) {
2799
					logMetacat.debug("Failed to authorize the Member Node Admin Subject: " + e.getMessage());
2800
2801
				} catch (ServiceFailure e) {
2802
					logMetacat.debug("Failed to authorize the Member Node Admin Subject: " + e.getMessage());
2803
2804
				}
2805
2806 9781 cjones
            	// Check for user authorization
2807
            	if ( !allow ) {
2808
                    allow = userHasPermission(session, pid, permission);
2809 9700 cjones
2810 9781 cjones
            	}
2811 9700 cjones
2812 9263 tao
            } else {
2813 10021 tao
                throw new NotAuthorized("4861", "Client can only call the request on the authoritative memember node of the object "+pid.getValue());
2814 9700 cjones
2815 9263 tao
            }
2816
        }
2817
        return allow;
2818
2819
    }
2820
2821 9478 tao
    /**
2822
     * Check if the metacat is in the read-only mode.
2823
     * @return true if it is; otherwise false.
2824
     */
2825
    protected boolean isReadOnlyMode() {
2826
        boolean readOnly = false;
2827
        ReadOnlyChecker checker = new ReadOnlyChecker();
2828
        readOnly = checker.isReadOnly();
2829
        return readOnly;
2830
    }
2831
2832 6179 cjones
}