Project

General

Profile

1 6179 cjones
/**
2
 *  '$RCSfile$'
3
 *  Copyright: 2000-2011 Regents of the University of California and the
4
 *              National Center for Ecological Analysis and Synthesis
5
 *
6
 *   '$Author:  $'
7
 *     '$Date:  $'
8
 *
9
 * This program is free software; you can redistribute it and/or modify
10
 * it under the terms of the GNU General Public License as published by
11
 * the Free Software Foundation; either version 2 of the License, or
12
 * (at your option) any later version.
13
 *
14
 * This program is distributed in the hope that it will be useful,
15
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 * GNU General Public License for more details.
18
 *
19
 * You should have received a copy of the GNU General Public License
20
 * along with this program; if not, write to the Free Software
21
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22
 */
23
24
package edu.ucsb.nceas.metacat.dataone;
25
26 7417 leinfelder
import java.io.ByteArrayInputStream;
27 7860 leinfelder
import java.io.ByteArrayOutputStream;
28 7850 leinfelder
import java.io.File;
29 8800 leinfelder
import java.io.FileInputStream;
30 7850 leinfelder
import java.io.FileOutputStream;
31 6228 cjones
import java.io.IOException;
32 6179 cjones
import java.io.InputStream;
33 8190 leinfelder
import java.io.InputStreamReader;
34 7860 leinfelder
import java.io.OutputStreamWriter;
35 7849 leinfelder
import java.io.UnsupportedEncodingException;
36 7860 leinfelder
import java.io.Writer;
37 7021 leinfelder
import java.math.BigInteger;
38 7849 leinfelder
import java.net.URISyntaxException;
39 6228 cjones
import java.security.NoSuchAlgorithmException;
40 7860 leinfelder
import java.sql.SQLException;
41 7417 leinfelder
import java.util.ArrayList;
42 6525 leinfelder
import java.util.Calendar;
43 6179 cjones
import java.util.Date;
44 8437 walker
import java.util.HashMap;
45 7680 tao
import java.util.HashSet;
46 7860 leinfelder
import java.util.Hashtable;
47 6250 cjones
import java.util.List;
48 7849 leinfelder
import java.util.Map;
49 7417 leinfelder
import java.util.Set;
50 6389 leinfelder
import java.util.Timer;
51 7489 leinfelder
import java.util.UUID;
52 7418 leinfelder
import java.util.Vector;
53 6179 cjones
54 6542 leinfelder
import javax.servlet.http.HttpServletRequest;
55
56 6258 cjones
import org.apache.commons.io.IOUtils;
57 6179 cjones
import org.apache.log4j.Logger;
58 8810 leinfelder
import org.dataone.client.v2.CNode;
59
import org.dataone.client.v2.itk.D1Client;
60
import org.dataone.client.v2.MNode;
61
import org.dataone.client.v2.formats.ObjectFormatCache;
62 6552 leinfelder
import org.dataone.client.auth.CertificateManager;
63 8810 leinfelder
import org.dataone.client.v2.formats.ObjectFormatInfo;
64 6552 leinfelder
import org.dataone.configuration.Settings;
65 7849 leinfelder
import org.dataone.ore.ResourceMapFactory;
66 6795 cjones
import org.dataone.service.exceptions.BaseException;
67 6179 cjones
import org.dataone.service.exceptions.IdentifierNotUnique;
68
import org.dataone.service.exceptions.InsufficientResources;
69
import org.dataone.service.exceptions.InvalidRequest;
70
import org.dataone.service.exceptions.InvalidSystemMetadata;
71
import org.dataone.service.exceptions.InvalidToken;
72
import org.dataone.service.exceptions.NotAuthorized;
73
import org.dataone.service.exceptions.NotFound;
74
import org.dataone.service.exceptions.NotImplemented;
75
import org.dataone.service.exceptions.ServiceFailure;
76 6185 leinfelder
import org.dataone.service.exceptions.SynchronizationFailed;
77 6179 cjones
import org.dataone.service.exceptions.UnsupportedType;
78 8810 leinfelder
import org.dataone.service.mn.tier1.v2.MNCore;
79
import org.dataone.service.mn.tier1.v2.MNRead;
80
import org.dataone.service.mn.tier2.v2.MNAuthorization;
81
import org.dataone.service.mn.tier3.v2.MNStorage;
82
import org.dataone.service.mn.tier4.v2.MNReplication;
83
import org.dataone.service.mn.v2.MNPackage;
84
import org.dataone.service.mn.v2.MNQuery;
85
import org.dataone.service.mn.v2.MNView;
86 8591 leinfelder
import org.dataone.service.types.v1.AccessRule;
87 6366 leinfelder
import org.dataone.service.types.v1.Checksum;
88 7144 leinfelder
import org.dataone.service.types.v1.DescribeResponse;
89 6366 leinfelder
import org.dataone.service.types.v1.Event;
90
import org.dataone.service.types.v1.Identifier;
91 8810 leinfelder
import org.dataone.service.types.v2.Log;
92
import org.dataone.service.types.v2.LogEntry;
93
import org.dataone.service.types.v2.OptionList;
94 6366 leinfelder
import org.dataone.service.types.v1.MonitorInfo;
95
import org.dataone.service.types.v1.MonitorList;
96 8810 leinfelder
import org.dataone.service.types.v2.Node;
97
import org.dataone.service.types.v2.NodeList;
98 6366 leinfelder
import org.dataone.service.types.v1.NodeReference;
99
import org.dataone.service.types.v1.NodeState;
100
import org.dataone.service.types.v1.NodeType;
101 8810 leinfelder
import org.dataone.service.types.v2.ObjectFormat;
102 6366 leinfelder
import org.dataone.service.types.v1.ObjectFormatIdentifier;
103
import org.dataone.service.types.v1.ObjectList;
104
import org.dataone.service.types.v1.Permission;
105
import org.dataone.service.types.v1.Ping;
106 6528 cjones
import org.dataone.service.types.v1.ReplicationStatus;
107 6366 leinfelder
import org.dataone.service.types.v1.Schedule;
108
import org.dataone.service.types.v1.Service;
109
import org.dataone.service.types.v1.Services;
110
import org.dataone.service.types.v1.Session;
111
import org.dataone.service.types.v1.Subject;
112
import org.dataone.service.types.v1.Synchronization;
113 8810 leinfelder
import org.dataone.service.types.v2.SystemMetadata;
114 7417 leinfelder
import org.dataone.service.types.v1.util.AuthUtils;
115 6366 leinfelder
import org.dataone.service.types.v1.util.ChecksumUtil;
116 7417 leinfelder
import org.dataone.service.types.v1_1.QueryEngineDescription;
117
import org.dataone.service.types.v1_1.QueryEngineList;
118 7418 leinfelder
import org.dataone.service.types.v1_1.QueryField;
119 6476 jones
import org.dataone.service.util.Constants;
120 8594 leinfelder
import org.dataone.service.util.TypeMarshaller;
121 7849 leinfelder
import org.dspace.foresite.OREException;
122
import org.dspace.foresite.OREParserException;
123
import org.dspace.foresite.ORESerialiserException;
124
import org.dspace.foresite.ResourceMap;
125 8437 walker
import org.ecoinformatics.datamanager.parser.DataPackage;
126
import org.ecoinformatics.datamanager.parser.Entity;
127
import org.ecoinformatics.datamanager.parser.generic.DataPackageParserInterface;
128
import org.ecoinformatics.datamanager.parser.generic.Eml200DataPackageParser;
129 6179 cjones
130 7448 leinfelder
import edu.ucsb.nceas.ezid.EZIDException;
131 7417 leinfelder
import edu.ucsb.nceas.metacat.DBQuery;
132 7860 leinfelder
import edu.ucsb.nceas.metacat.DBTransform;
133 6234 cjones
import edu.ucsb.nceas.metacat.EventLog;
134 6230 cjones
import edu.ucsb.nceas.metacat.IdentifierManager;
135 6234 cjones
import edu.ucsb.nceas.metacat.McdbDocNotFoundException;
136 7417 leinfelder
import edu.ucsb.nceas.metacat.MetaCatServlet;
137 6389 leinfelder
import edu.ucsb.nceas.metacat.MetacatHandler;
138 7772 tao
import edu.ucsb.nceas.metacat.common.query.EnabledQueryEngines;
139 7757 leinfelder
import edu.ucsb.nceas.metacat.common.query.stream.ContentTypeByteArrayInputStream;
140 6648 leinfelder
import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService;
141 7662 tao
import edu.ucsb.nceas.metacat.index.MetacatSolrEngineDescriptionHandler;
142 7620 tao
import edu.ucsb.nceas.metacat.index.MetacatSolrIndex;
143 6340 cjones
import edu.ucsb.nceas.metacat.properties.PropertyService;
144 7418 leinfelder
import edu.ucsb.nceas.metacat.shared.MetacatUtilException;
145 8026 leinfelder
import edu.ucsb.nceas.metacat.util.DeleteOnCloseFileInputStream;
146 7441 leinfelder
import edu.ucsb.nceas.metacat.util.DocumentUtil;
147 8810 leinfelder
import edu.ucsb.nceas.metacat.util.SkinUtil;
148 6542 leinfelder
import edu.ucsb.nceas.metacat.util.SystemUtil;
149 6340 cjones
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
150 8190 leinfelder
import edu.ucsb.nceas.utilities.XMLUtilities;
151 8800 leinfelder
import edu.ucsb.nceas.utilities.export.HtmlToPdf;
152 7850 leinfelder
import gov.loc.repository.bagit.Bag;
153
import gov.loc.repository.bagit.BagFactory;
154
import gov.loc.repository.bagit.writer.impl.ZipWriter;
155 6230 cjones
156 6179 cjones
/**
157
 * Represents Metacat's implementation of the DataONE Member Node
158
 * service API. Methods implement the various MN* interfaces, and methods common
159
 * to both Member Node and Coordinating Node interfaces are found in the
160
 * D1NodeService base class.
161 6288 cjones
 *
162
 * Implements:
163
 * MNCore.ping()
164
 * MNCore.getLogRecords()
165
 * MNCore.getObjectStatistics()
166
 * MNCore.getOperationStatistics()
167
 * MNCore.getStatus()
168
 * MNCore.getCapabilities()
169
 * MNRead.get()
170
 * MNRead.getSystemMetadata()
171
 * MNRead.describe()
172
 * MNRead.getChecksum()
173
 * MNRead.listObjects()
174
 * MNRead.synchronizationFailed()
175
 * MNAuthorization.isAuthorized()
176
 * MNAuthorization.setAccessPolicy()
177
 * MNStorage.create()
178
 * MNStorage.update()
179
 * MNStorage.delete()
180 9177 tao
 * MNStorage.updateSystemMetadata()
181 6288 cjones
 * MNReplication.replicate()
182
 *
183 6179 cjones
 */
184 6599 cjones
public class MNodeService extends D1NodeService
185 8810 leinfelder
    implements MNAuthorization, MNCore, MNRead, MNReplication, MNStorage, MNQuery, MNView, MNPackage {
186 6179 cjones
187 7772 tao
    //private static final String PATHQUERY = "pathquery";
188 7849 leinfelder
	public static final String UUID_SCHEME = "UUID";
189
	public static final String DOI_SCHEME = "DOI";
190 7489 leinfelder
	private static final String UUID_PREFIX = "urn:uuid:";
191 7418 leinfelder
192
	/* the logger instance */
193 6475 jones
    private Logger logMetacat = null;
194 6795 cjones
195
    /* A reference to a remote Memeber Node */
196 9295 tao
    //private MNode mn;
197 6795 cjones
198
    /* A reference to a Coordinating Node */
199
    private CNode cn;
200 6241 cjones
201 6795 cjones
202 6475 jones
    /**
203
     * Singleton accessor to get an instance of MNodeService.
204
     *
205
     * @return instance - the instance of MNodeService
206
     */
207 6542 leinfelder
    public static MNodeService getInstance(HttpServletRequest request) {
208
        return new MNodeService(request);
209 6179 cjones
    }
210
211 6475 jones
    /**
212
     * Constructor, private for singleton access
213
     */
214 6542 leinfelder
    private MNodeService(HttpServletRequest request) {
215
        super(request);
216 6475 jones
        logMetacat = Logger.getLogger(MNodeService.class);
217 6552 leinfelder
218
        // set the Member Node certificate file location
219
        CertificateManager.getInstance().setCertificateLocation(Settings.getConfiguration().getString("D1Client.certificate.file"));
220 6310 cjones
    }
221 6475 jones
222
    /**
223
     * Deletes an object from the Member Node, where the object is either a
224
     * data object or a science metadata object.
225
     *
226
     * @param session - the Session object containing the credentials for the Subject
227
     * @param pid - The object identifier to be deleted
228
     *
229
     * @return pid - the identifier of the object used for the deletion
230
     *
231
     * @throws InvalidToken
232
     * @throws ServiceFailure
233
     * @throws NotAuthorized
234
     * @throws NotFound
235
     * @throws NotImplemented
236
     * @throws InvalidRequest
237
     */
238
    @Override
239
    public Identifier delete(Session session, Identifier pid)
240 6610 cjones
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
241 6475 jones
242 7162 leinfelder
    	// only admin of  the MN or the CN is allowed a full delete
243
        boolean allowed = false;
244 7330 leinfelder
        allowed = isAdminAuthorized(session);
245 8360 tao
246 9050 tao
        String serviceFailureCode = "2902";
247
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
248
        if(sid != null) {
249
            pid = sid;
250
        }
251
252 8360 tao
        //check if it is the authoritative member node
253
        if(!allowed) {
254
            allowed = isAuthoritativeMNodeAdmin(session, pid);
255
        }
256
257 7162 leinfelder
        if (!allowed) {
258 7245 cjones
            throw new NotAuthorized("1320", "The provided identity does not have " + "permission to delete objects on the Node.");
259 7162 leinfelder
        }
260
261 7077 leinfelder
    	// defer to superclass implementation
262
        return super.delete(session, pid);
263 6250 cjones
    }
264
265 6475 jones
    /**
266
     * Updates an existing object by creating a new object identified by
267
     * newPid on the Member Node which explicitly obsoletes the object
268
     * identified by pid through appropriate changes to the SystemMetadata
269
     * of pid and newPid
270
     *
271
     * @param session - the Session object containing the credentials for the Subject
272
     * @param pid - The identifier of the object to be updated
273
     * @param object - the new object bytes
274
     * @param sysmeta - the new system metadata describing the object
275
     *
276
     * @return newPid - the identifier of the new object
277
     *
278
     * @throws InvalidToken
279
     * @throws ServiceFailure
280
     * @throws NotAuthorized
281
     * @throws NotFound
282
     * @throws NotImplemented
283
     * @throws IdentifierNotUnique
284
     * @throws UnsupportedType
285
     * @throws InsufficientResources
286
     * @throws InvalidSystemMetadata
287
     * @throws InvalidRequest
288
     */
289
    @Override
290 6575 cjones
    public Identifier update(Session session, Identifier pid, InputStream object,
291
        Identifier newPid, SystemMetadata sysmeta)
292
        throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique,
293
        UnsupportedType, InsufficientResources, NotFound,
294
        InvalidSystemMetadata, NotImplemented, InvalidRequest {
295 6250 cjones
296 9044 tao
        //transform a sid to a pid if it is applicable
297
        String serviceFailureCode = "1310";
298
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
299
        if(sid != null) {
300
            pid = sid;
301
        }
302
303 6475 jones
        String localId = null;
304
        boolean allowed = false;
305
        boolean isScienceMetadata = false;
306 6645 leinfelder
307
        if (session == null) {
308
        	throw new InvalidToken("1210", "No session has been provided");
309
        }
310 6475 jones
        Subject subject = session.getSubject();
311
312 7315 leinfelder
        // verify the pid is valid format
313 7318 leinfelder
        if (!isValidIdentifier(pid)) {
314 7315 leinfelder
        	throw new InvalidRequest("1202", "The provided identifier is invalid.");
315 6475 jones
        }
316 9044 tao
317
        // verify the new pid is valid format
318
        if (!isValidIdentifier(newPid)) {
319
            throw new InvalidRequest("1202", "The provided identifier is invalid.");
320
        }
321
322
        // make sure that the newPid doesn't exists
323
        boolean idExists = true;
324
        try {
325
            idExists = IdentifierManager.getInstance().identifierExists(newPid.getValue());
326
        } catch (SQLException e) {
327
            throw new ServiceFailure("1310",
328
                                    "The requested identifier " + newPid.getValue() +
329
                                    " couldn't be determined if it is unique since : "+e.getMessage());
330
        }
331
        if (idExists) {
332
                throw new IdentifierNotUnique("1220",
333
                          "The requested identifier " + newPid.getValue() +
334
                          " is already used by another object and" +
335
                          "therefore can not be used for this object. Clients should choose" +
336
                          "a new identifier that is unique and retry the operation or " +
337
                          "use CN.reserveIdentifier() to reserve one.");
338
339
        }
340
341
342 6475 jones
343
        // check for the existing identifier
344
        try {
345
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
346 6575 cjones
347 6475 jones
        } catch (McdbDocNotFoundException e) {
348 6575 cjones
            throw new InvalidRequest("1202", "The object with the provided " +
349
                "identifier was not found.");
350
351 9024 tao
        } catch (SQLException ee) {
352
            throw new ServiceFailure("1310", "The object with the provided " +
353
                    "identifier "+pid.getValue()+" can't be identified since - "+ee.getMessage());
354 6475 jones
        }
355 6518 leinfelder
356 6521 leinfelder
        // set the originating node
357
        NodeReference originMemberNode = this.getCapabilities().getIdentifier();
358
        sysmeta.setOriginMemberNode(originMemberNode);
359
360 6518 leinfelder
        // set the submitter to match the certificate
361
        sysmeta.setSubmitter(subject);
362 6525 leinfelder
        // set the dates
363
        Date now = Calendar.getInstance().getTime();
364 6575 cjones
        sysmeta.setDateSysMetadataModified(now);
365
        sysmeta.setDateUploaded(now);
366 7486 leinfelder
367
        // make sure serial version is set to something
368
        BigInteger serialVersion = sysmeta.getSerialVersion();
369
        if (serialVersion == null) {
370
        	sysmeta.setSerialVersion(BigInteger.ZERO);
371
        }
372 6475 jones
373
        // does the subject have WRITE ( == update) priveleges on the pid?
374 9263 tao
        //allowed = isAuthorized(session, pid, Permission.WRITE);
375
        //CN having the permission is allowed; user with the write permission and calling on the authoritative node is allowed.
376
        allowed = allowUpdating(session, pid, Permission.WRITE);
377 6475 jones
        if (allowed) {
378 6649 leinfelder
379
        	// check quality of SM
380
        	if (sysmeta.getObsoletedBy() != null) {
381
        		throw new InvalidSystemMetadata("1300", "Cannot include obsoletedBy when updating object");
382
        	}
383
        	if (sysmeta.getObsoletes() != null && !sysmeta.getObsoletes().getValue().equals(pid.getValue())) {
384
        		throw new InvalidSystemMetadata("1300", "The identifier provided in obsoletes does not match old Identifier");
385
        	}
386 6475 jones
387
            // get the existing system metadata for the object
388
            SystemMetadata existingSysMeta = getSystemMetadata(session, pid);
389 9200 tao
            //System.out.println("the archive is "+existingSysMeta.getArchived());
390
            //Base on documentation, we can't update an archived object:
391
            //The update operation MUST fail with Exceptions.InvalidRequest on objects that have the Types.SystemMetadata.archived property set to true.
392
            if(existingSysMeta.getArchived() != null && existingSysMeta.getArchived()) {
393
                throw new InvalidRequest("1202","An archived object"+pid.getValue()+" can't be updated");
394
            }
395 6475 jones
396 7400 leinfelder
            // check for previous update
397
            // see: https://redmine.dataone.org/issues/3336
398
            Identifier existingObsoletedBy = existingSysMeta.getObsoletedBy();
399
            if (existingObsoletedBy != null) {
400
            	throw new InvalidRequest("1202",
401
            			"The previous identifier has already been made obsolete by: " + existingObsoletedBy.getValue());
402
            }
403 9044 tao
            //check the sid in the system metadata. If it exists, it should be non-exist or match the old sid in the previous system metadata.
404
            Identifier sidInSys = sysmeta.getSeriesId();
405
            if(sidInSys != null) {
406 9046 tao
                if (!isValidIdentifier(sidInSys)) {
407
                    throw new InvalidSystemMetadata("1300", "The provided series id in the system metadata is invalid.");
408
                }
409 9044 tao
                Identifier previousSid = existingSysMeta.getSeriesId();
410
                if(previousSid != null) {
411
                    // there is a previous sid, if the new sid doesn't match it, the new sid should be non-existing.
412 9048 tao
                    if(!sidInSys.getValue().equals(previousSid.getValue())) {
413 9044 tao
                        try {
414
                            idExists = IdentifierManager.getInstance().identifierExists(sidInSys.getValue());
415
                        } catch (SQLException e) {
416
                            throw new ServiceFailure("1310",
417
                                                    "The requested identifier " + sidInSys.getValue() +
418
                                                    " couldn't be determined if it is unique since : "+e.getMessage());
419
                        }
420
                        if(idExists) {
421
                            throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata doesn't match the previous series id "
422
                                                            +previousSid.getValue()+", so it should NOT exist. However, it was used by another object.");
423
                        }
424
                    }
425
                } else {
426
                    // there is no previous sid, the new sid should be non-existing.
427
                    try {
428
                        idExists = IdentifierManager.getInstance().identifierExists(sidInSys.getValue());
429
                    } catch (SQLException e) {
430
                        throw new ServiceFailure("1310",
431
                                                "The requested identifier " + sidInSys.getValue() +
432
                                                " couldn't be determined if it is unique since : "+e.getMessage());
433
                    }
434
                    if(idExists) {
435
                        throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata should NOT exist since the previous series id is null."
436
                                                        +"However, it was used by another object.");
437
                    }
438
                }
439 9047 tao
                //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario)
440
                if(sidInSys.getValue().equals(newPid.getValue())) {
441
                    throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata shouldn't have the same value of the pid.");
442
                }
443 9044 tao
            }
444 6475 jones
445
            isScienceMetadata = isScienceMetadata(sysmeta);
446
447
            // do we have XML metadata or a data object?
448
            if (isScienceMetadata) {
449
450
                // update the science metadata XML document
451
                // TODO: handle non-XML metadata/data documents (like netCDF)
452
                // TODO: don't put objects into memory using stream to string
453 8948 tao
                //String objectAsXML = "";
454 6475 jones
                try {
455 8948 tao
                    //objectAsXML = IOUtils.toString(object, "UTF-8");
456 7443 leinfelder
                    // give the old pid so we can calculate the new local id
457 8948 tao
                    localId = insertOrUpdateDocument(object, "UTF-8", pid, session, "update");
458 6475 jones
                    // register the newPid and the generated localId
459
                    if (newPid != null) {
460
                        IdentifierManager.getInstance().createMapping(newPid.getValue(), localId);
461
462
                    }
463
464
                } catch (IOException e) {
465
                    String msg = "The Node is unable to create the object. " + "There was a problem converting the object to XML";
466
                    logMetacat.info(msg);
467
                    throw new ServiceFailure("1310", msg + ": " + e.getMessage());
468
469
                }
470
471
            } else {
472
473
                // update the data object
474
                localId = insertDataObject(object, newPid, session);
475
476
            }
477 8267 leinfelder
478
            // add the newPid to the obsoletedBy list for the existing sysmeta
479
            existingSysMeta.setObsoletedBy(newPid);
480 9283 tao
            //increase version
481
            BigInteger current = existingSysMeta.getSerialVersion();
482
            //System.out.println("the current version is "+current);
483
            current = current.add(BigInteger.ONE);
484
            //System.out.println("the new current version is "+current);
485
            existingSysMeta.setSerialVersion(current);
486 8267 leinfelder
            // then update the existing system metadata
487
            updateSystemMetadata(existingSysMeta);
488
489
            // prep the new system metadata, add pid to the affected lists
490
            sysmeta.setObsoletes(pid);
491
            //sysmeta.addDerivedFrom(pid);
492
493 6475 jones
            // and insert the new system metadata
494
            insertSystemMetadata(sysmeta);
495
496
            // log the update event
497 6542 leinfelder
            EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), subject.getValue(), localId, Event.UPDATE.toString());
498 7507 leinfelder
499
            // attempt to register the identifier - it checks if it is a doi
500
            try {
501 7512 leinfelder
    			DOIService.getInstance().registerDOI(sysmeta);
502 8795 leinfelder
    		} catch (Exception e) {
503 7507 leinfelder
                throw new ServiceFailure("1190", "Could not register DOI: " + e.getMessage());
504
    		}
505 6475 jones
506
        } else {
507
            throw new NotAuthorized("1200", "The provided identity does not have " + "permission to UPDATE the object identified by " + pid.getValue()
508
                    + " on the Member Node.");
509
        }
510
511
        return newPid;
512 6250 cjones
    }
513 6254 cjones
514 6475 jones
    public Identifier create(Session session, Identifier pid, InputStream object, SystemMetadata sysmeta) throws InvalidToken, ServiceFailure, NotAuthorized,
515
            IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest {
516 6250 cjones
517 6916 cjones
        // check for null session
518 6530 leinfelder
        if (session == null) {
519 6575 cjones
          throw new InvalidToken("1110", "Session is required to WRITE to the Node.");
520 6530 leinfelder
        }
521 9068 tao
        // verify the pid is valid format
522
        if (!isValidIdentifier(pid)) {
523
            throw new InvalidRequest("1102", "The provided identifier is invalid.");
524
        }
525 6518 leinfelder
        // set the submitter to match the certificate
526
        sysmeta.setSubmitter(session.getSubject());
527 6520 leinfelder
        // set the originating node
528
        NodeReference originMemberNode = this.getCapabilities().getIdentifier();
529
        sysmeta.setOriginMemberNode(originMemberNode);
530 6916 cjones
        sysmeta.setArchived(false);
531
532 6525 leinfelder
        // set the dates
533
        Date now = Calendar.getInstance().getTime();
534 6916 cjones
        sysmeta.setDateSysMetadataModified(now);
535
        sysmeta.setDateUploaded(now);
536 7021 leinfelder
537
        // set the serial version
538
        sysmeta.setSerialVersion(BigInteger.ZERO);
539 7083 cjones
540
        // check that we are not attempting to subvert versioning
541
        if (sysmeta.getObsoletes() != null && sysmeta.getObsoletes().getValue() != null) {
542
            throw new InvalidSystemMetadata("1180",
543
              "The supplied system metadata is invalid. " +
544
              "The obsoletes field cannot have a value when creating entries.");
545
        }
546 7021 leinfelder
547 7083 cjones
        if (sysmeta.getObsoletedBy() != null && sysmeta.getObsoletedBy().getValue() != null) {
548
            throw new InvalidSystemMetadata("1180",
549
              "The supplied system metadata is invalid. " +
550
              "The obsoletedBy field cannot have a value when creating entries.");
551
        }
552 9068 tao
553
        // verify the sid in the system metadata
554
        Identifier sid = sysmeta.getSeriesId();
555
        boolean idExists = false;
556
        if(sid != null) {
557
            if (!isValidIdentifier(sid)) {
558
                throw new InvalidSystemMetadata("1180", "The provided series id is invalid.");
559
            }
560
            try {
561
                idExists = IdentifierManager.getInstance().identifierExists(sid.getValue());
562
            } catch (SQLException e) {
563
                throw new ServiceFailure("1190",
564
                                        "The series identifier " + sid.getValue() +
565
                                        " in the system metadata couldn't be determined if it is unique since : "+e.getMessage());
566
            }
567
            if (idExists) {
568
                    throw new InvalidSystemMetadata("1180",
569
                              "The series identifier " + sid.getValue() +
570
                              " is already used by another object and" +
571
                              "therefore can not be used for this object. Clients should choose" +
572
                              "a new identifier that is unique and retry the operation or " +
573
                              "use CN.reserveIdentifier() to reserve one.");
574
575
            }
576
            //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario )
577
            if(sid.getValue().equals(pid.getValue())) {
578
                throw new InvalidSystemMetadata("1180", "The series id "+sid.getValue()+" in the system metadata shouldn't have the same value of the pid.");
579
            }
580
        }
581 7083 cjones
582 6518 leinfelder
        // call the shared impl
583 7507 leinfelder
        Identifier resultPid = super.create(session, pid, object, sysmeta);
584
585
        // attempt to register the identifier - it checks if it is a doi
586
        try {
587 7512 leinfelder
			DOIService.getInstance().registerDOI(sysmeta);
588 8795 leinfelder
		} catch (Exception e) {
589 7510 leinfelder
			ServiceFailure sf = new ServiceFailure("1190", "Could not register DOI: " + e.getMessage());
590
			sf.initCause(e);
591
            throw sf;
592 7507 leinfelder
		}
593
594
        // return
595
		return resultPid ;
596 6475 jones
    }
597 6250 cjones
598 6475 jones
    /**
599
     * Called by a Coordinating Node to request that the Member Node create a
600
     * copy of the specified object by retrieving it from another Member
601
     * Node and storing it locally so that it can be made accessible to
602
     * the DataONE system.
603
     *
604
     * @param session - the Session object containing the credentials for the Subject
605
     * @param sysmeta - Copy of the CN held system metadata for the object
606
     * @param sourceNode - A reference to node from which the content should be
607
     *                     retrieved. The reference should be resolved by
608
     *                     checking the CN node registry.
609
     *
610
     * @return true if the replication succeeds
611
     *
612
     * @throws ServiceFailure
613
     * @throws NotAuthorized
614
     * @throws NotImplemented
615
     * @throws UnsupportedType
616
     * @throws InsufficientResources
617
     * @throws InvalidRequest
618
     */
619
    @Override
620 6786 cjones
    public boolean replicate(Session session, SystemMetadata sysmeta,
621
            NodeReference sourceNode) throws NotImplemented, ServiceFailure,
622
            NotAuthorized, InvalidRequest, InsufficientResources,
623
            UnsupportedType {
624
625 6875 cjones
        if (session != null && sysmeta != null && sourceNode != null) {
626
            logMetacat.info("MNodeService.replicate() called with parameters: \n" +
627
                            "\tSession.Subject      = "                           +
628
                            session.getSubject().getValue() + "\n"                +
629 7082 cjones
                            "\tidentifier           = "                           +
630
                            sysmeta.getIdentifier().getValue()                    +
631 6875 cjones
                            "\n" + "\tSource NodeReference ="                     +
632
                            sourceNode.getValue());
633
        }
634 6475 jones
        boolean result = false;
635 6786 cjones
        String nodeIdStr = null;
636 6651 cjones
        NodeReference nodeId = null;
637 6786 cjones
638 6795 cjones
        // get the referenced object
639
        Identifier pid = sysmeta.getIdentifier();
640 9068 tao
        // verify the pid is valid format
641
        if (!isValidIdentifier(pid)) {
642
            throw new InvalidRequest("2153", "The provided identifier in the system metadata is invalid.");
643
        }
644 6795 cjones
645
        // get from the membernode
646
        // TODO: switch credentials for the server retrieval?
647
        this.cn = D1Client.getCN();
648
        InputStream object = null;
649
        Session thisNodeSession = null;
650
        SystemMetadata localSystemMetadata = null;
651
        BaseException failure = null;
652 6818 cjones
        String localId = null;
653
654 6795 cjones
        // TODO: check credentials
655
        // cannot be called by public
656 7063 leinfelder
        if (session == null || session.getSubject() == null) {
657 7082 cjones
            String msg = "No session was provided to replicate identifier " +
658
            sysmeta.getIdentifier().getValue();
659 8414 tao
            logMetacat.error(msg);
660 7192 cjones
            throw new NotAuthorized("2152", msg);
661
662 6795 cjones
        }
663
664
665 6651 cjones
        // get the local node id
666
        try {
667 7030 cjones
            nodeIdStr = PropertyService.getProperty("dataone.nodeId");
668 6651 cjones
            nodeId = new NodeReference();
669
            nodeId.setValue(nodeIdStr);
670 6786 cjones
671 6651 cjones
        } catch (PropertyNotFoundException e1) {
672 7030 cjones
            String msg = "Couldn't get dataone.nodeId property: " + e1.getMessage();
673 6795 cjones
            failure = new ServiceFailure("2151", msg);
674 8414 tao
            //setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
675 6795 cjones
            logMetacat.error(msg);
676 8414 tao
            //return true;
677
            throw new ServiceFailure("2151", msg);
678 6786 cjones
679 6651 cjones
        }
680 6795 cjones
681 6475 jones
682
        try {
683 6786 cjones
            // do we already have a replica?
684
            try {
685 6818 cjones
                localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
686 6822 cjones
                // if we have a local id, get the local object
687
                try {
688
                    object = MetacatHandler.read(localId);
689
                } catch (Exception e) {
690 7127 leinfelder
                	// NOTE: we may already know about this ID because it could be a data file described by a metadata file
691
                	// https://redmine.dataone.org/issues/2572
692
                	// TODO: fix this so that we don't prevent ourselves from getting replicas
693
694 6822 cjones
                    // let the CN know that the replication failed
695 7127 leinfelder
                	logMetacat.warn("Object content not found on this node despite having localId: " + localId);
696
                	String msg = "Can't read the object bytes properly, replica is invalid.";
697
                    ServiceFailure serviceFailure = new ServiceFailure("2151", msg);
698 7113 leinfelder
                    setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, serviceFailure);
699
                    logMetacat.warn(msg);
700 6822 cjones
                    throw serviceFailure;
701
702
                }
703
704 6817 cjones
            } catch (McdbDocNotFoundException e) {
705 6818 cjones
                logMetacat.info("No replica found. Continuing.");
706 6817 cjones
707 9024 tao
            } catch (SQLException ee) {
708
                throw new ServiceFailure("2151", "Couldn't identify the local id of the object with the specified identifier "
709
                                        +pid.getValue()+" since - "+ee.getMessage());
710 6819 cjones
            }
711
712 6786 cjones
            // no local replica, get a replica
713 6819 cjones
            if ( object == null ) {
714 9295 tao
                /*boolean success = true;
715 9288 tao
                try {
716
                    //use the v2 ping api to connect the source node
717
                    mn.ping();
718
                } catch (Exception e) {
719
                    success = false;
720 9295 tao
                }*/
721
                D1NodeVersionChecker checker = new D1NodeVersionChecker(sourceNode);
722
                String nodeVersion = checker.getVersion("MNRead");
723
                if(nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V1)) {
724
                    //The source node is a v1 node, we use the v1 api
725 9287 tao
                    org.dataone.client.v1.MNode mNodeV1 =  org.dataone.client.v1.itk.D1Client.getMN(sourceNode);
726 9295 tao
                    object = mNodeV1.getReplica(thisNodeSession, pid);
727
                } else if (nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V2)){
728 9287 tao
                 // session should be null to use the default certificate
729
                    // location set in the Certificate manager
730 9295 tao
                    MNode mn = D1Client.getMN(sourceNode);
731 9287 tao
                    object = mn.getReplica(thisNodeSession, pid);
732 9295 tao
                } else {
733
                    throw new ServiceFailure("2151", "The version of MNRead service is "+nodeVersion+" in the source node "+sourceNode.getValue()+" and it is supported. Please check the information in the cn");
734 9287 tao
                }
735
736 7082 cjones
                logMetacat.info("MNodeService.getReplica() called for identifier "
737 6786 cjones
                                + pid.getValue());
738
739
            }
740
741 6795 cjones
        } catch (InvalidToken e) {
742
            String msg = "Could not retrieve object to replicate (InvalidToken): "+ e.getMessage();
743
            failure = new ServiceFailure("2151", msg);
744
            setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
745
            logMetacat.error(msg);
746
            throw new ServiceFailure("2151", msg);
747 6786 cjones
748 6475 jones
        } catch (NotFound e) {
749 6795 cjones
            String msg = "Could not retrieve object to replicate (NotFound): "+ e.getMessage();
750
            failure = new ServiceFailure("2151", msg);
751
            setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
752
            logMetacat.error(msg);
753
            throw new ServiceFailure("2151", msg);
754 6786 cjones
755 8414 tao
        } catch (NotAuthorized e) {
756
            String msg = "Could not retrieve object to replicate (NotAuthorized): "+ e.getMessage();
757
            failure = new ServiceFailure("2151", msg);
758
            setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
759
            logMetacat.error(msg);
760
            throw new ServiceFailure("2151", msg);
761
        } catch (NotImplemented e) {
762
            String msg = "Could not retrieve object to replicate (mn.getReplica NotImplemented): "+ e.getMessage();
763
            failure = new ServiceFailure("2151", msg);
764
            setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
765
            logMetacat.error(msg);
766
            throw new ServiceFailure("2151", msg);
767
        } catch (ServiceFailure e) {
768
            String msg = "Could not retrieve object to replicate (ServiceFailure): "+ e.getMessage();
769
            failure = new ServiceFailure("2151", msg);
770
            setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
771
            logMetacat.error(msg);
772
            throw new ServiceFailure("2151", msg);
773
        } catch (InsufficientResources e) {
774
            String msg = "Could not retrieve object to replicate (InsufficientResources): "+ e.getMessage();
775
            failure = new ServiceFailure("2151", msg);
776
            setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
777
            logMetacat.error(msg);
778
            throw new ServiceFailure("2151", msg);
779 6475 jones
        }
780
781 6693 leinfelder
        // verify checksum on the object, if supported
782
        if (object.markSupported()) {
783 6786 cjones
            Checksum givenChecksum = sysmeta.getChecksum();
784
            Checksum computedChecksum = null;
785
            try {
786 7127 leinfelder
                computedChecksum = ChecksumUtil.checksum(object, givenChecksum.getAlgorithm());
787 6786 cjones
                object.reset();
788 6948 cjones
789 6786 cjones
            } catch (Exception e) {
790 7127 leinfelder
                String msg = "Error computing checksum on replica: " + e.getMessage();
791 7113 leinfelder
                logMetacat.error(msg);
792 6786 cjones
                ServiceFailure sf = new ServiceFailure("2151", msg);
793
                sf.initCause(e);
794 8414 tao
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, sf);
795 6786 cjones
                throw sf;
796
            }
797 6948 cjones
            if (!givenChecksum.getValue().equals(computedChecksum.getValue())) {
798 7113 leinfelder
                logMetacat.error("Given    checksum for " + pid.getValue() +
799 6948 cjones
                    "is " + givenChecksum.getValue());
800 7113 leinfelder
                logMetacat.error("Computed checksum for " + pid.getValue() +
801 6948 cjones
                    "is " + computedChecksum.getValue());
802 8414 tao
                String msg = "Computed checksum does not match declared checksum";
803
                failure = new ServiceFailure("2151", msg);
804
                setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
805
                throw new ServiceFailure("2151", msg);
806 6786 cjones
            }
807 6693 leinfelder
        }
808 6786 cjones
809 6475 jones
        // add it to local store
810
        Identifier retPid;
811
        try {
812 7127 leinfelder
            // skip the MN.create -- this mutates the system metadata and we don't want it to
813 6818 cjones
            if ( localId == null ) {
814 7127 leinfelder
                // TODO: this will fail if we already "know" about the identifier
815
            	// FIXME: see https://redmine.dataone.org/issues/2572
816 6818 cjones
                retPid = super.create(session, pid, object, sysmeta);
817
                result = (retPid.getValue().equals(pid.getValue()));
818
            }
819 6795 cjones
820 7125 leinfelder
        } catch (Exception e) {
821 7126 leinfelder
            String msg = "Could not save object to local store (" + e.getClass().getName() + "): " + e.getMessage();
822 7125 leinfelder
            failure = new ServiceFailure("2151", msg);
823
            setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure);
824
            logMetacat.error(msg);
825
            throw new ServiceFailure("2151", msg);
826
827 6475 jones
        }
828
829 6795 cjones
        // finish by setting the replication status
830
        setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.COMPLETED, null);
831 6475 jones
        return result;
832
833 6250 cjones
    }
834 9287 tao
835
    /*
836
     * If the given node supports v2 replication.
837
     */
838
    private boolean supportV2Replication(Node node) throws InvalidRequest {
839
        return supportVersionReplication(node, "v2");
840
    }
841
842
    /*
843
     * If the given node support the the given version replication. Return true if it does.
844
     */
845
    private boolean supportVersionReplication(Node node, String version) throws InvalidRequest{
846
        boolean support = false;
847
        if(node == null) {
848
            throw new InvalidRequest("2153", "There is no capacity information about the node in the replicate.");
849
        } else {
850
            Services services = node.getServices();
851
            if(services == null) {
852
                throw new InvalidRequest("2153", "Can't get replica from a node which doesn't have the replicate service.");
853
            } else {
854
               List<Service> list = services.getServiceList();
855
               if(list == null) {
856
                   throw new InvalidRequest("2153", "Can't get replica from a node which doesn't have the replicate service.");
857
               } else {
858
                   for(Service service : list) {
859
                       if(service != null && service.getName() != null && service.getName().equals("MNReplication") &&
860
                               service.getVersion() != null && service.getVersion().equalsIgnoreCase(version) && service.getAvailable() == true ) {
861
                           support = true;
862
863
                       }
864
                   }
865
               }
866
            }
867
        }
868
        return support;
869
    }
870 6179 cjones
871 6475 jones
    /**
872
     * Return the object identified by the given object identifier
873
     *
874
     * @param session - the Session object containing the credentials for the Subject
875
     * @param pid - the object identifier for the given object
876
     *
877
     * @return inputStream - the input stream of the given object
878
     *
879
     * @throws InvalidToken
880
     * @throws ServiceFailure
881
     * @throws NotAuthorized
882
     * @throws InvalidRequest
883
     * @throws NotImplemented
884
     */
885
    @Override
886 6610 cjones
    public InputStream get(Session session, Identifier pid)
887
    throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented {
888 6258 cjones
889 6475 jones
        return super.get(session, pid);
890 6258 cjones
891 6259 cjones
    }
892 6258 cjones
893 6475 jones
    /**
894
     * Returns a Checksum for the specified object using an accepted hashing algorithm
895
     *
896
     * @param session - the Session object containing the credentials for the Subject
897
     * @param pid - the object identifier for the given object
898
     * @param algorithm -  the name of an algorithm that will be used to compute
899
     *                     a checksum of the bytes of the object
900
     *
901
     * @return checksum - the checksum of the given object
902
     *
903
     * @throws InvalidToken
904
     * @throws ServiceFailure
905
     * @throws NotAuthorized
906
     * @throws NotFound
907
     * @throws InvalidRequest
908
     * @throws NotImplemented
909
     */
910
    @Override
911 6610 cjones
    public Checksum getChecksum(Session session, Identifier pid, String algorithm)
912
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
913
        InvalidRequest, NotImplemented {
914 6258 cjones
915 6475 jones
        Checksum checksum = null;
916 9040 tao
        String serviceFailure = "1410";
917
        String notFound = "1420";
918
        //Checkum only handles the pid, not sid
919
        checkV1SystemMetaPidExist(pid, serviceFailure, "The checksum for the object specified by "+pid.getValue()+" couldn't be returned ",  notFound,
920
                "The object specified by "+pid.getValue()+" does not exist at this node.");
921 6475 jones
        InputStream inputStream = get(session, pid);
922
923 6259 cjones
        try {
924 6475 jones
            checksum = ChecksumUtil.checksum(inputStream, algorithm);
925
926
        } catch (NoSuchAlgorithmException e) {
927
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned due to an internal error: "
928
                    + e.getMessage());
929 6259 cjones
        } catch (IOException e) {
930 6475 jones
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned due to an internal error: "
931
                    + e.getMessage());
932 6259 cjones
        }
933 6382 cjones
934 6475 jones
        if (checksum == null) {
935
            throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned.");
936
        }
937 6258 cjones
938 6475 jones
        return checksum;
939 6259 cjones
    }
940 6179 cjones
941 6475 jones
    /**
942
     * Return the system metadata for a given object
943
     *
944
     * @param session - the Session object containing the credentials for the Subject
945
     * @param pid - the object identifier for the given object
946
     *
947
     * @return inputStream - the input stream of the given system metadata object
948
     *
949
     * @throws InvalidToken
950
     * @throws ServiceFailure
951
     * @throws NotAuthorized
952
     * @throws NotFound
953
     * @throws InvalidRequest
954
     * @throws NotImplemented
955
     */
956
    @Override
957 6610 cjones
    public SystemMetadata getSystemMetadata(Session session, Identifier pid)
958
        throws InvalidToken, ServiceFailure, NotAuthorized, NotFound,
959
        NotImplemented {
960 6341 leinfelder
961 6475 jones
        return super.getSystemMetadata(session, pid);
962
    }
963 6341 leinfelder
964 6475 jones
    /**
965
     * Retrieve the list of objects present on the MN that match the calling parameters
966
     *
967
     * @param session - the Session object containing the credentials for the Subject
968
     * @param startTime - Specifies the beginning of the time range from which
969
     *                    to return object (>=)
970
     * @param endTime - Specifies the beginning of the time range from which
971
     *                  to return object (>=)
972
     * @param objectFormat - Restrict results to the specified object format
973
     * @param replicaStatus - Indicates if replicated objects should be returned in the list
974
     * @param start - The zero-based index of the first value, relative to the
975
     *                first record of the resultset that matches the parameters.
976
     * @param count - The maximum number of entries that should be returned in
977
     *                the response. The Member Node may return less entries
978
     *                than specified in this value.
979
     *
980
     * @return objectList - the list of objects matching the criteria
981
     *
982
     * @throws InvalidToken
983
     * @throws ServiceFailure
984
     * @throws NotAuthorized
985
     * @throws InvalidRequest
986
     * @throws NotImplemented
987
     */
988
    @Override
989 8810 leinfelder
    public ObjectList listObjects(Session session, Date startTime, Date endTime, ObjectFormatIdentifier objectFormatId, Identifier identifier, Boolean replicaStatus, Integer start,
990 6475 jones
            Integer count) throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken {
991 9249 tao
        NodeReference nodeId = null;
992
        if(!replicaStatus) {
993
            //not include those objects whose authoritative node is not this mn
994
            nodeId = new NodeReference();
995
            try {
996
                String currentNodeId = PropertyService.getInstance().getProperty("dataone.nodeId"); // return only pids for which this mn
997
                nodeId.setValue(currentNodeId);
998
            } catch(Exception e) {
999
                throw new ServiceFailure("1580", e.getMessage());
1000
            }
1001
        }
1002
        return super.listObjects(session, startTime, endTime, objectFormatId, identifier, nodeId, start, count);
1003 6229 cjones
    }
1004 6179 cjones
1005 6475 jones
    /**
1006 6476 jones
     * Return a description of the node's capabilities and services.
1007 6475 jones
     *
1008
     * @return node - the technical capabilities of the Member Node
1009
     *
1010
     * @throws ServiceFailure
1011
     * @throws NotAuthorized
1012
     * @throws InvalidRequest
1013
     * @throws NotImplemented
1014
     */
1015
    @Override
1016 6610 cjones
    public Node getCapabilities()
1017
        throws NotImplemented, ServiceFailure {
1018 6179 cjones
1019 6475 jones
        String nodeName = null;
1020
        String nodeId = null;
1021 6492 jones
        String subject = null;
1022 6938 cjones
        String contactSubject = null;
1023 6475 jones
        String nodeDesc = null;
1024 6476 jones
        String nodeTypeString = null;
1025
        NodeType nodeType = null;
1026 9253 tao
        List<String> mnCoreServiceVersions = null;
1027
        List<String> mnReadServiceVersions = null;
1028
        List<String> mnAuthorizationServiceVersions = null;
1029
        List<String> mnStorageServiceVersions = null;
1030
        List<String> mnReplicationServiceVersions = null;
1031 6179 cjones
1032 6475 jones
        boolean nodeSynchronize = false;
1033
        boolean nodeReplicate = false;
1034 9253 tao
        List<String> mnCoreServiceAvailables = null;
1035
        List<String> mnReadServiceAvailables = null;
1036
        List<String> mnAuthorizationServiceAvailables = null;
1037
        List<String> mnStorageServiceAvailables = null;
1038
        List<String> mnReplicationServiceAvailables = null;
1039 6179 cjones
1040 6475 jones
        try {
1041
            // get the properties of the node based on configuration information
1042 9253 tao
            nodeName = Settings.getConfiguration().getString("dataone.nodeName");
1043
            nodeId = Settings.getConfiguration().getString("dataone.nodeId");
1044
            subject = Settings.getConfiguration().getString("dataone.subject");
1045
            contactSubject = Settings.getConfiguration().getString("dataone.contactSubject");
1046
            nodeDesc = Settings.getConfiguration().getString("dataone.nodeDescription");
1047
            nodeTypeString = Settings.getConfiguration().getString("dataone.nodeType");
1048 6476 jones
            nodeType = NodeType.convert(nodeTypeString);
1049 9253 tao
            nodeSynchronize = new Boolean(Settings.getConfiguration().getString("dataone.nodeSynchronize")).booleanValue();
1050
            nodeReplicate = new Boolean(Settings.getConfiguration().getString("dataone.nodeReplicate")).booleanValue();
1051 6475 jones
1052 6476 jones
            // Set the properties of the node based on configuration information and
1053
            // calls to current status methods
1054 7286 leinfelder
            String serviceName = SystemUtil.getSecureContextURL() + "/" + PropertyService.getProperty("dataone.serviceName");
1055 6476 jones
            Node node = new Node();
1056 6542 leinfelder
            node.setBaseURL(serviceName + "/" + nodeTypeString);
1057 6476 jones
            node.setDescription(nodeDesc);
1058 6475 jones
1059 6476 jones
            // set the node's health information
1060
            node.setState(NodeState.UP);
1061
1062
            // set the ping response to the current value
1063
            Ping canPing = new Ping();
1064
            canPing.setSuccess(false);
1065
            try {
1066 6803 leinfelder
            	Date pingDate = ping();
1067
                canPing.setSuccess(pingDate != null);
1068
            } catch (BaseException e) {
1069 6476 jones
                e.printStackTrace();
1070 6803 leinfelder
                // guess it can't be pinged
1071 6476 jones
            }
1072 6610 cjones
1073 6476 jones
            node.setPing(canPing);
1074 6475 jones
1075 6476 jones
            NodeReference identifier = new NodeReference();
1076
            identifier.setValue(nodeId);
1077
            node.setIdentifier(identifier);
1078 6492 jones
            Subject s = new Subject();
1079
            s.setValue(subject);
1080
            node.addSubject(s);
1081 6938 cjones
            Subject contact = new Subject();
1082
            contact.setValue(contactSubject);
1083
            node.addContactSubject(contact);
1084 6476 jones
            node.setName(nodeName);
1085
            node.setReplicate(nodeReplicate);
1086
            node.setSynchronize(nodeSynchronize);
1087 6475 jones
1088 6476 jones
            // services: MNAuthorization, MNCore, MNRead, MNReplication, MNStorage
1089
            Services services = new Services();
1090 6475 jones
1091 9253 tao
            mnCoreServiceVersions = Settings.getConfiguration().getList("dataone.mnCore.serviceVersion");
1092
            mnCoreServiceAvailables = Settings.getConfiguration().getList("dataone.mnCore.serviceAvailable");
1093
            if(mnCoreServiceVersions != null && mnCoreServiceAvailables != null && mnCoreServiceVersions.size() == mnCoreServiceAvailables.size()) {
1094
                for(int i=0; i<mnCoreServiceVersions.size(); i++) {
1095
                    String version = mnCoreServiceVersions.get(i);
1096
                    boolean available = new Boolean(mnCoreServiceAvailables.get(i)).booleanValue();
1097
                    Service sMNCore = new Service();
1098
                    sMNCore.setName("MNCore");
1099
                    sMNCore.setVersion(version);
1100
                    sMNCore.setAvailable(available);
1101
                    services.addService(sMNCore);
1102
                }
1103
            }
1104
1105
            mnReadServiceVersions = Settings.getConfiguration().getList("dataone.mnRead.serviceVersion");
1106
            mnReadServiceAvailables = Settings.getConfiguration().getList("dataone.mnRead.serviceAvailable");
1107
            if(mnReadServiceVersions != null && mnReadServiceAvailables != null && mnReadServiceVersions.size()==mnReadServiceAvailables.size()) {
1108
                for(int i=0; i<mnReadServiceVersions.size(); i++) {
1109
                    String version = mnReadServiceVersions.get(i);
1110
                    boolean available = new Boolean(mnReadServiceAvailables.get(i)).booleanValue();
1111
                    Service sMNRead = new Service();
1112
                    sMNRead.setName("MNRead");
1113
                    sMNRead.setVersion(version);
1114
                    sMNRead.setAvailable(available);
1115
                    services.addService(sMNRead);
1116
                }
1117
            }
1118
1119
            mnAuthorizationServiceVersions = Settings.getConfiguration().getList("dataone.mnAuthorization.serviceVersion");
1120
            mnAuthorizationServiceAvailables = Settings.getConfiguration().getList("dataone.mnAuthorization.serviceAvailable");
1121
            if(mnAuthorizationServiceVersions != null && mnAuthorizationServiceAvailables != null && mnAuthorizationServiceVersions.size()==mnAuthorizationServiceAvailables.size()) {
1122
                for(int i=0; i<mnAuthorizationServiceVersions.size(); i++) {
1123
                    String version = mnAuthorizationServiceVersions.get(i);
1124
                    boolean available = new Boolean(mnAuthorizationServiceAvailables.get(i)).booleanValue();
1125
                    Service sMNAuthorization = new Service();
1126
                    sMNAuthorization.setName("MNAuthorization");
1127
                    sMNAuthorization.setVersion(version);
1128
                    sMNAuthorization.setAvailable(available);
1129
                    services.addService(sMNAuthorization);
1130
                }
1131
            }
1132
1133
            mnStorageServiceVersions = Settings.getConfiguration().getList("dataone.mnStorage.serviceVersion");
1134
            mnStorageServiceAvailables = Settings.getConfiguration().getList("dataone.mnStorage.serviceAvailable");
1135
            if(mnStorageServiceVersions != null && mnStorageServiceAvailables != null && mnStorageServiceVersions.size() == mnStorageServiceAvailables.size()) {
1136
                for(int i=0; i<mnStorageServiceVersions.size(); i++) {
1137
                    String version = mnStorageServiceVersions.get(i);
1138
                    boolean available = new Boolean(mnStorageServiceAvailables.get(i)).booleanValue();
1139
                    Service sMNStorage = new Service();
1140
                    sMNStorage.setName("MNStorage");
1141
                    sMNStorage.setVersion(version);
1142
                    sMNStorage.setAvailable(available);
1143
                    services.addService(sMNStorage);
1144
                }
1145
            }
1146
1147
            mnReplicationServiceVersions = Settings.getConfiguration().getList("dataone.mnReplication.serviceVersion");
1148
            mnReplicationServiceAvailables = Settings.getConfiguration().getList("dataone.mnReplication.serviceAvailable");
1149
            if(mnReplicationServiceVersions != null && mnReplicationServiceAvailables != null && mnReplicationServiceVersions.size() == mnReplicationServiceAvailables.size()) {
1150
                for (int i=0; i<mnReplicationServiceVersions.size(); i++) {
1151
                    String version = mnReplicationServiceVersions.get(i);
1152
                    boolean available = new Boolean(mnReplicationServiceAvailables.get(i)).booleanValue();
1153
                    Service sMNReplication = new Service();
1154
                    sMNReplication.setName("MNReplication");
1155
                    sMNReplication.setVersion(version);
1156
                    sMNReplication.setAvailable(available);
1157
                    services.addService(sMNReplication);
1158
                }
1159
            }
1160
1161 6476 jones
            node.setServices(services);
1162 6475 jones
1163 6476 jones
            // Set the schedule for synchronization
1164
            Synchronization synchronization = new Synchronization();
1165
            Schedule schedule = new Schedule();
1166
            Date now = new Date();
1167 6689 leinfelder
            schedule.setYear(PropertyService.getProperty("dataone.nodeSynchronization.schedule.year"));
1168
            schedule.setMon(PropertyService.getProperty("dataone.nodeSynchronization.schedule.mon"));
1169
            schedule.setMday(PropertyService.getProperty("dataone.nodeSynchronization.schedule.mday"));
1170
            schedule.setWday(PropertyService.getProperty("dataone.nodeSynchronization.schedule.wday"));
1171
            schedule.setHour(PropertyService.getProperty("dataone.nodeSynchronization.schedule.hour"));
1172
            schedule.setMin(PropertyService.getProperty("dataone.nodeSynchronization.schedule.min"));
1173
            schedule.setSec(PropertyService.getProperty("dataone.nodeSynchronization.schedule.sec"));
1174 6476 jones
            synchronization.setSchedule(schedule);
1175
            synchronization.setLastHarvested(now);
1176
            synchronization.setLastCompleteHarvest(now);
1177
            node.setSynchronization(synchronization);
1178 6475 jones
1179 6476 jones
            node.setType(nodeType);
1180
            return node;
1181 6475 jones
1182 6476 jones
        } catch (PropertyNotFoundException pnfe) {
1183
            String msg = "MNodeService.getCapabilities(): " + "property not found: " + pnfe.getMessage();
1184
            logMetacat.error(msg);
1185
            throw new ServiceFailure("2162", msg);
1186
        }
1187 6228 cjones
    }
1188 6179 cjones
1189 8810 leinfelder
1190 6179 cjones
1191 6475 jones
    /**
1192
     * A callback method used by a CN to indicate to a MN that it cannot
1193
     * complete synchronization of the science metadata identified by pid.  Log
1194
     * the event in the metacat event log.
1195
     *
1196
     * @param session
1197
     * @param syncFailed
1198
     *
1199
     * @throws ServiceFailure
1200
     * @throws NotAuthorized
1201
     * @throws NotImplemented
1202
     */
1203
    @Override
1204 6991 leinfelder
    public boolean synchronizationFailed(Session session, SynchronizationFailed syncFailed)
1205 6610 cjones
        throws NotImplemented, ServiceFailure, NotAuthorized {
1206 6179 cjones
1207 6475 jones
        String localId;
1208 7075 cjones
        Identifier pid;
1209
        if ( syncFailed.getPid() != null ) {
1210
            pid = new Identifier();
1211
            pid.setValue(syncFailed.getPid());
1212
            boolean allowed;
1213
1214
            //are we allowed? only CNs
1215
            try {
1216 7142 leinfelder
                allowed = isAdminAuthorized(session);
1217 7075 cjones
                if ( !allowed ){
1218
                    throw new NotAuthorized("2162",
1219
                            "Not allowed to call synchronizationFailed() on this node.");
1220
                }
1221
            } catch (InvalidToken e) {
1222
                throw new NotAuthorized("2162",
1223
                        "Not allowed to call synchronizationFailed() on this node.");
1224 6331 leinfelder
1225 7075 cjones
            }
1226
1227
        } else {
1228
            throw new ServiceFailure("2161", "The identifier cannot be null.");
1229
1230
        }
1231
1232 6475 jones
        try {
1233 7075 cjones
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1234 6475 jones
        } catch (McdbDocNotFoundException e) {
1235 7075 cjones
            throw new ServiceFailure("2161", "The identifier specified by " +
1236
                    syncFailed.getPid() + " was not found on this node.");
1237 6179 cjones
1238 9024 tao
        } catch (SQLException e) {
1239
            throw new ServiceFailure("2161", "Couldn't identify the local id of the identifier specified by " +
1240
                    syncFailed.getPid() + " since "+e.getMessage());
1241 6475 jones
        }
1242
        // TODO: update the CN URL below when the CNRead.SynchronizationFailed
1243
        // method is changed to include the URL as a parameter
1244 7075 cjones
        logMetacat.debug("Synchronization for the object identified by " +
1245
                pid.getValue() + " failed from " + syncFailed.getNodeId() +
1246
                " Logging the event to the Metacat EventLog as a 'syncFailed' event.");
1247 6475 jones
        // TODO: use the event type enum when the SYNCHRONIZATION_FAILED event is added
1248 6532 leinfelder
        String principal = Constants.SUBJECT_PUBLIC;
1249 6506 leinfelder
        if (session != null && session.getSubject() != null) {
1250 6575 cjones
          principal = session.getSubject().getValue();
1251 6506 leinfelder
        }
1252
        try {
1253 6575 cjones
          EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), principal, localId, "synchronization_failed");
1254 6506 leinfelder
        } catch (Exception e) {
1255 7075 cjones
            throw new ServiceFailure("2161", "Could not log the error for: " + pid.getValue());
1256 6991 leinfelder
        }
1257 6475 jones
        //EventLog.getInstance().log("CN URL WILL GO HERE",
1258
        //  session.getSubject().getValue(), localId, Event.SYNCHRONIZATION_FAILED);
1259 6991 leinfelder
        return true;
1260 6179 cjones
1261 6260 cjones
    }
1262
1263 6475 jones
    /**
1264
     * Essentially a get() but with different logging behavior
1265
     */
1266
    @Override
1267 6540 cjones
    public InputStream getReplica(Session session, Identifier pid)
1268 6653 leinfelder
        throws NotAuthorized, NotImplemented, ServiceFailure, InvalidToken {
1269 6179 cjones
1270 6540 cjones
        logMetacat.info("MNodeService.getReplica() called.");
1271
1272 6653 leinfelder
        // cannot be called by public
1273
        if (session == null) {
1274
        	throw new InvalidToken("2183", "No session was provided.");
1275
        }
1276
1277 6631 cjones
        logMetacat.info("MNodeService.getReplica() called with parameters: \n" +
1278
             "\tSession.Subject      = " + session.getSubject().getValue() + "\n" +
1279
             "\tIdentifier           = " + pid.getValue());
1280
1281 6475 jones
        InputStream inputStream = null; // bytes to be returned
1282
        handler = new MetacatHandler(new Timer());
1283
        boolean allowed = false;
1284
        String localId; // the metacat docid for the pid
1285 6179 cjones
1286 6475 jones
        // get the local docid from Metacat
1287
        try {
1288
            localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
1289
        } catch (McdbDocNotFoundException e) {
1290 6610 cjones
            throw new ServiceFailure("2181", "The object specified by " +
1291
                    pid.getValue() + " does not exist at this node.");
1292
1293 9024 tao
        } catch (SQLException e) {
1294
            throw new ServiceFailure("2181", "The local id of the object specified by " +
1295
                    pid.getValue() + " couldn't be identified since "+e.getMessage());
1296 6475 jones
        }
1297 6234 cjones
1298 6552 leinfelder
        Subject targetNodeSubject = session.getSubject();
1299 6185 leinfelder
1300 6552 leinfelder
        // check for authorization to replicate, null session to act as this source MN
1301 6610 cjones
        try {
1302 6777 leinfelder
            allowed = D1Client.getCN().isNodeAuthorized(null, targetNodeSubject, pid);
1303 6610 cjones
        } catch (InvalidToken e1) {
1304
            throw new ServiceFailure("2181", "Could not determine if node is authorized: "
1305
                + e1.getMessage());
1306
1307
        } catch (NotFound e1) {
1308
            throw new ServiceFailure("2181", "Could not determine if node is authorized: "
1309
                    + e1.getMessage());
1310 6384 cjones
1311 6610 cjones
        } catch (InvalidRequest e1) {
1312
            throw new ServiceFailure("2181", "Could not determine if node is authorized: "
1313
                    + e1.getMessage());
1314
1315
        }
1316
1317 6540 cjones
        logMetacat.info("Called D1Client.isNodeAuthorized(). Allowed = " + allowed +
1318
            " for identifier " + pid.getValue());
1319
1320 6475 jones
        // if the person is authorized, perform the read
1321
        if (allowed) {
1322
            try {
1323 6986 jones
                inputStream = MetacatHandler.read(localId);
1324 6475 jones
            } catch (Exception e) {
1325 6610 cjones
                throw new ServiceFailure("1020", "The object specified by " +
1326
                    pid.getValue() + "could not be returned due to error: " + e.getMessage());
1327 6475 jones
            }
1328
        }
1329 6384 cjones
1330 6475 jones
        // if we fail to set the input stream
1331
        if (inputStream == null) {
1332 6610 cjones
            throw new ServiceFailure("2181", "The object specified by " +
1333
                pid.getValue() + "does not exist at this node.");
1334 6475 jones
        }
1335
1336
        // log the replica event
1337
        String principal = null;
1338
        if (session.getSubject() != null) {
1339
            principal = session.getSubject().getValue();
1340
        }
1341 6576 cjones
        EventLog.getInstance().log(request.getRemoteAddr(),
1342
            request.getHeader("User-Agent"), principal, localId, "replicate");
1343 6475 jones
1344
        return inputStream;
1345
    }
1346
1347 6573 cjones
    /**
1348 6600 cjones
     * A method to notify the Member Node that the authoritative copy of
1349 6599 cjones
     * system metadata on the Coordinating Nodes has changed.
1350
     *
1351
     * @param session   Session information that contains the identity of the
1352
     *                  calling user as retrieved from the X.509 certificate
1353
     *                  which must be traceable to the CILogon service.
1354
     * @param serialVersion   The serialVersion of the system metadata
1355
     * @param dateSysMetaLastModified  The time stamp for when the system metadata was changed
1356
     * @throws NotImplemented
1357
     * @throws ServiceFailure
1358
     * @throws NotAuthorized
1359
     * @throws InvalidRequest
1360
     * @throws InvalidToken
1361
     */
1362 6991 leinfelder
    public boolean systemMetadataChanged(Session session, Identifier pid,
1363 6599 cjones
        long serialVersion, Date dateSysMetaLastModified)
1364
        throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest,
1365
        InvalidToken {
1366
1367 7600 cjones
        // cannot be called by public
1368
        if (session == null) {
1369 9051 tao
        	throw new InvalidToken("1332", "No session was provided.");
1370 7600 cjones
        }
1371
1372 9051 tao
        String serviceFailureCode = "1333";
1373
        Identifier sid = getPIDForSID(pid, serviceFailureCode);
1374
        if(sid != null) {
1375
            pid = sid;
1376
        }
1377
1378 6600 cjones
        SystemMetadata currentLocalSysMeta = null;
1379
        SystemMetadata newSysMeta = null;
1380
        CNode cn = D1Client.getCN();
1381
        NodeList nodeList = null;
1382
        Subject callingSubject = null;
1383
        boolean allowed = false;
1384
1385
        // are we allowed to call this?
1386
        callingSubject = session.getSubject();
1387
        nodeList = cn.listNodes();
1388
1389
        for(Node node : nodeList.getNodeList()) {
1390
            // must be a CN
1391
            if ( node.getType().equals(NodeType.CN)) {
1392
               List<Subject> subjectList = node.getSubjectList();
1393
               // the calling subject must be in the subject list
1394
               if ( subjectList.contains(callingSubject)) {
1395
                   allowed = true;
1396
1397
               }
1398
1399
            }
1400
        }
1401
1402
        if (!allowed ) {
1403
            String msg = "The subject identified by " + callingSubject.getValue() +
1404
              " is not authorized to call this service.";
1405
            throw new NotAuthorized("1331", msg);
1406
1407
        }
1408
1409
        // compare what we have locally to what is sent in the change notification
1410
        try {
1411 6692 leinfelder
            currentLocalSysMeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
1412
1413
        } catch (RuntimeException e) {
1414 6600 cjones
            String msg = "SystemMetadata for pid " + pid.getValue() +
1415 6692 leinfelder
              " couldn't be updated because it couldn't be found locally: " +
1416 6600 cjones
              e.getMessage();
1417 6692 leinfelder
            logMetacat.error(msg);
1418
            ServiceFailure sf = new ServiceFailure("1333", msg);
1419
            sf.initCause(e);
1420
            throw sf;
1421 6600 cjones
        }
1422
1423
        if (currentLocalSysMeta.getSerialVersion().longValue() < serialVersion ) {
1424
            try {
1425
                newSysMeta = cn.getSystemMetadata(null, pid);
1426
            } catch (NotFound e) {
1427
                // huh? you just said you had it
1428 6692 leinfelder
            	String msg = "On updating the local copy of system metadata " +
1429
                "for pid " + pid.getValue() +", the CN reports it is not found." +
1430
                " The error message was: " + e.getMessage();
1431
                logMetacat.error(msg);
1432
                ServiceFailure sf = new ServiceFailure("1333", msg);
1433
                sf.initCause(e);
1434
                throw sf;
1435 6600 cjones
            }
1436 6692 leinfelder
1437 9056 tao
            //check about the sid in the system metadata
1438
            Identifier newSID = newSysMeta.getSeriesId();
1439
            if(newSID != null) {
1440
                if (!isValidIdentifier(newSID)) {
1441
                    throw new InvalidRequest("1334", "The series identifier in the new system metadata is invalid.");
1442
                }
1443
                Identifier currentSID = currentLocalSysMeta.getSeriesId();
1444
                if( currentSID != null && currentSID.getValue() != null) {
1445
                    if(!newSID.getValue().equals(currentSID.getValue())) {
1446
                        //newSID doesn't match the currentSID. The newSID shouldn't be used.
1447
                        try {
1448
                            if(IdentifierManager.getInstance().identifierExists(newSID.getValue())) {
1449
                                throw new InvalidRequest("1334", "The series identifier "+newSID.getValue()+" in the new system metadata has been used by another object.");
1450
                            }
1451
                        } catch (SQLException sql) {
1452
                            throw new ServiceFailure("1333", "Couldn't determine if the SID "+newSID.getValue()+" in the system metadata exists in the node since "+sql.getMessage());
1453
                        }
1454
1455
                    }
1456
                } else {
1457
                    //newSID shouldn't be used
1458
                    try {
1459
                        if(IdentifierManager.getInstance().identifierExists(newSID.getValue())) {
1460
                            throw new InvalidRequest("1334", "The series identifier "+newSID.getValue()+" in the new system metadata has been used by another object.");
1461
                        }
1462
                    } catch (SQLException sql) {
1463
                        throw new ServiceFailure("1333", "Couldn't determine if the SID "+newSID.getValue()+" in the system metadata exists in the node since "+sql.getMessage());
1464
                    }
1465
                }
1466
            }
1467 6600 cjones
            // update the local copy of system metadata for the pid
1468
            try {
1469 6692 leinfelder
                HazelcastService.getInstance().getSystemMetadataMap().put(newSysMeta.getIdentifier(), newSysMeta);
1470 6600 cjones
                logMetacat.info("Updated local copy of system metadata for pid " +
1471
                    pid.getValue() + " after change notification from the CN.");
1472
1473 8599 leinfelder
                // TODO: consider inspecting the change for archive
1474
                // see: https://projects.ecoinformatics.org/ecoinfo/issues/6417
1475
//                if (newSysMeta.getArchived() != null && newSysMeta.getArchived().booleanValue()) {
1476
//                	try {
1477
//						this.archive(session, newSysMeta.getIdentifier());
1478
//					} catch (NotFound e) {
1479
//						// do we care? nothing to do about it now
1480
//						logMetacat.error(e.getMessage(), e);
1481
//					}
1482
//                }
1483
1484 6692 leinfelder
            } catch (RuntimeException e) {
1485 6600 cjones
                String msg = "SystemMetadata for pid " + pid.getValue() +
1486 6692 leinfelder
                  " couldn't be updated: " +
1487 6600 cjones
                  e.getMessage();
1488 6692 leinfelder
                logMetacat.error(msg);
1489
                ServiceFailure sf = new ServiceFailure("1333", msg);
1490
                sf.initCause(e);
1491
                throw sf;
1492 6600 cjones
            }
1493 8464 leinfelder
1494 9248 leinfelder
            // attempt to re-register the identifier (it checks if it is a doi)
1495
            try {
1496
    			DOIService.getInstance().registerDOI(newSysMeta);
1497
    		} catch (Exception e) {
1498
    			logMetacat.warn("Could not [re]register DOI: " + e.getMessage(), e);
1499
    		}
1500
1501 8464 leinfelder
            // submit for indexing
1502
            try {
1503 8647 leinfelder
				MetacatSolrIndex.getInstance().submit(newSysMeta.getIdentifier(), newSysMeta, null, true);
1504 8464 leinfelder
			} catch (Exception e) {
1505
                logMetacat.error("Could not submit changed systemMetadata for indexing, pid: " + newSysMeta.getIdentifier().getValue(), e);
1506
			}
1507 6600 cjones
        }
1508
1509 6991 leinfelder
        return true;
1510
1511 6599 cjones
    }
1512
1513 6795 cjones
    /*
1514
     * Set the replication status for the object on the Coordinating Node
1515
     *
1516
     * @param session - the session for the this target node
1517
     * @param pid - the identifier of the object being updated
1518
     * @param nodeId - the identifier of this target node
1519
     * @param status - the replication status to set
1520
     * @param failure - the exception to include, if any
1521
     */
1522
    private void setReplicationStatus(Session session, Identifier pid,
1523
        NodeReference nodeId, ReplicationStatus status, BaseException failure)
1524
        throws ServiceFailure, NotImplemented, NotAuthorized,
1525
        InvalidRequest {
1526
1527
        // call the CN as the MN to set the replication status
1528
        try {
1529
            this.cn = D1Client.getCN();
1530
            this.cn.setReplicationStatus(session, pid, nodeId,
1531
                    status, failure);
1532
1533
        } catch (InvalidToken e) {
1534 7091 leinfelder
        	String msg = "Could not set the replication status for " + pid.getValue() + " on the CN (InvalidToken): " + e.getMessage();
1535
            logMetacat.error(msg);
1536
        	throw new ServiceFailure("2151",
1537
                    msg);
1538 6795 cjones
1539
        } catch (NotFound e) {
1540 7091 leinfelder
        	String msg = "Could not set the replication status for " + pid.getValue() + " on the CN (NotFound): " + e.getMessage();
1541
            logMetacat.error(msg);
1542
        	throw new ServiceFailure("2151",
1543
                    msg);
1544 6795 cjones
1545
        }
1546
    }
1547 8591 leinfelder
1548
    private SystemMetadata makePublicIfNot(SystemMetadata sysmeta, Identifier pid) throws ServiceFailure, InvalidToken, NotFound, NotImplemented, InvalidRequest {
1549
    	// check if it is publicly readable
1550
		boolean isPublic = false;
1551
		Subject publicSubject = new Subject();
1552
		publicSubject.setValue(Constants.SUBJECT_PUBLIC);
1553
		Session publicSession = new Session();
1554
		publicSession.setSubject(publicSubject);
1555
		AccessRule publicRule = new AccessRule();
1556
		publicRule.addPermission(Permission.READ);
1557
		publicRule.addSubject(publicSubject);
1558
1559
		// see if we need to add the rule
1560
		try {
1561
			isPublic = this.isAuthorized(publicSession, pid, Permission.READ);
1562
		} catch (NotAuthorized na) {
1563
			// well, certainly not authorized for public read!
1564
		}
1565
		if (!isPublic) {
1566
			sysmeta.getAccessPolicy().addAllow(publicRule);
1567
		}
1568
1569
		return sysmeta;
1570
    }
1571 7099 leinfelder
1572
	@Override
1573 7441 leinfelder
	public Identifier generateIdentifier(Session session, String scheme, String fragment)
1574 7099 leinfelder
			throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented,
1575
			InvalidRequest {
1576 7448 leinfelder
1577 8210 leinfelder
		// check for null session
1578
        if (session == null) {
1579
          throw new InvalidToken("2190", "Session is required to generate an Identifier at this Node.");
1580
        }
1581
1582 7441 leinfelder
		Identifier identifier = new Identifier();
1583 7448 leinfelder
1584 7489 leinfelder
		// handle different schemes
1585
		if (scheme.equalsIgnoreCase(UUID_SCHEME)) {
1586
			// UUID
1587
			UUID uuid = UUID.randomUUID();
1588
            identifier.setValue(UUID_PREFIX + uuid.toString());
1589
		} else if (scheme.equalsIgnoreCase(DOI_SCHEME)) {
1590 7512 leinfelder
			// generate a DOI
1591 7448 leinfelder
			try {
1592 7512 leinfelder
				identifier = DOIService.getInstance().generateDOI();
1593 7448 leinfelder
			} catch (EZIDException e) {
1594 7512 leinfelder
				ServiceFailure sf = new ServiceFailure("2191", "Could not generate DOI: " + e.getMessage());
1595
				sf.initCause(e);
1596
				throw sf;
1597 7448 leinfelder
			}
1598 7489 leinfelder
		} else {
1599
			// default if we don't know the scheme
1600
			if (fragment != null) {
1601
				// for now, just autogen with fragment
1602
				String autogenId = DocumentUtil.generateDocumentId(fragment, 0);
1603
				identifier.setValue(autogenId);
1604
			} else {
1605
				// autogen with no fragment
1606
				String autogenId = DocumentUtil.generateDocumentId(0);
1607
				identifier.setValue(autogenId);
1608
			}
1609 7448 leinfelder
		}
1610
1611 7441 leinfelder
		// TODO: reserve the identifier with the CN. We can only do this when
1612
		// 1) the MN is part of a CN cluster
1613
		// 2) the request is from an authenticated user
1614
1615
		return identifier;
1616 7099 leinfelder
	}
1617 7144 leinfelder
1618 8810 leinfelder
1619 7144 leinfelder
1620
	@Override
1621 8810 leinfelder
	public QueryEngineDescription getQueryEngineDescription(Session session, String engine)
1622 7144 leinfelder
			throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented,
1623
			NotFound {
1624 7772 tao
	    if(engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) {
1625 8162 tao
	        if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) {
1626
                throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1627
            }
1628 7634 tao
	        QueryEngineDescription qed = new QueryEngineDescription();
1629 7772 tao
	        qed.setName(EnabledQueryEngines.PATHQUERYENGINE);
1630 7634 tao
	        qed.setQueryEngineVersion("1.0");
1631
	        qed.addAdditionalInfo("This is the traditional structured query for Metacat");
1632
	        Vector<String> pathsForIndexing = null;
1633
	        try {
1634
	            pathsForIndexing = SystemUtil.getPathsForIndexing();
1635
	        } catch (MetacatUtilException e) {
1636
	            logMetacat.warn("Could not get index paths", e);
1637
	        }
1638
	        for (String fieldName: pathsForIndexing) {
1639
	            QueryField field = new QueryField();
1640
	            field.addDescription("Indexed field for path '" + fieldName + "'");
1641
	            field.setName(fieldName);
1642
	            field.setReturnable(true);
1643
	            field.setSearchable(true);
1644
	            field.setSortable(false);
1645
	            // TODO: determine type and multivaluedness
1646
	            field.setType(String.class.getName());
1647
	            //field.setMultivalued(true);
1648
	            qed.addQueryField(field);
1649
	        }
1650
	        return qed;
1651 7772 tao
	    } else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) {
1652 7781 tao
	        if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) {
1653 7772 tao
                throw new NotImplemented("0000", "MNodeService.getQueryEngineDescription - the query engine "+engine +" hasn't been implemented or has been disabled.");
1654 7781 tao
            }
1655 7634 tao
	        try {
1656 7662 tao
	            QueryEngineDescription qed = MetacatSolrEngineDescriptionHandler.getInstance().getQueryEngineDescritpion();
1657 7634 tao
	            return qed;
1658
	        } catch (Exception e) {
1659
	            e.printStackTrace();
1660
	            throw new ServiceFailure("Solr server error", e.getMessage());
1661
	        }
1662
	    } else {
1663
	        throw new NotFound("404", "The Metacat member node can't find the query engine - "+engine);
1664
	    }
1665
1666 7417 leinfelder
	}
1667
1668
	@Override
1669 8810 leinfelder
	public QueryEngineList listQueryEngines(Session session) throws InvalidToken,
1670 7417 leinfelder
			ServiceFailure, NotAuthorized, NotImplemented {
1671
		QueryEngineList qel = new QueryEngineList();
1672 7781 tao
		//qel.addQueryEngine(EnabledQueryEngines.PATHQUERYENGINE);
1673
		//qel.addQueryEngine(EnabledQueryEngines.SOLRENGINE);
1674
		List<String> enables = EnabledQueryEngines.getInstance().getEnabled();
1675 7772 tao
		for(String name : enables) {
1676
		    qel.addQueryEngine(name);
1677 7781 tao
		}
1678 7417 leinfelder
		return qel;
1679
	}
1680
1681
	@Override
1682 8810 leinfelder
	public InputStream query(Session session, String engine, String query) throws InvalidToken,
1683 7417 leinfelder
			ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented,
1684
			NotFound {
1685 7648 tao
	    String user = Constants.SUBJECT_PUBLIC;
1686
        String[] groups= null;
1687 7680 tao
        Set<Subject> subjects = null;
1688 7648 tao
        if (session != null) {
1689
            user = session.getSubject().getValue();
1690 7680 tao
            subjects = AuthUtils.authorizedClientSubjects(session);
1691 7648 tao
            if (subjects != null) {
1692
                List<String> groupList = new ArrayList<String>();
1693
                for (Subject subject: subjects) {
1694
                    groupList.add(subject.getValue());
1695
                }
1696
                groups = groupList.toArray(new String[0]);
1697
            }
1698 7680 tao
        } else {
1699
            //add the public user subject to the set
1700
            Subject subject = new Subject();
1701
            subject.setValue(Constants.SUBJECT_PUBLIC);
1702 7757 leinfelder
            subjects = new HashSet<Subject>();
1703 7680 tao
            subjects.add(subject);
1704 7648 tao
        }
1705 7680 tao
        //System.out.println("====== user is "+user);
1706
        //System.out.println("====== groups are "+groups);
1707 7772 tao
		if (engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) {
1708 8162 tao
		    if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) {
1709
                throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1710
            }
1711 7417 leinfelder
			try {
1712
				DBQuery queryobj = new DBQuery();
1713 7648 tao
1714 7417 leinfelder
				String results = queryobj.performPathquery(query, user, groups);
1715 7757 leinfelder
				ContentTypeByteArrayInputStream ctbais = new ContentTypeByteArrayInputStream(results.getBytes(MetaCatServlet.DEFAULT_ENCODING));
1716
				ctbais.setContentType("text/xml");
1717
				return ctbais;
1718 7417 leinfelder
1719
			} catch (Exception e) {
1720 7757 leinfelder
				throw new ServiceFailure("Pathquery error", e.getMessage());
1721 7417 leinfelder
			}
1722
1723 7772 tao
		} else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) {
1724 7781 tao
		    if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) {
1725 7772 tao
		        throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled.");
1726 7781 tao
		    }
1727 7634 tao
		    logMetacat.info("The query is ==================================== \n"+query);
1728 7620 tao
		    try {
1729 7634 tao
1730 7680 tao
                return MetacatSolrIndex.getInstance().query(query, subjects);
1731 7620 tao
            } catch (Exception e) {
1732
                // TODO Auto-generated catch block
1733
                throw new ServiceFailure("Solr server error", e.getMessage());
1734
            }
1735 7417 leinfelder
		}
1736
		return null;
1737
	}
1738 7849 leinfelder
1739
	/**
1740
	 * Given an existing Science Metadata PID, this method mints a DOI
1741
	 * and updates the original object "publishing" the update with the DOI.
1742
	 * This includes updating the ORE map that describes the Science Metadata+data.
1743
	 * TODO: ensure all referenced objects allow public read
1744
	 *
1745
	 * @see https://projects.ecoinformatics.org/ecoinfo/issues/6014
1746
	 *
1747
	 * @param originalIdentifier
1748
	 * @param request
1749 7864 leinfelder
	 * @throws InvalidRequest
1750
	 * @throws NotImplemented
1751
	 * @throws NotAuthorized
1752
	 * @throws ServiceFailure
1753
	 * @throws InvalidToken
1754 7849 leinfelder
	 * @throws NotFound
1755 7864 leinfelder
	 * @throws InvalidSystemMetadata
1756
	 * @throws InsufficientResources
1757
	 * @throws UnsupportedType
1758
	 * @throws IdentifierNotUnique
1759 7849 leinfelder
	 */
1760 7864 leinfelder
	public Identifier publish(Session session, Identifier originalIdentifier) throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, NotFound, IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata {
1761 7849 leinfelder
1762 9092 tao
	    String serviceFailureCode = "1030";
1763
	    Identifier sid = getPIDForSID(originalIdentifier, serviceFailureCode);
1764
	    if(sid != null) {
1765
	        originalIdentifier = sid;
1766
	    }
1767 8141 leinfelder
		// get the original SM
1768
		SystemMetadata originalSystemMetadata = this.getSystemMetadata(session, originalIdentifier);
1769
1770 8594 leinfelder
		// make copy of it using the marshaller to ensure DEEP copy
1771
		SystemMetadata sysmeta = null;
1772 8141 leinfelder
		try {
1773 8594 leinfelder
			ByteArrayOutputStream baos = new ByteArrayOutputStream();
1774
			TypeMarshaller.marshalTypeToOutputStream(originalSystemMetadata, baos);
1775
			sysmeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, new ByteArrayInputStream(baos.toByteArray()));
1776 8141 leinfelder
		} catch (Exception e) {
1777
			// report as service failure
1778
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1779
			sf.initCause(e);
1780
			throw sf;
1781
		}
1782
1783 7849 leinfelder
		// mint a DOI for the new revision
1784
		Identifier newIdentifier = this.generateIdentifier(session, MNodeService.DOI_SCHEME, null);
1785 8141 leinfelder
1786
		// set new metadata values
1787 7849 leinfelder
		sysmeta.setIdentifier(newIdentifier);
1788
		sysmeta.setObsoletes(originalIdentifier);
1789
		sysmeta.setObsoletedBy(null);
1790
1791 8591 leinfelder
		// ensure it is publicly readable
1792
		sysmeta = makePublicIfNot(sysmeta, originalIdentifier);
1793
1794 7849 leinfelder
		// get the bytes
1795
		InputStream inputStream = this.get(session, originalIdentifier);
1796
1797
		// update the object
1798 7864 leinfelder
		this.update(session, originalIdentifier, inputStream, newIdentifier, sysmeta);
1799 7849 leinfelder
1800 7864 leinfelder
		// update ORE that references the scimeta
1801 8190 leinfelder
		// first try the naive method, then check the SOLR index
1802 7849 leinfelder
		try {
1803 7864 leinfelder
			String localId = IdentifierManager.getInstance().getLocalId(originalIdentifier.getValue());
1804 7849 leinfelder
1805 7864 leinfelder
			Identifier potentialOreIdentifier = new Identifier();
1806
			potentialOreIdentifier.setValue(SystemMetadataFactory.RESOURCE_MAP_PREFIX + localId);
1807 7849 leinfelder
1808 7864 leinfelder
			InputStream oreInputStream = null;
1809
			try {
1810
				oreInputStream = this.get(session, potentialOreIdentifier);
1811
			} catch (NotFound nf) {
1812
				// this is probably okay for many sci meta data docs
1813
				logMetacat.warn("No potential ORE map found for: " + potentialOreIdentifier.getValue());
1814 8190 leinfelder
				// try the SOLR index
1815 8200 leinfelder
				List<Identifier> potentialOreIdentifiers = this.lookupOreFor(originalIdentifier, false);
1816 8190 leinfelder
				if (potentialOreIdentifiers != null) {
1817
					potentialOreIdentifier = potentialOreIdentifiers.get(0);
1818
					try {
1819
						oreInputStream = this.get(session, potentialOreIdentifier);
1820
					} catch (NotFound nf2) {
1821
						// this is probably okay for many sci meta data docs
1822
						logMetacat.warn("No potential ORE map found for: " + potentialOreIdentifier.getValue());
1823
					}
1824
				}
1825 7864 leinfelder
			}
1826
			if (oreInputStream != null) {
1827
				Identifier newOreIdentifier = MNodeService.getInstance(request).generateIdentifier(session, MNodeService.UUID_SCHEME, null);
1828
1829
				Map<Identifier, Map<Identifier, List<Identifier>>> resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap(oreInputStream);
1830
				Map<Identifier, List<Identifier>> sciMetaMap = resourceMapStructure.get(potentialOreIdentifier);
1831
				List<Identifier> dataIdentifiers = sciMetaMap.get(originalIdentifier);
1832 8591 leinfelder
1833 7864 leinfelder
				// reconstruct the ORE with the new identifiers
1834
				sciMetaMap.remove(originalIdentifier);
1835
				sciMetaMap.put(newIdentifier, dataIdentifiers);
1836
1837
				ResourceMap resourceMap = ResourceMapFactory.getInstance().createResourceMap(newOreIdentifier, sciMetaMap);
1838
				String resourceMapString = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap);
1839
1840
				// get the original ORE SM and update the values
1841 8141 leinfelder
				SystemMetadata originalOreSysMeta = this.getSystemMetadata(session, potentialOreIdentifier);
1842
				SystemMetadata oreSysMeta = new SystemMetadata();
1843
				try {
1844 8594 leinfelder
					ByteArrayOutputStream baos = new ByteArrayOutputStream();
1845
					TypeMarshaller.marshalTypeToOutputStream(originalOreSysMeta, baos);
1846
					oreSysMeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, new ByteArrayInputStream(baos.toByteArray()));
1847 8141 leinfelder
				} catch (Exception e) {
1848
					// report as service failure
1849
					ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1850
					sf.initCause(e);
1851
					throw sf;
1852
				}
1853
1854 7864 leinfelder
				oreSysMeta.setIdentifier(newOreIdentifier);
1855
				oreSysMeta.setObsoletes(potentialOreIdentifier);
1856
				oreSysMeta.setObsoletedBy(null);
1857
				oreSysMeta.setSize(BigInteger.valueOf(resourceMapString.getBytes("UTF-8").length));
1858
				oreSysMeta.setChecksum(ChecksumUtil.checksum(resourceMapString.getBytes("UTF-8"), oreSysMeta.getChecksum().getAlgorithm()));
1859
1860 8591 leinfelder
				// ensure ORE is publicly readable
1861 8594 leinfelder
				oreSysMeta = makePublicIfNot(oreSysMeta, potentialOreIdentifier);
1862 8591 leinfelder
1863
				// ensure all data objects allow public read
1864
				List<String> pidsToSync = new ArrayList<String>();
1865
				for (Identifier dataId: dataIdentifiers) {
1866
					SystemMetadata dataSysMeta = this.getSystemMetadata(session, dataId);
1867
					dataSysMeta = makePublicIfNot(dataSysMeta, dataId);
1868
					this.updateSystemMetadata(dataSysMeta);
1869
					pidsToSync.add(dataId.getValue());
1870
				}
1871
				SyncAccessPolicy sap = new SyncAccessPolicy();
1872
				try {
1873
					sap.sync(pidsToSync);
1874
				} catch (Exception e) {
1875
					// ignore
1876
					logMetacat.warn("Error attempting to sync access for data objects when publishing package");
1877
				}
1878
1879 7864 leinfelder
				// save the updated ORE
1880
				this.update(
1881
						session,
1882
						potentialOreIdentifier,
1883
						new ByteArrayInputStream(resourceMapString.getBytes("UTF-8")),
1884
						newOreIdentifier,
1885
						oreSysMeta);
1886
1887 8362 leinfelder
			} else {
1888
				// create a new ORE for them
1889
				// https://projects.ecoinformatics.org/ecoinfo/issues/6194
1890
				try {
1891
					// find the local id for the NEW package.
1892
					String newLocalId = IdentifierManager.getInstance().getLocalId(newIdentifier.getValue());
1893
1894
					@SuppressWarnings("unused")
1895
					SystemMetadata extraSysMeta = SystemMetadataFactory.createSystemMetadata(newLocalId, true, false);
1896 8591 leinfelder
					// should be done generating the ORE here, and the same permissions were used from the metadata object
1897 8362 leinfelder
1898
				} catch (Exception e) {
1899
					// oops, guess there was a problem - no package for you
1900
					logMetacat.error("Could not generate new ORE for published object: " + newIdentifier.getValue(), e);
1901
				}
1902 7864 leinfelder
			}
1903
		} catch (McdbDocNotFoundException e) {
1904
			// report as service failure
1905
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1906
			sf.initCause(e);
1907
			throw sf;
1908
		} catch (UnsupportedEncodingException e) {
1909
			// report as service failure
1910
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1911
			sf.initCause(e);
1912
			throw sf;
1913
		} catch (OREException e) {
1914
			// report as service failure
1915
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1916
			sf.initCause(e);
1917
			throw sf;
1918
		} catch (URISyntaxException e) {
1919
			// report as service failure
1920
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1921
			sf.initCause(e);
1922
			throw sf;
1923
		} catch (OREParserException e) {
1924
			// report as service failure
1925
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1926
			sf.initCause(e);
1927
			throw sf;
1928
		} catch (ORESerialiserException e) {
1929
			// report as service failure
1930
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1931
			sf.initCause(e);
1932
			throw sf;
1933
		} catch (NoSuchAlgorithmException e) {
1934
			// report as service failure
1935
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1936
			sf.initCause(e);
1937
			throw sf;
1938 9024 tao
		} catch (SQLException e) {
1939
            // report as service failure
1940
            ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
1941
            sf.initCause(e);
1942
            throw sf;
1943
        }
1944 7849 leinfelder
1945
		return newIdentifier;
1946
	}
1947 7850 leinfelder
1948
	/**
1949 8190 leinfelder
	 * Determines if we already have registered an ORE map for this package
1950
	 * NOTE: uses a solr query to locate OREs for the object
1951
	 * @param guid of the EML/packaging object
1952
	 * @return list of resource map identifiers for the given pid
1953
	 */
1954 8200 leinfelder
	public List<Identifier> lookupOreFor(Identifier guid, boolean includeObsolete) {
1955 8190 leinfelder
		// Search for the ORE if we can find it
1956
		String pid = guid.getValue();
1957
		List<Identifier> retList = null;
1958
		try {
1959 8745 leinfelder
			String query = "fl=id,resourceMap&wt=xml&q=-obsoletedBy:[* TO *]+resourceMap:[* TO *]+id:\"" + pid + "\"";
1960 8200 leinfelder
			if (includeObsolete) {
1961 8745 leinfelder
				query = "fl=id,resourceMap&wt=xml&q=resourceMap:[* TO *]+id:\"" + pid + "\"";
1962 8200 leinfelder
			}
1963
1964 8810 leinfelder
			InputStream results = this.query(null, "solr", query);
1965 8190 leinfelder
			org.w3c.dom.Node rootNode = XMLUtilities.getXMLReaderAsDOMTreeRootNode(new InputStreamReader(results, "UTF-8"));
1966
			//String resultString = XMLUtilities.getDOMTreeAsString(rootNode);
1967
			org.w3c.dom.NodeList nodeList = XMLUtilities.getNodeListWithXPath(rootNode, "//arr[@name=\"resourceMap\"]/str");
1968
			if (nodeList != null && nodeList.getLength() > 0) {
1969
				retList = new ArrayList<Identifier>();
1970
				for (int i = 0; i < nodeList.getLength(); i++) {
1971
					String found = nodeList.item(i).getFirstChild().getNodeValue();
1972
					Identifier oreId = new Identifier();
1973
					oreId.setValue(found);
1974
					retList.add(oreId);
1975
				}
1976
			}
1977
		} catch (Exception e) {
1978
			logMetacat.error("Error checking for resourceMap[s] on pid " + pid + ". " + e.getMessage(), e);
1979
		}
1980
1981
		return retList;
1982
	}
1983
1984 8810 leinfelder
1985
	@Override
1986
	public InputStream getPackage(Session session, ObjectFormatIdentifier formatId,
1987
			Identifier pid) throws InvalidToken, ServiceFailure,
1988
			NotAuthorized, InvalidRequest, NotImplemented, NotFound {
1989 9290 tao
	    if(formatId == null) {
1990
	        throw new InvalidRequest("2873", "The format type can't be null in the getpackage method.");
1991
	    } else if(!formatId.getValue().equals("application/bagit")) {
1992
	        throw new NotImplemented("", "The format "+formatId.getValue()+" is not supported in the getpackage method");
1993
	    }
1994 9092 tao
	    String serviceFailureCode = "2871";
1995
	    Identifier sid = getPIDForSID(pid, serviceFailureCode);
1996
	    if(sid != null) {
1997
	        pid = sid;
1998
	    }
1999 7850 leinfelder
		InputStream bagInputStream = null;
2000
		BagFactory bagFactory = new BagFactory();
2001
		Bag bag = bagFactory.createBag();
2002
2003
		// track the temp files we use so we can delete them when finished
2004
		List<File> tempFiles = new ArrayList<File>();
2005
2006
		// the pids to include in the package
2007
		List<Identifier> packagePids = new ArrayList<Identifier>();
2008
2009 7855 leinfelder
		// catch non-D1 service errors and throw as ServiceFailures
2010
		try {
2011 8437 walker
			//Create a map of dataone ids and file names
2012
			Map<Identifier, String> fileNames = new HashMap<Identifier, String>();
2013 7855 leinfelder
2014 8800 leinfelder
			// track the pid-to-file mapping
2015
			StringBuffer pidMapping = new StringBuffer();
2016
2017 7855 leinfelder
			// find the package contents
2018
			SystemMetadata sysMeta = this.getSystemMetadata(session, pid);
2019 8025 leinfelder
			if (ObjectFormatCache.getInstance().getFormat(sysMeta.getFormatId()).getFormatType().equals("RESOURCE")) {
2020 8437 walker
				//Get the resource map as a map of Identifiers
2021 7855 leinfelder
				InputStream oreInputStream = this.get(session, pid);
2022
				Map<Identifier, Map<Identifier, List<Identifier>>> resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap(oreInputStream);
2023
				packagePids.addAll(resourceMapStructure.keySet());
2024 8437 walker
				//Loop through each object in this resource map
2025 7855 leinfelder
				for (Map<Identifier, List<Identifier>> entries: resourceMapStructure.values()) {
2026 8437 walker
					//Loop through each metadata object in this entry
2027
					Set<Identifier> metadataIdentifiers = entries.keySet();
2028
					for(Identifier metadataID: metadataIdentifiers){
2029
						try{
2030
							//Get the system metadata for this metadata object
2031
							SystemMetadata metadataSysMeta = this.getSystemMetadata(session, metadataID);
2032
2033 8800 leinfelder
							// include user-friendly metadata
2034
							if (ObjectFormatCache.getInstance().getFormat(metadataSysMeta.getFormatId()).getFormatType().equals("METADATA")) {
2035
								InputStream metadataStream = this.get(session, metadataID);
2036
2037
								try {
2038
									// transform
2039
						            String format = "default";
2040
2041
									DBTransform transformer = new DBTransform();
2042
						            String documentContent = IOUtils.toString(metadataStream, "UTF-8");
2043
						            String sourceType = metadataSysMeta.getFormatId().getValue();
2044
						            String targetType = "-//W3C//HTML//EN";
2045
						            ByteArrayOutputStream baos = new ByteArrayOutputStream();
2046
						            Writer writer = new OutputStreamWriter(baos , "UTF-8");
2047
						            // TODO: include more params?
2048
						            Hashtable<String, String[]> params = new Hashtable<String, String[]>();
2049
						            String localId = null;
2050
									try {
2051
										localId = IdentifierManager.getInstance().getLocalId(pid.getValue());
2052
									} catch (McdbDocNotFoundException e) {
2053
										throw new NotFound("1020", e.getMessage());
2054
									}
2055
									params.put("qformat", new String[] {format});
2056
						            params.put("docid", new String[] {localId});
2057
						            params.put("pid", new String[] {pid.getValue()});
2058
						            params.put("displaymodule", new String[] {"printall"});
2059
2060
						            transformer.transformXMLDocument(
2061
						                    documentContent ,
2062
						                    sourceType,
2063
						                    targetType ,
2064
						                    format,
2065
						                    writer,
2066
						                    params,
2067
						                    null //sessionid
2068
						                    );
2069
2070
						            // finally, get the HTML back
2071
						            ContentTypeByteArrayInputStream resultInputStream = new ContentTypeByteArrayInputStream(baos.toByteArray());
2072
2073
						            // write to temp file with correct css path
2074
						            File tmpDir = File.createTempFile("package_", "_dir");
2075
						            tmpDir.delete();
2076
						            tmpDir.mkdir();
2077
						            File htmlFile = File.createTempFile("metadata", ".html", tmpDir);
2078
						            File cssDir = new File(tmpDir, format);
2079
						            cssDir.mkdir();
2080
						            File cssFile = new File(tmpDir, format + "/" + format + ".css");
2081
						            String pdfFileName = metadataID.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-METADATA.pdf";
2082
						            File pdfFile = new File(tmpDir, pdfFileName);
2083
						            //File pdfFile = File.createTempFile("metadata", ".pdf", tmpDir);
2084
2085
						            // put the CSS file in place for the html to find it
2086
						            String originalCssPath = SystemUtil.getContextDir() + "/style/skins/" + format + "/" + format + ".css";
2087
						            IOUtils.copy(new FileInputStream(originalCssPath), new FileOutputStream(cssFile));
2088
2089
						            // write the HTML file
2090
						            IOUtils.copy(resultInputStream, new FileOutputStream(htmlFile));
2091
2092
						            // convert to PDF
2093
						            HtmlToPdf.export(htmlFile.getAbsolutePath(), pdfFile.getAbsolutePath());
2094
2095
						            //add to the package
2096
						            bag.addFileToPayload(pdfFile);
2097
									pidMapping.append(metadataID.getValue() + " (pdf)" +  "\t" + "data/" + pdfFile.getName() + "\n");
2098
2099
						            // mark for clean up after we are done
2100
									htmlFile.delete();
2101
									cssFile.delete();
2102
									cssDir.delete();
2103
						            tempFiles.add(tmpDir);
2104
									tempFiles.add(pdfFile); // delete this first later on
2105
2106
								} catch (Exception e) {
2107
									logMetacat.warn("Could not transform metadata", e);
2108
								}
2109
							}
2110
2111
2112 8437 walker
							//If this is in eml format, extract the filename and GUID from each entity in its package
2113
							if (metadataSysMeta.getFormatId().getValue().startsWith("eml://")) {
2114
								//Get the package
2115
								DataPackageParserInterface parser = new Eml200DataPackageParser();
2116
								InputStream emlStream = this.get(session, metadataID);
2117
								parser.parse(emlStream);
2118
								DataPackage dataPackage = parser.getDataPackage();
2119
2120
								//Get all the entities in this package and loop through each to extract its ID and file name
2121
								Entity[] entities = dataPackage.getEntityList();
2122
								for(Entity entity: entities){
2123
									try{
2124
										//Get the file name from the metadata
2125
										String fileNameFromMetadata = entity.getName();
2126
2127
										//Get the ecogrid URL from the metadata
2128
										String ecogridIdentifier = entity.getEntityIdentifier();
2129
										//Parse the ecogrid URL to get the local id
2130
										String idFromMetadata = DocumentUtil.getAccessionNumberFromEcogridIdentifier(ecogridIdentifier);
2131
2132
										//Get the docid and rev pair
2133
										String docid = DocumentUtil.getDocIdFromString(idFromMetadata);
2134
										String rev = DocumentUtil.getRevisionStringFromString(idFromMetadata);
2135
2136
										//Get the GUID
2137
										String guid = IdentifierManager.getInstance().getGUID(docid, Integer.valueOf(rev));
2138
										Identifier dataIdentifier = new Identifier();
2139
										dataIdentifier.setValue(guid);
2140
2141
										//Add the GUID to our GUID & file name map
2142
										fileNames.put(dataIdentifier, fileNameFromMetadata);
2143
									}
2144
									catch(Exception e){
2145
										//Prevent just one entity error
2146
										e.printStackTrace();
2147
										logMetacat.debug(e.getMessage(), e);
2148
									}
2149
								}
2150
							}
2151
						}
2152
						catch(Exception e){
2153
							//Catch errors that would prevent package download
2154
							logMetacat.debug(e.toString());
2155
						}
2156
					}
2157 7855 leinfelder
					packagePids.addAll(entries.keySet());
2158
					for (List<Identifier> dataPids: entries.values()) {
2159
						packagePids.addAll(dataPids);
2160
					}
2161 7850 leinfelder
				}
2162 7855 leinfelder
			} else {
2163
				// just the lone pid in this package
2164
				packagePids.add(pid);
2165 7850 leinfelder
			}
2166 8436 walker
2167 8437 walker
			//Create a temp file, then delete it and make a directory with that name
2168
			File tempDir = File.createTempFile("temp", Long.toString(System.nanoTime()));
2169
			tempDir.delete();
2170
			tempDir = new File(tempDir.getPath() + "_dir");
2171
			tempDir.mkdir();
2172 8436 walker
			tempFiles.add(tempDir);
2173 8800 leinfelder
			File pidMappingFile = new File(tempDir, "pid-mapping.txt");
2174 8436 walker
2175 7855 leinfelder
			// loop through the package contents
2176
			for (Identifier entryPid: packagePids) {
2177 8436 walker
				//Get the system metadata for each item
2178 8437 walker
				SystemMetadata entrySysMeta = this.getSystemMetadata(session, entryPid);
2179 8436 walker
2180
				String objectFormatType = ObjectFormatCache.getInstance().getFormat(entrySysMeta.getFormatId()).getFormatType();
2181 8437 walker
				String fileName = null;
2182 8436 walker
2183 8437 walker
				//TODO: Be more specific of what characters to replace. Make sure periods arent replaced for the filename from metadata
2184
				//Our default file name is just the ID + format type (e.g. walker.1.1-DATA)
2185
				fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + objectFormatType;
2186
2187
				if(fileNames.containsKey(entryPid)){
2188
					//Let's use the file name and extension from the metadata is we have it
2189
					fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + fileNames.get(entryPid).replaceAll("[^a-zA-Z0-9\\-\\.]", "_");
2190
				}
2191
				else{
2192
					//If we couldn't find a given file name, use the system metadata extension
2193
					String extension = ObjectFormatInfo.instance().getExtension(entrySysMeta.getFormatId().getValue());
2194
					fileName += extension;
2195
				}
2196
2197 8436 walker
		        //Create a new file for this item and add to the list
2198 8437 walker
				File tempFile = new File(tempDir, fileName);
2199 7855 leinfelder
				tempFiles.add(tempFile);
2200 8436 walker
2201
				InputStream entryInputStream = this.get(session, entryPid);
2202 7855 leinfelder
				IOUtils.copy(entryInputStream, new FileOutputStream(tempFile));
2203
				bag.addFileToPayload(tempFile);
2204
				pidMapping.append(entryPid.getValue() + "\t" + "data/" + tempFile.getName() + "\n");
2205
			}
2206
2207
			//add the the pid to data file map
2208
			IOUtils.write(pidMapping.toString(), new FileOutputStream(pidMappingFile));
2209
			bag.addFileAsTag(pidMappingFile);
2210 8026 leinfelder
			tempFiles.add(pidMappingFile);
2211
2212 7855 leinfelder
			bag = bag.makeComplete();
2213 8026 leinfelder
2214 8436 walker
			///Now create the zip file
2215 8437 walker
			//Use the pid as the file name prefix, replacing all non-word characters
2216
			String zipName = pid.getValue().replaceAll("\\W", "_");
2217 8160 leinfelder
2218 8436 walker
			File bagFile = new File(tempDir, zipName+".zip");
2219
2220 7855 leinfelder
			bag.setFile(bagFile);
2221 7860 leinfelder
			ZipWriter zipWriter = new ZipWriter(bagFactory);
2222 7855 leinfelder
			bag.write(zipWriter, bagFile);
2223
			bagFile = bag.getFile();
2224 8026 leinfelder
			// use custom FIS that will delete the file when closed
2225
			bagInputStream = new DeleteOnCloseFileInputStream(bagFile);
2226
			// also mark for deletion on shutdown in case the stream is never closed
2227
			bagFile.deleteOnExit();
2228 8436 walker
			tempFiles.add(bagFile);
2229 7855 leinfelder
2230 8026 leinfelder
			// clean up other temp files
2231 8800 leinfelder
			for (int i=tempFiles.size()-1; i>=0; i--){
2232
				tempFiles.get(i).delete();
2233 7855 leinfelder
			}
2234 8436 walker
2235 7855 leinfelder
		} catch (IOException e) {
2236
			// report as service failure
2237
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2238
			sf.initCause(e);
2239
			throw sf;
2240
		} catch (OREException e) {
2241
			// report as service failure
2242
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2243
			sf.initCause(e);
2244
			throw sf;
2245
		} catch (URISyntaxException e) {
2246
			// report as service failure
2247
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2248
			sf.initCause(e);
2249
			throw sf;
2250
		} catch (OREParserException e) {
2251
			// report as service failure
2252
			ServiceFailure sf = new ServiceFailure("1030", e.getMessage());
2253
			sf.initCause(e);
2254
			throw sf;
2255 7850 leinfelder
		}
2256
2257
		return bagInputStream;
2258 8810 leinfelder
	}
2259 9190 tao
2260
	/**
2261
	 * Update the system metadata of the specified pid.
2262
	 */
2263 9177 tao
	@Override
2264
	public boolean updateSystemMetadata(Session session, Identifier pid,
2265
            SystemMetadata sysmeta) throws NotImplemented, NotAuthorized,
2266
            ServiceFailure, InvalidRequest, InvalidSystemMetadata, InvalidToken {
2267
	 if(sysmeta == null) {
2268
	     throw  new InvalidRequest("4863", "The system metadata object should NOT be null in the updateSystemMetadata request.");
2269
	 }
2270 9190 tao
	 if(pid == null || pid.getValue() == null) {
2271
         throw new InvalidRequest("4863", "Please specify the id in the updateSystemMetadata request ") ;
2272
     }
2273
2274
     if (session == null) {
2275
         //TODO: many of the thrown exceptions do not use the correct error codes
2276
         //check these against the docs and correct them
2277
         throw new NotAuthorized("4861", "No Session - could not authorize for updating system metadata." +
2278
                 "  If you are not logged in, please do so and retry the request.");
2279
     } else {
2280
         try {
2281
             //Following session can do the change:
2282 9263 tao
           //- Authoritative Member Node (we can use isNodeAdmin since we checked isAuthoritativeNode )
2283 9190 tao
             //- Owner of object (coved by the userHasPermission method)
2284
             //- user subjects with the change permission
2285
             //Note: Coordinating Node can not because MN is authoritative
2286 9263 tao
             /*if(!isAuthoritativeNode(pid)) {
2287
                throw  new InvalidRequest("4863", "Client can only call updateSystemMetadata request on the authoritative memember node.");
2288
             }
2289 9190 tao
             if(!isNodeAdmin(session) && !userHasPermission(session, pid, Permission.CHANGE_PERMISSION)) {
2290
                 throw new NotAuthorized("4861", "The client -"+ session.getSubject().getValue()+ "is not authorized for updating the system metadata of the object "+pid.getValue());
2291 9263 tao
             }*/
2292
             if(!allowUpdating(session, pid, Permission.CHANGE_PERMISSION)) {
2293
                 throw new NotAuthorized("4861", "The client -"+ session.getSubject().getValue()+ "is not authorized for updating the system metadata of the object "+pid.getValue());
2294 9190 tao
             }
2295
         } catch (NotFound e) {
2296
             throw new InvalidRequest("4863", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription());
2297
         }
2298
2299
     }
2300 9177 tao
      //update the system metadata locally
2301
      boolean success = super.updateSystemMetadata(session, pid, sysmeta);
2302
2303
      if(success) {
2304 9190 tao
          //TODO
2305 9177 tao
          //notify the cns the synchornize the new system metadata.
2306 9246 tao
          this.cn = D1Client.getCN();
2307
          try {
2308 9256 tao
              if(this.cn == null)  {
2309
                  logMetacat.warn("updateSystemMetadata - can't get the instance of the CN. So the system metadata in CN can't be updated.");
2310
              } else {
2311
                  this.cn.synchronize(null, pid);
2312
              }
2313 9257 tao
          } catch (BaseException e) {
2314
              e.printStackTrace();
2315
              logMetacat.error("It is a DataONEBaseException and its detail code is "+e.getDetail_code() +" and its code is "+e.getCode());
2316
              logMetacat.error("Can't update the systemmetadata of pid "+pid.getValue()+" in CNs since "+e.getMessage());
2317 9246 tao
          } catch (Exception e) {
2318 9256 tao
              e.printStackTrace();
2319 9246 tao
              logMetacat.error("Can't update the systemmetadata of pid "+pid.getValue()+" in CNs since "+e.getMessage());
2320
          }
2321 9248 leinfelder
2322
          // attempt to re-register the identifier (it checks if it is a doi)
2323
          try {
2324
        	  DOIService.getInstance().registerDOI(sysmeta);
2325
          } catch (Exception e) {
2326
  			logMetacat.warn("Could not [re]register DOI: " + e.getMessage(), e);
2327
          }
2328 9177 tao
      }
2329
      return success;
2330
    }
2331 9190 tao
2332
	/*
2333
     * Determine if the current node is the authoritative node for the given pid.
2334
     */
2335
    protected boolean isAuthoritativeNode(Identifier pid) {
2336
        boolean isAuthoritativeNode = false;
2337
        if(pid != null && pid.getValue() != null) {
2338
            SystemMetadata sys = HazelcastService.getInstance().getSystemMetadataMap().get(pid);
2339
            if(sys != null) {
2340
                NodeReference node = sys.getAuthoritativeMemberNode();
2341
                if(node != null) {
2342
                    String nodeValue = node.getValue();
2343
                    logMetacat.debug("The authoritative node for id "+pid.getValue()+" is "+nodeValue);
2344
                    //System.out.println("The authoritative node for id "+pid.getValue()+" is "+nodeValue);
2345
                    String currentNodeId = Settings.getConfiguration().getString("dataone.nodeId");
2346
                    logMetacat.debug("The node id in metacat.properties is "+currentNodeId);
2347
                    //System.out.println("The node id in metacat.properties is "+currentNodeId);
2348
                    if(currentNodeId != null && !currentNodeId.trim().equals("") && currentNodeId.equals(nodeValue)) {
2349
                        logMetacat.debug("They are matching");
2350
                        //System.out.println("They are matching");
2351
                        isAuthoritativeNode = true;
2352
                    }
2353
                }
2354
            }
2355
        }
2356
        return isAuthoritativeNode;
2357
    }
2358 6795 cjones
2359 9263 tao
    /*
2360
     * Rules are:
2361
     * 1. If the session has an cn object, it is allowed.
2362
     * 2. If it is not a cn object, the client should have approperate permission and it should also happen on the authorative node.
2363
     */
2364
    private boolean allowUpdating(Session session, Identifier pid, Permission permission) throws NotAuthorized, NotFound{
2365
        boolean allow = false;
2366
        if(isCNAdmin (session)) {
2367
            allow = true;
2368
        } else {
2369
            if(isAuthoritativeNode(pid)) {
2370
                if(userHasPermission(session, pid, permission)) {
2371
                    allow = true;
2372
                } else {
2373
                    allow = false;
2374
                }
2375
            } else {
2376
                throw new NotAuthorized("4861", "Client can only call the request on the authoritative memember node.");
2377
            }
2378
        }
2379
        return allow;
2380
2381
    }
2382
2383 6179 cjones
}