Project

General

Profile

« Previous | Next » 

Revision 6107

persist system metadata replication policy and status using db tables

View differences:

src/xmltables-oracle.sql
414 414
   submitter VARCHAR2(256), -- the user who originally submitted the doc
415 415
   object_format VARCHAR2(256), --the format of the object
416 416
   size VARCHAR2(256), --the size of the object
417
   replication_allowed boolean,	 -- replication allowed
418
   number_replicas NUMBER(8), 	-- the number of replicas allowed
417 419
   CONSTRAINT systemMetadata_pk 
418 420
		PRIMARY KEY (guid)
419 421
)
......
429 431
		FOREIGN KEY (guid) REFERENCES systemMetadata
430 432
);
431 433

  
434
CREATE TABLE systemMetadataReplicationPolicy (
435
	guid VARCHAR2(2000),	-- the globally unique string identifier of the object that the system metadata describes
436
	member_node VARCHAR(250),	 -- replication member node
437
	policy VARCHAR2(2000),	 -- the policy (preferred, blocked, etc...TBD)
438
	CONSTRAINT systemMetadataReplicationPolicy_fk 
439
		FOREIGN KEY (guid) REFERENCES systemMetadata
440
);
441

  
442
CREATE TABLE systemMetadataReplicationStatus (
443
	guid VARCHAR2(2000),	-- the globally unique string identifier of the object that the system metadata describes
444
	member_node VARCHAR(250),	 -- replication member node
445
	status VARCHAR(250),	 -- replication status
446
	date_verified DATE, 	-- the date replication was verified   
447
	CONSTRAINT systemMetadataReplicationStatus_fk 
448
		FOREIGN KEY (guid) REFERENCES systemMetadata
449
);
450

  
432 451
/*
433 452
 * Table used to store all document identifiers in metacat.  Each identifier
434 453
 * has a globally unique, unconstrained string, which we will refer to as a
src/xmltables-postgres.sql
289 289
 * data first or the systemMetadata first.
290 290
 */
291 291
CREATE TABLE systemMetadata (
292
   guid   text,          -- the globally unique string identifier of the object that the system metadata describes
293
   date_uploaded TIMESTAMP, -- the date/time the document was first submitted
294
   rights_holder VARCHAR(250), --the user who has rights to the document, usually the first persons to upload it
295
   checksum VARCHAR(512), --the checksum of the doc using the given algorithm (see below)
296
   checksum_algorithm VARCHAR(250), --the algorithm used to calculate the checksum
297
   origin_member_node VARCHAR(250), --the member node where the document was first uploaded
298
   authoritive_member_node VARCHAR(250), --the member node that currently controls the document
299
   date_modified TIMESTAMP, -- the last date/time that the file was changed
300
   submitter VARCHAR(256), -- the user who originally submitted the doc
301
   object_format VARCHAR(256), --the format of the object
302
   size VARCHAR(256), --the size of the object
303
   CONSTRAINT systemMetadata_pk PRIMARY KEY (guid)
292
	guid   text,          -- the globally unique string identifier of the object that the system metadata describes
293
	date_uploaded TIMESTAMP, -- the date/time the document was first submitted
294
	rights_holder VARCHAR(250), --the user who has rights to the document, usually the first persons to upload it
295
	checksum VARCHAR(512), --the checksum of the doc using the given algorithm (see below)
296
	checksum_algorithm VARCHAR(250), --the algorithm used to calculate the checksum
297
	origin_member_node VARCHAR(250), --the member node where the document was first uploaded
298
	authoritive_member_node VARCHAR(250), --the member node that currently controls the document
299
	date_modified TIMESTAMP, -- the last date/time that the file was changed
300
	submitter VARCHAR(256), -- the user who originally submitted the doc
301
	object_format VARCHAR(256), --the format of the object
302
	size VARCHAR(256), --the size of the object
303
	replication_allowed boolean,	 -- replication allowed
304
	number_replicas INT8, 	-- the number of replicas allowed
305
	CONSTRAINT systemMetadata_pk PRIMARY KEY (guid)
304 306
);
305 307
/*
306 308
 * For devs to remove docid, rev
307 309
 * ALTER TABLE systemMetadata DROP COLUMN docid;
308 310
 * ALTER TABLE systemMetadata DROP COLUMN rev;
311
 * ALTER TABLE systemMetadata ADD COLUMN replication_allowed boolean;
312
 * ALTER TABLE systemMetadata ADD COLUMN number_replicas INT8;
309 313
 */
310 314

  
311 315
/*
......
315 319
   guid   		text,          -- the globally unique string identifier of the object that the system metadata describes
316 320
   relationship	VARCHAR(250),	 -- the provenance relationship defined between objects
317 321
   target_guid	text,          -- the globally unique string identifier of the other object
318
   
319 322
   CONSTRAINT systemMetadataProvenance_fk 
320 323
		FOREIGN KEY (guid) REFERENCES systemMetadata
321 324
);
322 325

  
326
CREATE TABLE systemMetadataReplicationPolicy (
327
	guid text,	-- the globally unique string identifier of the object that the system metadata describes
328
	member_node VARCHAR(250),	 -- replication member node
329
	policy text,	 -- the policy (preferred, blocked, etc...TBD)
330
	CONSTRAINT systemMetadataReplicationPolicy_fk 
331
		FOREIGN KEY (guid) REFERENCES systemMetadata
332
);
333

  
334
CREATE TABLE systemMetadataReplicationStatus (
335
	guid text,	-- the globally unique string identifier of the object that the system metadata describes
336
	member_node VARCHAR(250),	 -- replication member node
337
	status VARCHAR(250),	 -- replication status
338
	date_verified TIMESTAMP, 	-- the date replication was verified   
339
	CONSTRAINT systemMetadataReplicationStatus_fk 
340
		FOREIGN KEY (guid) REFERENCES systemMetadata
341
);
342

  
323 343
/*
324 344
 * accesssubtree -- table to store access subtree info
325 345
 */
src/edu/ucsb/nceas/metacat/IdentifierManager.java
24 24

  
25 25
package edu.ucsb.nceas.metacat;
26 26

  
27
import java.util.*;
28

  
29 27
import java.sql.PreparedStatement;
30 28
import java.sql.ResultSet;
31 29
import java.sql.SQLException;
32 30
import java.sql.Timestamp;
31
import java.util.ArrayList;
32
import java.util.Date;
33
import java.util.Hashtable;
34
import java.util.List;
35
import java.util.Vector;
33 36

  
34 37
import org.apache.log4j.Logger;
35 38
import org.dataone.service.types.Checksum;
36 39
import org.dataone.service.types.ChecksumAlgorithm;
40
import org.dataone.service.types.Identifier;
37 41
import org.dataone.service.types.NodeReference;
38 42
import org.dataone.service.types.ObjectFormat;
39 43
import org.dataone.service.types.ObjectInfo;
40 44
import org.dataone.service.types.ObjectList;
45
import org.dataone.service.types.Replica;
46
import org.dataone.service.types.ReplicationPolicy;
47
import org.dataone.service.types.ReplicationStatus;
41 48
import org.dataone.service.types.Subject;
42 49
import org.dataone.service.types.SystemMetadata;
43
import org.dataone.service.types.Identifier;
44 50

  
45 51
import edu.ucsb.nceas.metacat.database.DBConnection;
46 52
import edu.ucsb.nceas.metacat.database.DBConnectionPool;
......
294 300
        sysMeta.setDescribedByList(getSystemMetadataProvenance(sysMeta.getIdentifier().getValue(), "describedBy"));
295 301
        sysMeta.setDerivedFromList(getSystemMetadataProvenance(sysMeta.getIdentifier().getValue(), "derivedFrom"));
296 302
        
303
        // look up replication policy
304
        ReplicationPolicy replicationPolicy = new ReplicationPolicy();
305
        replicationPolicy.setBlockedMemberNodeList(getReplicationPolicy(guid, "blocked"));
306
        replicationPolicy.setPreferredMemberNodeList(getReplicationPolicy(guid, "preferred"));
307
		sysMeta.setReplicationPolicy(replicationPolicy);
308
		
309
		// look up replication status
310
		sysMeta.setReplicaList(getReplicationStatus(guid));
311
        
297 312
        return sysMeta;
298 313
    }
299 314
    
......
303 318
    	List<Identifier> identifiers = new ArrayList<Identifier>();
304 319
    	String sql = "select guid, relationship, target_guid " +
305 320
    		"from systemMetadataProvenance where guid = ? and relationship = ?";
306
    DBConnection dbConn = null;
307
    int serialNumber = -1;
308
    try 
309
    {
310
        // Get a database connection from the pool
311
        dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getSystemMetadataProvenance");
312
        serialNumber = dbConn.getCheckOutSerialNumber();
313

  
314
        // Execute the statement
315
        PreparedStatement stmt = dbConn.prepareStatement(sql);
316
        stmt.setString(1, guid);
317
        stmt.setString(2, relationship);
318
        ResultSet rs = stmt.executeQuery();
319
        while (rs.next()) 
320
        {
321
            String targetGuid = rs.getString(3);
322
            Identifier id = new Identifier();
323
            id.setValue(targetGuid);
324
            identifiers.add(id);
325
        
326
        } 
327
        stmt.close();
328
        
329
    } 
330
    catch (SQLException e) 
331
    {
332
        logMetacat.error("Error while getting system metadata provenance for guid " + guid, e);
333
    } 
334
    finally 
335
    {
336
        // Return database connection to the pool
337
        DBConnectionPool.returnDBConnection(dbConn, serialNumber);
338
    }
321
	    DBConnection dbConn = null;
322
	    int serialNumber = -1;
323
	    try {
324
	        // Get a database connection from the pool
325
	        dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getSystemMetadataProvenance");
326
	        serialNumber = dbConn.getCheckOutSerialNumber();
327
	
328
	        // Execute the statement
329
	        PreparedStatement stmt = dbConn.prepareStatement(sql);
330
	        stmt.setString(1, guid);
331
	        stmt.setString(2, relationship);
332
	        ResultSet rs = stmt.executeQuery();
333
	        while (rs.next()) 
334
	        {
335
	            String targetGuid = rs.getString(3);
336
	            Identifier id = new Identifier();
337
	            id.setValue(targetGuid);
338
	            identifiers.add(id);
339
	        
340
	        } 
341
	        stmt.close();
342
	        
343
	    } 
344
	    catch (SQLException e) {
345
	        logMetacat.error("Error while getting system metadata provenance for guid " + guid, e);
346
	    } 
347
	    finally {
348
	        // Return database connection to the pool
349
	        DBConnectionPool.returnDBConnection(dbConn, serialNumber);
350
	    }
351
	    
352
	    return identifiers;
353
	}
339 354
    
340
    return identifiers;
341
}
355
    private List<NodeReference> getReplicationPolicy(String guid, String policy)
356
		throws McdbDocNotFoundException {
357
		
358
		List<NodeReference> nodes = new ArrayList<NodeReference>();
359
		String sql = "select guid, policy, member_node " +
360
			"from systemMetadataReplicationPolicy where guid = ? and policy = ?";
361
	    DBConnection dbConn = null;
362
	    int serialNumber = -1;
363
	    try {
364
	        // Get a database connection from the pool
365
	        dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getReplicationPolicy");
366
	        serialNumber = dbConn.getCheckOutSerialNumber();
367
	
368
	        // Execute the statement
369
	        PreparedStatement stmt = dbConn.prepareStatement(sql);
370
	        stmt.setString(1, guid);
371
	        stmt.setString(2, policy);
372
	        ResultSet rs = stmt.executeQuery();
373
	        while (rs.next()) 
374
	        {
375
	            String memberNode = rs.getString(3);
376
	            NodeReference node = new NodeReference();
377
	            node.setValue(memberNode);
378
	            nodes.add(node);
379
	        
380
	        } 
381
	        stmt.close();
382
	        
383
	    } catch (SQLException e) {
384
	        logMetacat.error("Error while getting system metadata replication policy for guid " + guid, e);
385
	    } 
386
	    finally {
387
	        // Return database connection to the pool
388
	        DBConnectionPool.returnDBConnection(dbConn, serialNumber);
389
	    }
390
	    
391
	    return nodes;
392
	}
342 393
    
394
    private List<Replica> getReplicationStatus(String guid) throws McdbDocNotFoundException {
395
		
396
		List<Replica> replicas = new ArrayList<Replica>();
397
		String sql = "select guid, member_node, status, date_verified " +
398
			"from systemMetadataReplicationStatus where guid = ?";
399
	    DBConnection dbConn = null;
400
	    int serialNumber = -1;
401
	    try {
402
	        // Get a database connection from the pool
403
	        dbConn = DBConnectionPool.getDBConnection("IdentifierManager.getReplicas");
404
	        serialNumber = dbConn.getCheckOutSerialNumber();
405
	
406
	        // Execute the statement
407
	        PreparedStatement stmt = dbConn.prepareStatement(sql);
408
	        stmt.setString(1, guid);
409
	        ResultSet rs = stmt.executeQuery();
410
	        while (rs.next()) 
411
	        {
412
	            String memberNode = rs.getString(2);
413
	            String status = rs.getString(3);
414
	            java.sql.Date verified = rs.getDate(4);
415
	            
416
	            Replica replica = new Replica();	            
417
	            NodeReference node = new NodeReference();
418
	            node.setValue(memberNode);
419
	            replica.setReplicaMemberNode(node);
420
	            replica.setReplicationStatus(ReplicationStatus.convert(status));
421
	            replica.setReplicaVerified(new Date(verified.getTime()));
422
	            replicas.add(replica);
423
	        } 
424
	        stmt.close();
425
	        
426
	    } catch (SQLException e) {
427
	        logMetacat.error("Error while getting system metadata replication policy for guid " + guid, e);
428
	    } 
429
	    finally {
430
	        // Return database connection to the pool
431
	        DBConnectionPool.returnDBConnection(dbConn, serialNumber);
432
	    }
433
	    
434
	    return replicas;
435
	}
436
    
343 437
    /**
344 438
     * return information on the document with localId.  These are the fields
345 439
     * from the xml_documents table.  They can be used to contstruct metadata 
......
780 874
    private void updateSystemMetadataFields(long dateUploaded, String rightsHolder,
781 875
            String checksum, String checksumAlgorithm, String originMemberNode,
782 876
            String authoritativeMemberNode, long modifiedDate, String submitter, 
783
            String guid, String objectFormat, long size)
877
            String guid, String objectFormat, long size, boolean replicationAllowed,
878
            int numberReplicas)
784 879
    {
785 880
        DBConnection dbConn = null;
786 881
        int serialNumber = -1;
......
795 890
            String query = "update " + TYPE_SYSTEM_METADATA + 
796 891
                " set (date_uploaded, rights_holder, checksum, checksum_algorithm, " +
797 892
                "origin_member_node, authoritive_member_node, date_modified, " +
798
                "submitter, object_format, size) " +
799
                "= (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) where guid = ?";
893
                "submitter, object_format, size, replication_allowed, number_replicas) " +
894
                "= (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) where guid = ?";
800 895
            PreparedStatement stmt = dbConn.prepareStatement(query);
801 896
            
802 897
            //data values
......
810 905
            stmt.setString(8, submitter);
811 906
            stmt.setString(9, objectFormat);
812 907
            stmt.setString(10, new Long(size).toString());
908
            stmt.setBoolean(11, replicationAllowed);
909
            stmt.setInt(12, numberReplicas);
813 910
            //where clause
814
            stmt.setString(11, guid);
911
            stmt.setString(13, guid);
815 912
            logMetacat.debug("stmt: " + stmt.toString());
816 913
            //execute
817 914
            int rows = stmt.executeUpdate();
......
839 936
        try {
840 937
            // Get a database connection from the pool
841 938
            dbConn = 
842
                DBConnectionPool.getDBConnection("IdentifierManager.createMapping");
939
                DBConnectionPool.getDBConnection("IdentifierManager.insertSystemMetadataProvenance");
843 940
            serialNumber = dbConn.getCheckOutSerialNumber();
844 941

  
845 942
            // remove existing values first
......
876 973
        }
877 974
    }
878 975
    
976
    private void insertReplicationPolicy(String guid, String policy, List<String> memberNodes)
977
    {
978
        DBConnection dbConn = null;
979
        int serialNumber = -1;
980
        
981
        try {
982
            // Get a database connection from the pool
983
            dbConn = 
984
                DBConnectionPool.getDBConnection("IdentifierManager.insertReplicationPolicy");
985
            serialNumber = dbConn.getCheckOutSerialNumber();
986

  
987
            // remove existing values first
988
            String delete = "delete from systemMetadataReplicationPolicy " + 
989
            "where guid = ? and policy = ?";
990
	        PreparedStatement stmt = dbConn.prepareStatement(delete);
991
	        //data values
992
	        stmt.setString(1, guid);
993
	        stmt.setString(2, policy);
994
	        //execute
995
	        int deletedCount = stmt.executeUpdate();
996
	        stmt.close();
997
            
998
            for (String memberNode: memberNodes) {
999
	            // Execute the insert statement
1000
	            String insert = "insert into systemMetadataReplicationPolicy " + 
1001
	                "(guid, policy, member_node) " +
1002
	                "values (?, ?, ?)";
1003
	            PreparedStatement insertStatement = dbConn.prepareStatement(insert);
1004
	            
1005
	            //data values
1006
	            insertStatement.setString(1, guid);
1007
	            insertStatement.setString(2, policy);
1008
	            insertStatement.setString(3, memberNode);
1009
	            //execute
1010
	            int rows = insertStatement.executeUpdate();
1011
	            insertStatement.close();
1012
            }
1013
        } catch (SQLException e) {
1014
            logMetacat.error("SQL error while adding systemMetadataReplicationPolicy for: " + guid, e); 
1015
        } finally {
1016
            // Return database connection to the pool
1017
            DBConnectionPool.returnDBConnection(dbConn, serialNumber);
1018
        }
1019
    }
1020
    
1021
    private void insertReplicationStatus(String guid, List<Replica> replicas) {
1022
        DBConnection dbConn = null;
1023
        int serialNumber = -1;
1024
        
1025
        try {
1026
            // Get a database connection from the pool
1027
            dbConn = DBConnectionPool.getDBConnection("IdentifierManager.insertReplicas");
1028
            serialNumber = dbConn.getCheckOutSerialNumber();
1029

  
1030
            // remove existing values first
1031
            String delete = "delete from systemMetadataReplicationStatus " + 
1032
            "where guid = ?";
1033
	        PreparedStatement stmt = dbConn.prepareStatement(delete);
1034
	        //data values
1035
	        stmt.setString(1, guid);
1036
	        //execute
1037
	        int deletedCount = stmt.executeUpdate();
1038
	        stmt.close();
1039
            
1040
            for (Replica replica: replicas) {
1041
	            // Execute the insert statement
1042
	            String insert = "insert into systemMetadataReplicationStatus " + 
1043
	                "(guid, member_node, status, date_verified) " +
1044
	                "values (?, ?, ?, ?)";
1045
	            PreparedStatement insertStatement = dbConn.prepareStatement(insert);
1046
	            
1047
	            //data values
1048
	            String memberNode = replica.getReplicaMemberNode().getValue();
1049
	            String status = replica.getReplicationStatus().toString();
1050
	            java.sql.Date sqlDate = new java.sql.Date(replica.getReplicaVerified().getTime());
1051
	            insertStatement.setString(1, guid);
1052
	            insertStatement.setString(2, memberNode);
1053
	            insertStatement.setString(3, status);
1054
	            insertStatement.setDate(4, sqlDate);
1055

  
1056
	            //execute
1057
	            int rows = insertStatement.executeUpdate();
1058
	            insertStatement.close();
1059
            }
1060
        } catch (SQLException e) {
1061
            logMetacat.error("SQL error while adding systemMetadataReplicationStatus for: " + guid, e); 
1062
        } finally {
1063
            // Return database connection to the pool
1064
            DBConnectionPool.returnDBConnection(dbConn, serialNumber);
1065
        }
1066
    }
1067
    
879 1068
    /**
880 1069
     * Insert the system metadata fields into the db
881 1070
     * @param sm
882 1071
     */
883
    public void updateSystemMetadata(SystemMetadata sm)
884
    {
885
        updateSystemMetadataFields(
1072
    public void updateSystemMetadata(SystemMetadata sm) {
1073
    	
1074
        Boolean replicationAllowed = false;
1075
		Integer numberReplicas = -1;
1076
    	ReplicationPolicy replicationPolicy = sm.getReplicationPolicy();
1077
    	if (replicationPolicy != null) {
1078
    		replicationAllowed = replicationPolicy.getReplicationAllowed();
1079
    		numberReplicas = replicationPolicy.getNumberReplicas();
1080
    		replicationAllowed = replicationAllowed == null ? false: replicationAllowed;
1081
    		numberReplicas = numberReplicas == null ? -1: numberReplicas;
1082
    	}
1083

  
1084
    	// the main systemMetadata fields
1085
		updateSystemMetadataFields(
886 1086
                sm.getDateUploaded().getTime(),
887 1087
                sm.getRightsHolder().getValue(), 
888 1088
                sm.getChecksum().getValue(), 
......
893 1093
                sm.getSubmitter().getValue(), 
894 1094
                sm.getIdentifier().getValue(),
895 1095
                sm.getObjectFormat().toString(),
896
                sm.getSize());
1096
                sm.getSize(),
1097
                replicationAllowed, 
1098
                numberReplicas);
897 1099
        
898 1100
        // add provenance information
899 1101
        String guid = sm.getIdentifier().getValue();
......
934 1136
        	targetGuids.add(id.getValue());
935 1137
        }
936 1138
        this.insertSystemMetadataProvenance(guid, relationship, targetGuids);
1139
                
1140
        // save replication policies
1141
        if (replicationPolicy != null) {
1142
		    List<String> nodes = null;
1143
		    String policy = null;
1144
		    
1145
		    nodes = new ArrayList<String>();
1146
		    policy = "blocked";
1147
		    for (NodeReference node: replicationPolicy.getBlockedMemberNodeList()) {
1148
		    	nodes.add(node.getValue());
1149
		    }
1150
		    this.insertReplicationPolicy(guid, policy, nodes);
1151
		    
1152
		    nodes = new ArrayList<String>();
1153
		    policy = "preferred";
1154
		    for (NodeReference node: replicationPolicy.getPreferredMemberNodeList()) {
1155
		    	nodes.add(node.getValue());
1156
		    }
1157
	        this.insertReplicationPolicy(guid, policy, nodes);
1158
        }
937 1159
        
1160
        // save replica information
1161
        this.insertReplicationStatus(guid, sm.getReplicaList());
1162
        
938 1163
    }
939 1164
    
940 1165
    /**

Also available in: Unified diff