Project

General

Profile

« Previous | Next » 

Revision 6534

Added by ben leinfelder over 12 years ago

cleaned up to use for populating a DataONE MN using the D1 api. Retrieves packages from a Metacat and generates system metadata for them before calling MN.create() for both data and metadata.
NOTE: you need a client certificate that the target server accepts (either DataONE-generated for testing or a CILogon one for more official use). I was only able to get the former certificate type to work with our existing MN servers

View differences:

MetacatPopulator.java
31 31 
import java.math.BigInteger;
32 32 
import java.net.HttpURLConnection;
33 33 
import java.net.URL;
34 
import java.security.MessageDigest;
35 34 
import java.util.Calendar;
36 35 
import java.util.Date;
37 36 
import java.util.Vector;
......
57 56 
import org.dataone.service.types.v1.Session;
58 57 
import org.dataone.service.types.v1.Subject;
59 58 
import org.dataone.service.types.v1.SystemMetadata;
59 
import org.dataone.service.types.v1.util.ChecksumUtil;
60 
import org.dataone.service.util.Constants;
60 61 
import org.ecoinformatics.datamanager.DataManager;
61 62 
import org.ecoinformatics.datamanager.database.DatabaseConnectionPoolInterface;
62 63 
import org.ecoinformatics.datamanager.parser.DataPackage;
......
97 98 
        this.username = username;
98 99 
        this.password = password;
99 100 
        this.destinationUrl = destUrl;
100 
        // TODO: set up certificate for D1 interaction
101 
        CertificateManager.getInstance();
102 
        this.session = new Session();
103 
        Subject subject = new Subject();
104 
        subject.setValue(username);
101 
        // TODO: use specific certificate?
102 
        this.session = null; //new Session();
105 103 
    }
106 104
107 105 
    /**
......
110 108 
    public void populate()
111 109 
      throws Exception
112 110 
    {
113 
        printHeader("Source login");
114 
        String sourceSessionid = loginSource();
111 
        //String sourceSessionid = loginSource();
115 112
116 113 
        //do a query
117 114 
        String params = "returndoctype=eml://ecoinformatics.org/eml-2.1.0&" +
......
122 119 
        params += "anyfield=" + query;
123 120
124 121 
        printHeader("Searching source");
125 
        System.out.println("searching '" + sourceUrl + "' for '" + query + "' with sessionid '" + sourceSessionid + "'");
126 
        InputStream is = getResponse(sourceUrl, "/metacat",
127 
                params, "POST");
122 
        System.out.println("searching '" + sourceUrl + "' for '" + query + "'");
123 
        InputStream is = getResponse(sourceUrl, "/metacat", params, "POST");
128 124 
        String response = streamToString(is);
129 125 
        //System.out.println("response: " + response);
130 126 
        Vector<Document> docs = parseResponse(response);
131 127
132
133 128 
        printHeader("Parsing source results");
134 129 
        System.out.println("creating MN with url: " + destinationUrl + "/");
135 130 
        MNode mn = D1Client.getMN(destinationUrl + "/");
136 131
137 132 
        printHeader("Processing " + docs.size() + " results.");
138 
        printHeader("logging in to the destination " + destinationUrl);
139
140 
        System.out.println("session: " + session.getSubject());
141 133 
        for(int i=0; i<docs.size(); i++)
142 134 
        {
143 135 
            //for each document in the query
......
155 147 
        	DataManager dataManager = DataManager.getInstance(connectionPool, connectionPool.getDBAdapterName());
156 148 
        	DataPackage dataPackage = dataManager.parseMetadata(is);
157 149
158 
            if(dataPackage == null)
150 
            if (dataPackage == null)
159 151 
            {
160 152 
                continue;
161 153 
            }
......
207 199 
							System.out.println(e.getMessage());
208 200 
						}
209 201 
						dataDocSysMeta.setFmtid(format.getFmtid());
210 
	                    Checksum checksum = new Checksum();
211 202 
	                    dataDocIs = stringToStream(dataDocText);
212 
	                    String ca = "MD5";
213 
	                    checksum.setAlgorithm(ca);
214 
	                    checksum.setValue(checksum(dataDocIs));
203 
	                    Checksum checksum = ChecksumUtil.checksum(dataDocIs, "MD5");
215 204 
	                    dataDocSysMeta.setChecksum(checksum);
216 205 
	                    String sizeStr =
217 206 
	                    	Long.toString(dataDocText.getBytes(MetaCatServlet.DEFAULT_ENCODING).length);
......
220 209 
	                    //dataDocSysMeta.addDescribedBy(sysmeta.getIdentifier());
221 210 
	                    boolean error = false;
222 211
223 
	                    // create access policy
224 
	                    //"public", "read", "allow", "allowFirst"
225 
	                    AccessPolicy accessPolicy = new AccessPolicy();
226 
	                    AccessRule accessRule = new AccessRule();
227 
						accessRule.addPermission(Permission.READ);
228 
	                    Subject subject = new Subject();
229 
	                    subject.setValue("public");
230 
						accessRule.addSubject(subject );
231 
						accessPolicy.addAllow(accessRule );
232 212 
	                    //create the data doc on d1
233 213 
	                    try
234 214 
	                    {
235 215 
	                        mn.create(session, dataDocSysMeta.getIdentifier(), IOUtils.toInputStream(dataDocText), dataDocSysMeta);
236 
							mn.setAccessPolicy(session, dataDocSysMeta.getIdentifier(), accessPolicy);
237 216 
	                    }
238 217 
	                    catch(Exception e)
239 218 
	                    {
......
243 222 
	                    }
244 223 
	                    finally
245 224 
	                    {
246 
	                        if(error)
225 
	                        if (error)
247 226 
	                        {
248 227 
	                            printHeader("Insertion of document " + dataDocSysMeta.getIdentifier().getValue() +
249 228 
	                                    "FAILED.");
......
284 263 
            }
285 264 
        }
286 265
287 
        logout();
266 
        //logout();
288 267 
    }
289 268
290 269 

  		




  ......




  319
  298
  
    
        


  		




  320
  299
  
    
        //create the checksum


  		




  321
  300
  
    
        ByteArrayInputStream bais = new ByteArrayInputStream(doc.doctext.getBytes(MetaCatServlet.DEFAULT_ENCODING));


  		




  322
  
  
    
        String checksumS = checksum(bais);


  		




  323
  
  
    
        String ca = "MD5";


  		




  324
  
  
    
        Checksum checksum = new Checksum();


  		




  325
  
  
    
        checksum.setValue(checksumS);


  		




  326
  
  
    
        checksum.setAlgorithm(ca);


  		




  
  301
  
    
        Checksum checksum = ChecksumUtil.checksum(bais, "MD5");


  		




  327
  302
  
    
        sm.setChecksum(checksum);


  		




  328
  303
  
    
        


  		




  329
  304
  
    
        //set the size


  		




  ......




  355
  330
  
    
        sm.setOriginMemberNode(nr);


  		




  356
  331
  
    
        sm.setAuthoritativeMemberNode(nr);


  		




  357
  332
  
    
        


  		




  
  333
  
    
        // create access policy


  		




  
  334
  
    
        AccessPolicy accessPolicy = new AccessPolicy();


  		




  
  335
  
    
        AccessRule accessRule = new AccessRule();


  		




  
  336
  
    
		accessRule.addPermission(Permission.READ);


  		




  
  337
  
    
        Subject subject = new Subject();


  		




  
  338
  
    
        subject.setValue(Constants.SUBJECT_PUBLIC);


  		




  
  339
  
    
		accessRule.addSubject(subject);


  		




  
  340
  
    
		accessPolicy.addAllow(accessRule);


  		




  
  341
  
    
		sm.setAccessPolicy(accessPolicy);


  		




  
  342
  
    
        


  		




  358
  343
  
    
        return sm;


  		




  359
  344
  
    
    }


  		




  360
  345
  
    
    


  		




  ......




  363
  348
  
    
        System.out.println("****** " + s + " *******");


  		




  364
  349
  
    
    }


  		




  365
  350
  
    
    


  		




  366
  
  
    
    /**


  		




  367
  
  
    
     * produce an md5 checksum for item


  		




  368
  
  
    
     */


  		




  369
  
  
    
    private String checksum(InputStream is)


  		




  370
  
  
    
      throws Exception


  		




  371
  
  
    
    {        


  		




  372
  
  
    
        byte[] buffer = new byte[1024];


  		




  373
  
  
    
        MessageDigest complete = MessageDigest.getInstance("MD5");


  		




  374
  
  
    
        int numRead;


  		




  375
  
  
    
        


  		




  376
  
  
    
        do 


  		




  377
  
  
    
        {


  		




  378
  
  
    
          numRead = is.read(buffer);


  		




  379
  
  
    
          if (numRead > 0) 


  		




  380
  
  
    
          {


  		




  381
  
  
    
            complete.update(buffer, 0, numRead);


  		




  382
  
  
    
          }


  		




  383
  
  
    
        } while (numRead != -1);


  		




  384
  
  
    
        


  		




  385
  
  
    
        


  		




  386
  
  
    
        return getHex(complete.digest());


  		




  387
  
  
    
    }


  		




  388
  351
  
    
    


  		




  389
  
  
    
    /**


  		




  390
  
  
    
     * convert a byte array to a hex string


  		




  391
  
  
    
     */


  		




  392
  
  
    
    private static String getHex( byte [] raw ) 


  		




  393
  
  
    
    {


  		




  394
  
  
    
        final String HEXES = "0123456789ABCDEF";


  		




  395
  
  
    
        if ( raw == null ) {


  		




  396
  
  
    
          return null;


  		




  397
  
  
    
        }


  		




  398
  
  
    
        final StringBuilder hex = new StringBuilder( 2 * raw.length );


  		




  399
  
  
    
        for ( final byte b : raw ) {


  		




  400
  
  
    
          hex.append(HEXES.charAt((b & 0xF0) >> 4))


  		




  401
  
  
    
             .append(HEXES.charAt((b & 0x0F)));


  		




  402
  
  
    
        }


  		




  403
  
  
    
        return hex.toString();


  		




  404
  
  
    
    }


  		




  405
  352
  
    
    


  		




  406
  353
  
    
    /**


  		




  407
  354
  
    
     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into


  		




  ......




  576
  523
  
    
        throws Exception


  		




  577
  524
  
    
    {


  		




  578
  525
  
    
        getResponse(sourceUrl, "/metacat", "action=logout&username=" + username, "POST");


  		




  579
  
  
    
        getResponse(destinationUrl, "/metacat", "action=logout&username=" + username, "POST");


  		




  580
  526
  
    
    }


  		




  581
  527
  
    
    


  		




  582
  528
  
    
    /**


  		












Also available in:  Unified diff