Project

General

Profile

« Previous | Next » 

Revision 6534

cleaned up to use for populating a DataONE MN using the D1 api. Retrieves packages from a Metacat and generates system metadata for them before calling MN.create() for both data and metadata.
NOTE: you need a client certificate that the target server accepts (either DataONE-generated for testing or a CILogon one for more official use). I was only able to get the former certificate type to work with our existing MN servers

View differences:

src/edu/ucsb/nceas/metacat/util/MetacatPopulator.java
31 31
import java.math.BigInteger;
32 32
import java.net.HttpURLConnection;
33 33
import java.net.URL;
34
import java.security.MessageDigest;
35 34
import java.util.Calendar;
36 35
import java.util.Date;
37 36
import java.util.Vector;
......
57 56
import org.dataone.service.types.v1.Session;
58 57
import org.dataone.service.types.v1.Subject;
59 58
import org.dataone.service.types.v1.SystemMetadata;
59
import org.dataone.service.types.v1.util.ChecksumUtil;
60
import org.dataone.service.util.Constants;
60 61
import org.ecoinformatics.datamanager.DataManager;
61 62
import org.ecoinformatics.datamanager.database.DatabaseConnectionPoolInterface;
62 63
import org.ecoinformatics.datamanager.parser.DataPackage;
......
97 98
        this.username = username;
98 99
        this.password = password;
99 100
        this.destinationUrl = destUrl;
100
        // TODO: set up certificate for D1 interaction
101
        CertificateManager.getInstance();
102
        this.session = new Session();
103
        Subject subject = new Subject();
104
        subject.setValue(username);
101
        // TODO: use specific certificate?
102
        this.session = null; //new Session();
105 103
    }
106 104
    
107 105
    /**
......
110 108
    public void populate()
111 109
      throws Exception
112 110
    {
113
        printHeader("Source login");
114
        String sourceSessionid = loginSource();
111
        //String sourceSessionid = loginSource();
115 112
        
116 113
        //do a query
117 114
        String params = "returndoctype=eml://ecoinformatics.org/eml-2.1.0&" +
......
122 119
        params += "anyfield=" + query;
123 120
        
124 121
        printHeader("Searching source");
125
        System.out.println("searching '" + sourceUrl + "' for '" + query + "' with sessionid '" + sourceSessionid + "'");
126
        InputStream is = getResponse(sourceUrl, "/metacat",
127
                params, "POST");
122
        System.out.println("searching '" + sourceUrl + "' for '" + query + "'");
123
        InputStream is = getResponse(sourceUrl, "/metacat", params, "POST");
128 124
        String response = streamToString(is);
129 125
        //System.out.println("response: " + response);
130 126
        Vector<Document> docs = parseResponse(response);
131 127
        
132
        
133 128
        printHeader("Parsing source results");
134 129
        System.out.println("creating MN with url: " + destinationUrl + "/");
135 130
        MNode mn = D1Client.getMN(destinationUrl + "/");
136 131
        
137 132
        printHeader("Processing " + docs.size() + " results.");
138
        printHeader("logging in to the destination " + destinationUrl);
139
        
140
        System.out.println("session: " + session.getSubject());
141 133
        for(int i=0; i<docs.size(); i++)
142 134
        {
143 135
            //for each document in the query
......
155 147
        	DataManager dataManager = DataManager.getInstance(connectionPool, connectionPool.getDBAdapterName());
156 148
        	DataPackage dataPackage = dataManager.parseMetadata(is);
157 149
        	
158
            if(dataPackage == null)
150
            if (dataPackage == null)
159 151
            {
160 152
                continue;
161 153
            }
......
207 199
							System.out.println(e.getMessage());
208 200
						}
209 201
						dataDocSysMeta.setFmtid(format.getFmtid());
210
	                    Checksum checksum = new Checksum();
211 202
	                    dataDocIs = stringToStream(dataDocText);
212
	                    String ca = "MD5";
213
	                    checksum.setAlgorithm(ca);
214
	                    checksum.setValue(checksum(dataDocIs));
203
	                    Checksum checksum = ChecksumUtil.checksum(dataDocIs, "MD5");
215 204
	                    dataDocSysMeta.setChecksum(checksum);
216 205
	                    String sizeStr = 
217 206
	                    	Long.toString(dataDocText.getBytes(MetaCatServlet.DEFAULT_ENCODING).length);
......
220 209
	                    //dataDocSysMeta.addDescribedBy(sysmeta.getIdentifier());
221 210
	                    boolean error = false;
222 211
	                    
223
	                    // create access policy
224
	                    //"public", "read", "allow", "allowFirst"
225
	                    AccessPolicy accessPolicy = new AccessPolicy();
226
	                    AccessRule accessRule = new AccessRule();
227
						accessRule.addPermission(Permission.READ);
228
	                    Subject subject = new Subject();
229
	                    subject.setValue("public");
230
						accessRule.addSubject(subject );
231
						accessPolicy.addAllow(accessRule );
232 212
	                    //create the data doc on d1
233 213
	                    try
234 214
	                    {
235 215
	                        mn.create(session, dataDocSysMeta.getIdentifier(), IOUtils.toInputStream(dataDocText), dataDocSysMeta);
236
							mn.setAccessPolicy(session, dataDocSysMeta.getIdentifier(), accessPolicy);
237 216
	                    }
238 217
	                    catch(Exception e)
239 218
	                    {
......
243 222
	                    }
244 223
	                    finally
245 224
	                    {
246
	                        if(error)
225
	                        if (error)
247 226
	                        {
248 227
	                            printHeader("Insertion of document " + dataDocSysMeta.getIdentifier().getValue() + 
249 228
	                                    "FAILED.");
......
284 263
            }
285 264
        }
286 265
        
287
        logout();
266
        //logout();
288 267
    }
289 268
    
290 269

  
......
319 298
        
320 299
        //create the checksum
321 300
        ByteArrayInputStream bais = new ByteArrayInputStream(doc.doctext.getBytes(MetaCatServlet.DEFAULT_ENCODING));
322
        String checksumS = checksum(bais);
323
        String ca = "MD5";
324
        Checksum checksum = new Checksum();
325
        checksum.setValue(checksumS);
326
        checksum.setAlgorithm(ca);
301
        Checksum checksum = ChecksumUtil.checksum(bais, "MD5");
327 302
        sm.setChecksum(checksum);
328 303
        
329 304
        //set the size
......
355 330
        sm.setOriginMemberNode(nr);
356 331
        sm.setAuthoritativeMemberNode(nr);
357 332
        
333
        // create access policy
334
        AccessPolicy accessPolicy = new AccessPolicy();
335
        AccessRule accessRule = new AccessRule();
336
		accessRule.addPermission(Permission.READ);
337
        Subject subject = new Subject();
338
        subject.setValue(Constants.SUBJECT_PUBLIC);
339
		accessRule.addSubject(subject);
340
		accessPolicy.addAllow(accessRule);
341
		sm.setAccessPolicy(accessPolicy);
342
        
358 343
        return sm;
359 344
    }
360 345
    
......
363 348
        System.out.println("****** " + s + " *******");
364 349
    }
365 350
    
366
    /**
367
     * produce an md5 checksum for item
368
     */
369
    private String checksum(InputStream is)
370
      throws Exception
371
    {        
372
        byte[] buffer = new byte[1024];
373
        MessageDigest complete = MessageDigest.getInstance("MD5");
374
        int numRead;
375
        
376
        do 
377
        {
378
          numRead = is.read(buffer);
379
          if (numRead > 0) 
380
          {
381
            complete.update(buffer, 0, numRead);
382
          }
383
        } while (numRead != -1);
384
        
385
        
386
        return getHex(complete.digest());
387
    }
388 351
    
389
    /**
390
     * convert a byte array to a hex string
391
     */
392
    private static String getHex( byte [] raw ) 
393
    {
394
        final String HEXES = "0123456789ABCDEF";
395
        if ( raw == null ) {
396
          return null;
397
        }
398
        final StringBuilder hex = new StringBuilder( 2 * raw.length );
399
        for ( final byte b : raw ) {
400
          hex.append(HEXES.charAt((b & 0xF0) >> 4))
401
             .append(HEXES.charAt((b & 0x0F)));
402
        }
403
        return hex.toString();
404
    }
405 352
    
406 353
    /**
407 354
     * parse the metacat date which looks like 2010-06-08 (YYYY-MM-DD) into
......
576 523
        throws Exception
577 524
    {
578 525
        getResponse(sourceUrl, "/metacat", "action=logout&username=" + username, "POST");
579
        getResponse(destinationUrl, "/metacat", "action=logout&username=" + username, "POST");
580 526
    }
581 527
    
582 528
    /**

Also available in: Unified diff