Project

General

Profile

Revision 7285

first pass: DataONE-specific log retrieval to avoid java-based post-processing.

View differences:

src/edu/ucsb/nceas/metacat/EventLog.java
30 30
import java.util.ArrayList;
31 31
import java.util.Date;
32 32
import java.util.List;
33
import java.util.Vector;
33 34

  
34 35
import org.apache.log4j.Logger;
36
import org.dataone.service.types.v1.Event;
37
import org.dataone.service.types.v1.Identifier;
38
import org.dataone.service.types.v1.Log;
39
import org.dataone.service.types.v1.LogEntry;
40
import org.dataone.service.types.v1.NodeReference;
41
import org.dataone.service.types.v1.Subject;
35 42
import org.dataone.service.util.DateTimeMarshaller;
36 43

  
37 44
import edu.ucsb.nceas.metacat.database.DBConnection;
38 45
import edu.ucsb.nceas.metacat.database.DBConnectionPool;
46
import edu.ucsb.nceas.metacat.properties.PropertyService;
47
import edu.ucsb.nceas.utilities.PropertyNotFoundException;
39 48

  
40 49
/**
41 50
 * EventLog is used to intialize and store a log of events that occur in an
......
291 300
        return resultDoc.toString();
292 301
    }
293 302
    
303
    
304
    
305
    public Log getD1Report(String[] ipAddress, String[] principal, String[] docid,
306
            Event event, Timestamp startDate, Timestamp endDate, boolean anonymous, Integer start, Integer count)
307
    {
308
        
309
        Log log = new Log();
310
    	
311
    	NodeReference memberNode = new NodeReference();
312
        String nodeId = "localhost";
313
        try {
314
            nodeId = PropertyService.getProperty("dataone.nodeId");
315
        } catch (PropertyNotFoundException e1) {
316
            // TODO Auto-generated catch block
317
            e1.printStackTrace();
318
        }
319
        memberNode.setValue(nodeId);
320
        
321
        StringBuffer query = new StringBuffer();
322
        query.append(
323
        		"select " +
324
        		"entryid, " +
325
        		"id.guid as identifier, " +
326
        		"ip_address, " +
327
        		"user_agent, " +
328
        		"principal, " +
329
        		"case " +
330
        		"	when event = 'insert' then 'create' " +
331
        		"	else event " +
332
        		"end as event, " +
333
        		"date_logged " +
334
        		"from access_log al, identifier id " +
335
        		"where al.docid = id.docid||'.'||id.rev "
336
        );
337
        
338
        
339
        boolean clauseAdded = false;
340
        int startIndex = 0;
341
        int endIndex = 0;
342
        
343
        List<String> paramValues = new ArrayList<String>();
344
        if (ipAddress != null) {
345
        	query.append("ip_address in (");
346
        	for (int i = 0; i < ipAddress.length; i++) {
347
        		if (i > 0) {
348
            		query.append(", ");
349
        		}
350
        		query.append("?");
351
        		paramValues.add(ipAddress[i]);
352
        	}
353
        	query.append(") ");
354
            clauseAdded = true;
355
        }
356
        if (principal != null) {
357
        	query.append("principal in (");
358
        	for (int i = 0; i < principal.length; i++) {
359
        		if (i > 0) {
360
            		query.append(", ");
361
        		}
362
        		query.append("?");
363
        		paramValues.add(principal[i]);
364
        	}
365
        	query.append(") ");
366
            clauseAdded = true;
367
        }
368
        if (docid != null) {
369
        	query.append("al.docid in (");
370
        	for (int i = 0; i < docid.length; i++) {
371
        		if (i > 0) {
372
            		query.append(", ");
373
        		}
374
        		query.append("?");
375
        		paramValues.add(docid[i]);
376
        	}
377
        	query.append(") ");
378
            clauseAdded = true;
379
        }
380
        if (event != null) {
381
        	query.append("event in (");
382
    		query.append("?");
383
    		String eventString = event.toString();
384
    		if (event.equals(Event.CREATE)) {
385
    			eventString = "insert";
386
    		}
387
    		paramValues.add(eventString);
388
        	query.append(") ");
389
            clauseAdded = true;
390
        }
391
        if (startDate != null) {
392
            if (clauseAdded) {
393
                query.append(" and ");
394
            }
395
            query.append("date_logged >= ?");
396
            clauseAdded = true;
397
            startIndex++;
398
        }
399
        if (endDate != null) {
400
            if (clauseAdded) {
401
                query.append(" and ");
402
            }
403
            query.append("date_logged < ?");
404
            clauseAdded = true;
405
            endIndex = startIndex + 1;
406
        }
407
        DBConnection dbConn = null;
408
        int serialNumber = -1;
409
        try {
410
            // Get a database connection from the pool
411
            dbConn = DBConnectionPool.getDBConnection("EventLog.getD1Report");
412
            serialNumber = dbConn.getCheckOutSerialNumber();
413

  
414
            // Execute the query statement
415
            PreparedStatement stmt = dbConn.prepareStatement(query.toString());
416
            //set the param values
417
            int parameterIndex = 1;
418
            for (String val: paramValues) {
419
            	stmt.setString(parameterIndex, val);
420
            }
421
            if (startDate != null) {
422
                stmt.setTimestamp(parameterIndex++, startDate); 
423
            }
424
            if (endDate != null) {
425
            	stmt.setTimestamp(parameterIndex++, endDate);
426
            }
427
            stmt.execute();
428
            ResultSet rs = stmt.getResultSet();
429
            //process the result and return it
430
            List<LogEntry> logs = new Vector<LogEntry>();
431
            
432
            while (rs.next()) {
433
            	LogEntry logEntry = new LogEntry();
434
            	logEntry.setEntryId(rs.getString(1));
435
            	
436
            	Identifier identifier = new Identifier();
437
            	identifier.setValue(rs.getString(2));
438
				logEntry.setIdentifier(identifier);
439

  
440
            	logEntry.setIpAddress(anonymous ? "N/A" : rs.getString(3));
441
            	logEntry.setUserAgent(rs.getString(4));
442
            	
443
            	Subject subject = new Subject();
444
            	subject.setValue(anonymous ? "N/A" : rs.getString(5));
445
				logEntry.setSubject(subject);
446
				
447
				Event logEvent = Event.valueOf(rs.getString(6));
448
				logEntry.setEvent(logEvent);
449
				logEntry.setDateLogged(rs.getTimestamp(7));
450
				
451
				logEntry.setNodeIdentifier(memberNode);
452
				logs.add(logEntry);
453
            }
454
            
455
			log.setLogEntryList(logs);
456
			// d1 paging
457
		    int total = logs.size();
458
		    if (start != null && count != null) {
459
		    	int toIndex = start + count;
460
		    	// do not exceed total
461
		    	toIndex = Math.min(toIndex, total);
462
		    	// do not start greater than total
463
		    	start = Math.min(start, total);
464
		    	// sub set of the list
465
		    	logs = new ArrayList<LogEntry>(logs.subList(start, toIndex));
466
		    }
467

  
468
		    log.setLogEntryList(logs);
469
		    log.setStart(start);
470
		    log.setCount(logs.size());
471
		    log.setTotal(total);
472

  
473
            stmt.close();
474
        } catch (SQLException e) {
475
        	logMetacat.info("Error while getting log events: " + e.getMessage());
476
        } finally {
477
            // Return database connection to the pool
478
            DBConnectionPool.returnDBConnection(dbConn, serialNumber);
479
        }
480
        return log;
481
    }
482
    
294 483
    /**
295 484
     * Format each returned log record as an XML structure.
296 485
     * 
src/edu/ucsb/nceas/metacat/dataone/D1NodeService.java
439 439
	  // see https://redmine.dataone.org/issues/2855
440 440
	  if (!isAdminAuthorized(session)) {
441 441
		  throw new NotAuthorized("1460", "Only the CN or admin is allowed to harvest logs from this node");
442
		  
443 442
	  }
444 443
	  
445
    Log log = new Log();
446
    List<LogEntry> logs = new Vector<LogEntry>();
447 444
    IdentifierManager im = IdentifierManager.getInstance();
448 445
    EventLog el = EventLog.getInstance();
449 446
    if ( fromDate == null ) {
......
456 453
    }
457 454

  
458 455
    if ( start == null ) {
459
    	start = 0;
460
    	
456
    	start = 0;	
461 457
    }
462 458
    
463 459
    if ( count == null ) {
464 460
    	count = 1000;
465
    	
466 461
    }
467 462

  
468 463
    String[] filterDocid = null;
......
480 475
    logMetacat.debug("fromDate: " + fromDate);
481 476
    logMetacat.debug("toDate: " + toDate);
482 477

  
483
    String report = el.getReport(null, null, filterDocid, null,
478
    Log log = el.getD1Report(null, null, filterDocid, event,
484 479
        new java.sql.Timestamp(fromDate.getTime()),
485
        new java.sql.Timestamp(toDate.getTime()), false);
486

  
487
    logMetacat.debug("report: " + report);
480
        new java.sql.Timestamp(toDate.getTime()), false, start, count);
488 481
    
489
    NodeReference memberNode = new NodeReference();
490
    String nodeId = "localhost";
491
    try {
492
        nodeId = PropertyService.getProperty("dataone.nodeId");
493
    } catch (PropertyNotFoundException e1) {
494
        // TODO Auto-generated catch block
495
        e1.printStackTrace();
496
    }
497
    memberNode.setValue(nodeId);
498

  
499
    String logEntry = "<logEntry>";
500
    String endLogEntry = "</logEntry>";
501
    int startIndex = 0;
502
    int foundIndex = report.indexOf(logEntry, startIndex);
503
    while (foundIndex != -1) {
504
      // parse out each entry
505
      int endEntryIndex = report.indexOf(endLogEntry, foundIndex);
506
      String entry = report.substring(foundIndex, endEntryIndex);
507
      logMetacat.debug("entry: " + entry);
508
      startIndex = endEntryIndex + endLogEntry.length();
509
      foundIndex = report.indexOf(logEntry, startIndex);
510

  
511
      String entryId = getLogEntryField("entryid", entry);
512
      String ipAddress = getLogEntryField("ipAddress", entry);
513
      String principal = getLogEntryField("principal", entry);
514
      String userAgent = getLogEntryField("userAgent", entry);
515
      String docid = getLogEntryField("docid", entry);
516
      String eventS = getLogEntryField("event", entry);
517
      String dateLogged = getLogEntryField("dateLogged", entry);
518

  
519
      LogEntry le = new LogEntry();
520

  
521
      // handle Metacat -> DataONE event conversion
522
      if (eventS.equalsIgnoreCase("insert")) {
523
    	  eventS = Event.CREATE.xmlValue();
524
      }
525
      Event e = Event.convert(eventS);
526
      if (e == null) { // skip any events that are not Dataone Crud events
527
          logMetacat.warn("skipping unknown event type: '" + eventS + "'");
528
          continue;
529
      }
530
      le.setEvent(e);
531
      le.setEntryId(entryId);
532
      Identifier identifier = new Identifier();
533
      try {
534
        logMetacat.debug("converting docid '" + docid + "' to a pid.");
535
        if (docid == null || docid.trim().equals("") || docid.trim().equals("null")) {
536
          continue;
537
        }
538
        String docidNoRev = DocumentUtil.getSmartDocId(docid);
539
        //docid = docid.substring(0, docid.lastIndexOf("."));
540
        int rev = DocumentUtil.getRevisionFromAccessionNumber(docid);
541
        //int rev = im.getLatestRevForLocalId(docid);
542
        String guid = im.getGUID(docidNoRev, rev);
543
        identifier.setValue(guid);
544
      } catch (Exception ex) { 
545
        // try to get the pid, if that doesn't
546
        // work, just use the local id
547
        // throw new ServiceFailure("1030",
548
        // "Error getting pid for localId " +
549
        // docid + ": " + ex.getMessage());\
550

  
551
          logMetacat.warn("could not find pid for docid '" + docid + "'");
552

  
553
        // skip it if the pid can't be found
554
        continue;
555
      }
556
      
557
      le.setIdentifier(identifier);
558
      le.setIpAddress(ipAddress);
559
      Date logDate = DateTimeMarshaller.deserializeDateToUTC(dateLogged);
560
      le.setDateLogged(logDate);
561
      le.setNodeIdentifier(memberNode);
562
      Subject princ = new Subject();
563
      princ.setValue(principal);
564
      le.setSubject(princ);
565
      le.setUserAgent(userAgent);
566

  
567
      // event filtering?
568
      if (event == null) {
569
    	  logs.add(le);
570
      } else if (le.getEvent().equals(event)) {
571
    	  logs.add(le);
572
      }
573
    }
574
    
575
    // d1 paging
576
    int total = logs.size();
577
    if (start != null && count != null) {
578
    	int toIndex = start + count;
579
    	// do not exceed total
580
    	toIndex = Math.min(toIndex, total);
581
    	// do not start greater than total
582
    	start = Math.min(start, total);
583
    	// sub set of the list
584
    	logs = new ArrayList<LogEntry>(logs.subList(start, toIndex));
585
    }
586

  
587
    log.setLogEntryList(logs);
588
    log.setStart(start);
589
    log.setCount(logs.size());
590
    log.setTotal(total);
591 482
    logMetacat.info("getLogRecords");
592 483
    return log;
593 484
  }

Also available in: Unified diff