Metacat

#!/bin/sh

#

# Backup files needed for metacat.  This script creates a temporary directory,

# backs up the postgres database, metacat data files, and ldap directory, and 

# then creates a gzipped tar file from those files.  It then writes the backup

# file to a DVD drive and then cleans up any old backup files.

#

# To run this file, install it in /usr/sbin or a similar location and add an

# entry in the root user's crontab to run the command periodically.  The

# following crontab entry would run it every night at 2:00 am

# 0 2 * * *       /usr/sbin/backup-metacat.sh >> $HOME/cron-output 2>&1

#

# This is really just an example script and may not work in your environment

# uless you modify it appropriately.

#

# 13 March 2006 Matt Jones

# Name of the metacat database in postgres to be backed up

DBNAME=metacat

# The day of the week on which the DVD is swapped (1-7, 1=Monday)

# This must be correct or else the write to DVD will not work

SWAPDAY=2

# Number of days of backups to keep online, anything older is removed

DAYSTOKEEP=7

# Device to be used for the DVD writer -- this may vary on your system

DVD=/dev/dvd

#

# Below here lie demons

#

# Set up our umask to protect files from prying eyes

umask 007

# Make a temp dir for the backed up files

TAG=`date +%F-%H%M%S`

DATADIR="/var/metacat"

ARCHROOT="/var/metacat/metacat-backup"

mkdir $ARCHROOT

chgrp postgres $ARCHROOT

chmod g+rwxs $ARCHROOT

ARCHNAME="metacat-backup-$TAG"

ARCHDIR="$ARCHROOT/$ARCHNAME"

mkdir $ARCHDIR

# Shut down the tomcat server so nobody else changes anything while we backup

/etc/init.d/tomcat stop

# Shut down ldap too

/etc/init.d/slapd stop

# Backup postgres

su - postgres -c "pg_dump $DBNAME > $ARCHDIR/metacat-postgres-backup.sql"

# Backup the data files

tar czf $ARCHDIR/datafiles-backup.tgz --exclude=$ARCHROOT $DATADIR

# Backup LDAP to an LDIF file

slapcat -l $ARCHDIR/$DBNAME-ldap.ldif

# Restart LDAP

/etc/init.d/slapd start

# Restart tomcat

/etc/init.d/tomcat start

# Tar up the archive and copy it to archive media

cd $ARCHROOT

tar czf $ARCHDIR.tgz $ARCHNAME

# Clean up the temp files

rm -rf $ARCHDIR

# Write the backup file to DVD

DAY=`date +%u`

DVDFLAG=-M

if [ $DAY == $SWAPDAY ] ; then

  DVDFLAG=-Z

fi

growisofs $DVDFLAG $DVD -R -J $ARCHDIR.tgz

# clean up any of the backup files that are older than DAYSTOKEEP

find $ARCHROOT -mtime +$DAYSTOKEEP -exec rm -f {} \;

#!/bin/sh

#This script will send a buildIndex command to 

#remote metacat with docid parameter. Then metacat will build index for this docid.

#This script will have one argument - it is a text file name which contains docid list.

#The text file will look like

#nceas.1.1

#nceas.2.1

#nceas.3.1

if  [ $# -ne 1 ]

 then

    echo "USAGE: $0 docidfile"

    exit 1

fi 

echo 'Please type metacat url (It should look like "http://pacific.msi.ucsb.edu:8080/knb/metacat")'

read metacatBaseURL

echo 'Please type KNB user name (It should look like "uid=tao,o=NCEAS,dc=ecoinformatics,dc=org")'

read username

echo 'Please type password'

read password

date

#login to metacat

loginurl="$metacatBaseURL?action=login&username=$username&password=$password"

wget -q -O - --save-cookies cookies --keep-session-cookies "$loginurl"

suffix="?action=buildindex"

metacatURL="$metacatBaseURL$suffix"

FILE=$1

# make sure file exist and readable

if [ ! -f $FILE ]; then 

  	echo "$FILE : does not exists"

  	exit 1

elif [ ! -r $FILE ]; then

  	echo "$FILE: can not read"

  	exit 2

fi

index=0

length=3

# read $FILE using the file descriptors

exec 3<&0

exec 0<$FILE

while read line

do

    if [ "$line" != "" ]

    then

      # construct metacat with given length of docid

      metacatURL="$metacatURL&docid=$line"

      index=`expr $index + 1`

	    if [ $index -eq $length ]

	    then

	       echo "here is url $metacatURL"

	       wget -q -O - --load-cookies cookies "$metacatURL"

	       index=0;

	       metacatURL="$metacatBaseURL$suffix"

	    fi

	fi		

done

#This will send out some remained docid

if [ $index -ne 0 ]

then   

    echo "here is url $metacatURL"

    wget -q -O - --load-cookies cookies "$metacatURL"

fi

exec 0<&3

date

exit 0

#!/bin/bash

#Call a java client class to query metacat automatically and metacat server

#will record the performance data into a file. Since we need restart postgres,

#this script should be run as root user

#Note: before running the script, user should run both "ant jar" and "ant clientjar"

#in Metacat to get metaca-client.jar and metacat.jar in build directory. 

#variables

JUNIT=/usr/local/devtools/apache-ant/lib/junit-4.3.1.jar

METACAT=../build/metacat.jar

METACAT_CLIENT=../build/metacat-client.jar

UTILITIES=../lib/utilities.jar

METACATURL=http://chico.dyndns.org:8081/knb/metacat

QUERY=../test/performance_measure_query

TIME=2

TOMCATUSER=tao

#copy the java client to build directory

rm -rf ../build/tests/

mkdir ../build/tests

mkdir ../build/tests/edu

mkdir ../build/tests/edu/ucsb

mkdir ../build/tests/edu/ucsb/nceas

mkdir ../build/tests/edu/ucsb/nceas/metacattest

cp ../test/edu/ucsb/nceas/metacattest/MetaCatQueryPerformanceTest.java ../build/tests/edu/ucsb/nceas/metacattest/.

#compile the class

javac -classpath $JUNIT:$METACAT:$METACAT_CLIENT:$UTILITIES ../build/tests/edu/ucsb/nceas/metacattest/MetaCatQueryPerformanceTest.java 

#Iterate to restart postgres and tomcat, and call java client class

for (( i = 0; i < $TIME; i++))

do

    #retart postgres

    /etc/init.d/postgresql-8.2 restart

    #Check if posgres restart successfully

    successPQL=false;

    while [ $successPQL = false ]

    do

      #if found netstat command has posgres lisenter is ready,

      #reset the success value to jump the while loop.

      for fn in `netstat -plt | grep postgres`

      do

        #reset success value

        successPQL=true

      done

    done

    echo "success value for restarting postgres is $successPQL"

    #stop tomcat

    su - $TOMCATUSER /usr/local/devtools/apache-tomcat-5.5.23/bin/shutdown.sh

     #Check if tomcat start successfully

    successStop=false;

    tomcatRunning=false

    while [ $successStop = false ]

    do

      #if found netstat command still has tcomat lisenter (port 8005) ,

      #reset the tomcatRunning value to true

      for fn in `netstat -plt | grep 8005`

      do

        #reset success value

        tomcatRunning=true;

      done

      if [ $tomcatRunning = true ]

         then

           successStop=false

         else

           successStop=true

      fi

    done

    echo "success value for stoping tomcat is $successStop"

    #start tomcat

   su - $TOMCATUSER  /usr/local/devtools/apache-tomcat-5.5.23/bin/startup.sh

    #Check if tomcat start successfully

    successStart=false;

    while [ $successStart = false ]

    do

      #if found netstat command has tcomat lisenter (port 8005) is ready,

      #reset the success value to jump the while loop.

      for fn in `netstat -plt | grep 8005`

      do

        #reset success value

        successStart=true

      done

    done

    echo "success value for starting tomcat is $successStart"

    #run the class -- query the remote metacat

    java -cp $JUNIT:$METACAT:$METACAT_CLIENT:$UTILITIES:../build/tests edu.ucsb.nceas.metacattest.MetaCatQueryPerformanceTest $METACATURL $QUERY

    echo "Successfully query the metacat"

done

#!/bin/bash

#This file will check if shape files exist or not.

#If one is deleted and a email will be sent to tao with the tomcat log.

#Note: if file eml_message already exists in current dir, email wouldn't send again.

#This setting will make sure only one email will sent. Otherwise, my email account will be over quota.

target_file1=/var/www/org.ecoinformatics.knb/knb/data/metacat_shps/data_points.dbf

target_file2=/var/www/org.ecoinformatics.knb/knb/data/metacat_shps/data_points.shp

target_file3=/var/www/org.ecoinformatics.knb/knb/data/metacat_shps/data_points.shx

target_file4=/var/www/org.ecoinformatics.knb/knb/data/metacat_shps/data_bounds.dbf

target_file5=/var/www/org.ecoinformatics.knb/knb/data/metacat_shps/data_bounds.shp

target_file6=/var/www/org.ecoinformatics.knb/knb/data/metacat_shps/data_bounds.shx

log_file=/var/log/tomcat/catalina.out

sender_email_address=knb_tomcat@ecoinformatics.org

receiver_email_address=tao@nceas.ucsb.edu

tmp=/root/email_message

if [ ! -e $target_file1 -o ! -e $target_file2 -o ! -e $target_file3 -o ! -e $target_file4 -o ! -e $target_file5 -o ! -e $target_file6 ]

 then

   if [ ! -e $tmp ]

     then

       touch $tmp && chmod 600 $tmp

       echo -e "subject: Shape file was deleted at `date`\n" > $tmp

       echo -e "\n log file info: " >> $tmp

       echo -e "\n `tail -300 $log_file`" >> $tmp

       /usr/sbin/sendmail -f $sender_email_address $receiver_email_address < $tmp

   fi

fi

#!/bin/bash

#Call a java client class to query metacat automatically and metacat server

#will record the performance data into a file.  This script will be put in /etc/init.d directory

#and called in system booting. After script is down, it will call reboot system too.

#So every time to excecute the quyer, memory will be in fresh.

#Note: before running the script, user should run both "ant jar" and "ant clientjar"

#in Metacat to get metaca-client.jar and metacat.jar in build directory. 

#variables

JUNIT=/usr/local/devtools/apache-ant/lib/junit-4.3.1.jar

PROJECT=/home/tao/project/metacat

METACATURL=http://chico.dyndns.org:8081/knb/metacat

TOMCATUSER=tao

QUERY=$PROJECT/test/performance_measure_query

TIMEFILE=$PROJECT/test/times

METACAT=$PROJECT/build/metacat.jar

METACAT_CLIENT=$PROJECT/build/metacat-client.jar

UTILITIES=$PROJECT/lib/utilities.jar

#copy the java client to build directory

rm -rf $PROJECT/build/tests/

mkdir -p $PROJECT/build/tests/edu/ucsb/nceas/metacattest

cp $PROJECT/test/edu/ucsb/nceas/metacattest/MetaCatQueryPerformanceTest.java $PROJECT/build/tests/edu/ucsb/nceas/metacattest/.

#compile the class

javac -classpath $JUNIT:$METACAT:$METACAT_CLIENT:$UTILITIES $PROJECT/build/tests/edu/ucsb/nceas/metacattest/MetaCatQueryPerformanceTest.java 

case "$1" in

  start)

    echo "Starting running query script"

    #Iterate to restart postgres and tomcat, and call java client class

    TIMES=`cat $TIMEFILE`

    echo "The value from TIMEFILE is $TIMES" 

    #if times greater than 0, it will query the metacat

    if [ $TIMES -gt 0 ]

      then

      #Check if postgres start successfully

      successPQL=false;

      while [ $successPQL = false ]

         do

             #if found netstat command has posgres lisenter is ready,

             #reset the success value to jump the while loop.

             for fn in `netstat -plt | grep postgres`

               do

                 #reset success value

                 successPQL=true

            done

      done

      echo "success value for starting postgres is $successPQL"

      #Check if tomcat start successfully

      successStart=false;

      while [ $successStart = false ]

        do

           #if found netstat command has tcomat lisenter (port 8005) is ready,

           #reset the success value to jump the while loop.

           for fn in `netstat -plt | grep 8005`

             do

                #reset success value

                successStart=true

           done

      done

     echo "success value for starting tomcat is $successStart"

     #run the class -- query the remote metacat

      java -cp $JUNIT:$METACAT:$METACAT_CLIENT:$UTILITIES:$PROJECT/build/tests edu.ucsb.nceas.metacattest.MetaCatQueryPerformanceTest $METACATURL $QUERY

      echo "Successfully query the metacat"

      #Drecease 1 from value of TIMES

      TIMES=`expr $TIMES - 1`

      echo "The new TIMES value  is $TIMES"

      #write the new TIMES to the file

      echo $TIMES >$TIMEFILE

      #reboot machine

      reboot

    fi

   ;;

 stop)

    echo "Stopping running query script - do nothing"

     ;;

  *)

    echo "Usage: /etc/init.d/blah {start|stop}"

    exit 1

    ;;

esac

exit 0