| 1 |
10244
|
jones
|
#!/bin/bash
|
| 2 |
10237
|
jones
|
|
| 3 |
|
|
#
|
| 4 |
|
|
# Backup files needed for metacat. This script creates a directory in /var/metacat/backup,
|
| 5 |
|
|
# backs up the postgres database, metacat data files, and certificate and web config files,
|
| 6 |
|
|
# then syncs those files to an Amazon S3 bucket for backup.
|
| 7 |
|
|
#
|
| 8 |
|
|
# To run this file, install it in /usr/sbin or a similar location and add an
|
| 9 |
|
|
# entry in the root user's crontab to run the command periodically. The
|
| 10 |
|
|
# following crontab entry would run it every night at 2:00 am
|
| 11 |
|
|
# 0 2 * * * /usr/sbin/backup-aws.sh >> $HOME/cron-output 2>&1
|
| 12 |
|
|
#
|
| 13 |
|
|
# This is really just an example script and may not work in your environment
|
| 14 |
|
|
# uless you modify it appropriately.
|
| 15 |
|
|
#
|
| 16 |
|
|
# 25 Apr 2017 Matt Jones
|
| 17 |
|
|
|
| 18 |
|
|
# Name of the metacat database in postgres to be backed up
|
| 19 |
|
|
DBNAME=metacat
|
| 20 |
|
|
|
| 21 |
|
|
# Number of days of backups to keep online, anything older is removed
|
| 22 |
|
|
DAYSTOKEEP=7
|
| 23 |
|
|
|
| 24 |
|
|
# AWS S3 bucket to be used for backup
|
| 25 |
|
|
BUCKET=s3://arcticdata.io/backup
|
| 26 |
|
|
|
| 27 |
|
|
# Location of the metacat.properties file
|
| 28 |
|
|
METACATPROPERTIESPATH=/var/lib/tomcat7/webapps/metacat/WEB-INF/metacat.properties
|
| 29 |
|
|
|
| 30 |
|
|
# Location of the apache configuration file
|
| 31 |
|
|
APACHECONF=/etc/apache2/sites-enabled
|
| 32 |
|
|
|
| 33 |
|
|
#Location of the server key
|
| 34 |
|
|
KEYLOCATION=/etc/ssl/private
|
| 35 |
|
|
|
| 36 |
|
|
#Location of the server certificate
|
| 37 |
|
|
CERTLOCATION=/etc/ssl/certs/www_arcticdata_io.crt
|
| 38 |
10244
|
jones
|
|
| 39 |
10245
|
jones
|
# Location of the AWS client utility
|
| 40 |
|
|
AWS=/usr/local/bin/aws
|
| 41 |
|
|
|
| 42 |
10237
|
jones
|
#
|
| 43 |
|
|
# Below here lie demons
|
| 44 |
|
|
#
|
| 45 |
|
|
|
| 46 |
|
|
# Set up our umask to protect files from prying eyes
|
| 47 |
|
|
umask 007
|
| 48 |
|
|
|
| 49 |
|
|
# Make a temp dir for the backed up files
|
| 50 |
|
|
TAG=`date +%F-%H%M%S`
|
| 51 |
|
|
DATADIR="/var/metacat"
|
| 52 |
|
|
ARCHROOT="/var/metacat/metacat-backup"
|
| 53 |
|
|
mkdir -p $ARCHROOT
|
| 54 |
|
|
chgrp postgres $ARCHROOT
|
| 55 |
|
|
chmod g+rwxs $ARCHROOT
|
| 56 |
|
|
|
| 57 |
|
|
ARCHDIR="$ARCHROOT"
|
| 58 |
|
|
mkdir -p $ARCHDIR
|
| 59 |
|
|
|
| 60 |
10244
|
jones
|
# Locations of file lists already archived
|
| 61 |
|
|
DATALIST=${ARCHDIR}/data-list
|
| 62 |
|
|
METALIST=${ARCHDIR}/meta-list
|
| 63 |
|
|
|
| 64 |
10237
|
jones
|
# Shut down the tomcat server so nobody else changes anything while we backup
|
| 65 |
|
|
#/etc/init.d/tomcat7 stop
|
| 66 |
|
|
|
| 67 |
10244
|
jones
|
echo Generate a list of new metadata files since the last backup
|
| 68 |
|
|
if [ ! -e ${METALIST} ];
|
| 69 |
|
|
then
|
| 70 |
10245
|
jones
|
$AWS s3 ls ${BUCKET}/documents/ |awk -F" " '{print $4}' > ${METALIST}
|
| 71 |
10244
|
jones
|
fi
|
| 72 |
|
|
diff --old-line-format="" --unchanged-line-format="" <(sort ${METALIST}) <(ls -1 ${DATADIR}/documents | sort) > ${METALIST}-new
|
| 73 |
|
|
|
| 74 |
|
|
echo Generate a list of new data files since the last backup
|
| 75 |
|
|
if [ ! -e ${DATALIST} ];
|
| 76 |
|
|
then
|
| 77 |
10245
|
jones
|
$AWS s3 ls ${BUCKET}/data/ |awk -F" " '{print $4}' > ${DATALIST}
|
| 78 |
10244
|
jones
|
fi
|
| 79 |
|
|
diff --old-line-format="" --unchanged-line-format="" <(sort ${DATALIST}) <(ls -1 ${DATADIR}/data | sort) > ${DATALIST}-new
|
| 80 |
|
|
|
| 81 |
10237
|
jones
|
echo Copy the metacat.properties file to /var/metacat
|
| 82 |
|
|
cp $METACATPROPERTIESPATH $ARCHDIR
|
| 83 |
|
|
|
| 84 |
|
|
echo Backup postgres
|
| 85 |
|
|
su - postgres -c "pg_dumpall | gzip > $ARCHDIR/metacat-postgres-backup.gz"
|
| 86 |
|
|
|
| 87 |
|
|
echo Copy the apache configuration files
|
| 88 |
|
|
tar czhf $ARCHDIR/apache-config-backup.tgz $APACHECONF $KEYLOCATION $CERTLOCATION
|
| 89 |
|
|
|
| 90 |
|
|
echo Sync the backup directory to Amazon S3
|
| 91 |
10244
|
jones
|
echo Handle each of the subdirectories independently
|
| 92 |
10245
|
jones
|
$AWS s3 sync $DATADIR/certs $BUCKET/certs
|
| 93 |
|
|
$AWS s3 sync $DATADIR/dataone $BUCKET/dataone
|
| 94 |
|
|
$AWS s3 sync $DATADIR/inline-data $BUCKET/inline-data
|
| 95 |
|
|
$AWS s3 sync $DATADIR/logs $BUCKET/logs
|
| 96 |
|
|
$AWS s3 sync $DATADIR/.metacat $BUCKET/.metacat
|
| 97 |
|
|
$AWS s3 sync $DATADIR/metacat-backup $BUCKET/metacat-backup
|
| 98 |
|
|
#$AWS s3 sync $DATADIR/solr-home $BUCKET/solr-home
|
| 99 |
|
|
#$AWS s3 sync $DATADIR/tdb $BUCKET/tdb
|
| 100 |
|
|
#$AWS s3 sync $DATADIR/temporary $BUCKET/temporary
|
| 101 |
10237
|
jones
|
|
| 102 |
10244
|
jones
|
echo Backup metadata files to S3
|
| 103 |
10245
|
jones
|
cat ${METALIST}-new | xargs -n1 -P30 -I {} $AWS s3 cp ${DATADIR}/documents/{} $BUCKET/documents/{}
|
| 104 |
10244
|
jones
|
cat ${METALIST}-new >> ${METALIST}
|
| 105 |
|
|
|
| 106 |
|
|
echo Backup data files to S3
|
| 107 |
10245
|
jones
|
cat ${DATALIST}-new | xargs -n1 -P30 -I {} $AWS s3 cp ${DATADIR}/data/{} $BUCKET/data/{}
|
| 108 |
10244
|
jones
|
cat ${DATALIST}-new >> ${DATALIST}
|
| 109 |
|
|
|
| 110 |
10237
|
jones
|
# Restart tomcat
|
| 111 |
|
|
#/etc/init.d/tomcat7 start
|
| 112 |
|
|
|
| 113 |
|
|
# Clean up the temp files
|
| 114 |
|
|
#rm -rf $ARCHDIR
|
| 115 |
|
|
|
| 116 |
|
|
# clean up any of the backup files that are older than DAYSTOKEEP
|
| 117 |
|
|
#find $ARCHROOT -mtime +$DAYSTOKEEP -exec rm -f {} \;
|
| 118 |
|
|
|
| 119 |
|
|
echo "DONE backup for $TAG"
|