Revision 10245
Added by Matt Jones over 7 years ago
| src/scripts/bash/backup-aws.sh | ||
|---|---|---|
| 36 | 36 |
#Location of the server certificate |
| 37 | 37 |
CERTLOCATION=/etc/ssl/certs/www_arcticdata_io.crt |
| 38 | 38 |
|
| 39 |
# Location of the AWS client utility |
|
| 40 |
AWS=/usr/local/bin/aws |
|
| 41 |
|
|
| 39 | 42 |
# |
| 40 | 43 |
# Below here lie demons |
| 41 | 44 |
# |
| ... | ... | |
| 64 | 67 |
echo Generate a list of new metadata files since the last backup |
| 65 | 68 |
if [ ! -e ${METALIST} ];
|
| 66 | 69 |
then |
| 67 |
aws s3 ls ${BUCKET}/documents/ |awk -F" " '{print $4}' > ${METALIST}
|
|
| 70 |
$AWS s3 ls ${BUCKET}/documents/ |awk -F" " '{print $4}' > ${METALIST}
|
|
| 68 | 71 |
fi |
| 69 | 72 |
diff --old-line-format="" --unchanged-line-format="" <(sort ${METALIST}) <(ls -1 ${DATADIR}/documents | sort) > ${METALIST}-new
|
| 70 | 73 |
|
| 71 | 74 |
echo Generate a list of new data files since the last backup |
| 72 | 75 |
if [ ! -e ${DATALIST} ];
|
| 73 | 76 |
then |
| 74 |
aws s3 ls ${BUCKET}/data/ |awk -F" " '{print $4}' > ${DATALIST}
|
|
| 77 |
$AWS s3 ls ${BUCKET}/data/ |awk -F" " '{print $4}' > ${DATALIST}
|
|
| 75 | 78 |
fi |
| 76 | 79 |
diff --old-line-format="" --unchanged-line-format="" <(sort ${DATALIST}) <(ls -1 ${DATADIR}/data | sort) > ${DATALIST}-new
|
| 77 | 80 |
|
| ... | ... | |
| 86 | 89 |
|
| 87 | 90 |
echo Sync the backup directory to Amazon S3 |
| 88 | 91 |
echo Handle each of the subdirectories independently |
| 89 |
aws s3 sync $DATADIR/certs $BUCKET/certs
|
|
| 90 |
aws s3 sync $DATADIR/dataone $BUCKET/dataone
|
|
| 91 |
aws s3 sync $DATADIR/inline-data $BUCKET/inline-data
|
|
| 92 |
aws s3 sync $DATADIR/logs $BUCKET/logs
|
|
| 93 |
aws s3 sync $DATADIR/.metacat $BUCKET/.metacat
|
|
| 94 |
aws s3 sync $DATADIR/metacat-backup $BUCKET/metacat-backup
|
|
| 95 |
#aws s3 sync $DATADIR/solr-home $BUCKET/solr-home
|
|
| 96 |
#aws s3 sync $DATADIR/tdb $BUCKET/tdb
|
|
| 97 |
#aws s3 sync $DATADIR/temporary $BUCKET/temporary
|
|
| 92 |
$AWS s3 sync $DATADIR/certs $BUCKET/certs
|
|
| 93 |
$AWS s3 sync $DATADIR/dataone $BUCKET/dataone
|
|
| 94 |
$AWS s3 sync $DATADIR/inline-data $BUCKET/inline-data
|
|
| 95 |
$AWS s3 sync $DATADIR/logs $BUCKET/logs
|
|
| 96 |
$AWS s3 sync $DATADIR/.metacat $BUCKET/.metacat
|
|
| 97 |
$AWS s3 sync $DATADIR/metacat-backup $BUCKET/metacat-backup
|
|
| 98 |
#$AWS s3 sync $DATADIR/solr-home $BUCKET/solr-home
|
|
| 99 |
#$AWS s3 sync $DATADIR/tdb $BUCKET/tdb
|
|
| 100 |
#$AWS s3 sync $DATADIR/temporary $BUCKET/temporary
|
|
| 98 | 101 |
|
| 99 | 102 |
echo Backup metadata files to S3 |
| 100 |
cat ${METALIST}-new | xargs -n1 -P30 -I {} aws s3 cp ${DATADIR}/documents/{} $BUCKET/documents/{}
|
|
| 103 |
cat ${METALIST}-new | xargs -n1 -P30 -I {} $AWS s3 cp ${DATADIR}/documents/{} $BUCKET/documents/{}
|
|
| 101 | 104 |
cat ${METALIST}-new >> ${METALIST}
|
| 102 | 105 |
|
| 103 | 106 |
echo Backup data files to S3 |
| 104 |
cat ${DATALIST}-new | xargs -n1 -P30 -I {} aws s3 cp ${DATADIR}/data/{} $BUCKET/data/{}
|
|
| 107 |
cat ${DATALIST}-new | xargs -n1 -P30 -I {} $AWS s3 cp ${DATADIR}/data/{} $BUCKET/data/{}
|
|
| 105 | 108 |
cat ${DATALIST}-new >> ${DATALIST}
|
| 106 | 109 |
|
| 107 | 110 |
# Restart tomcat |
Also available in: Unified diff
Define location of the AWS binary so the tool works under CRON.