merge 2.1.4 into master

bin/backup_omva.sh

@@ -1,25 +1,68 @@
 #!/bin/bash
 
 # Backup script for the OpenManage Virtual Appliance.
-# Today: Generates a tarball with the important-to-backup data.
-# Tomorrow: This will eventually send the tarball automatically for secure offsite backup.
+
+if [ "$EUID" -ne 0 ]
+  then echo "Please run as root"
+  exit
+fi
 
 . /etc/default/openmanage
 
-backup_workspace=$HOME/omva-backup
-backup_date=`date -u +%Y%m%d_%H%M`
-# Stage one: prepare the destination
-mkdir -p $backup_workspace
+BACKUP_HASH_FILE=$OPENMANAGE_ROOT/.backup_hashes
+
+lock() {
+    exec 200>/var/lock/.myscript.exclusivelock
+
+    flock -n 200 \
+        && return 0 \
+        || return 1
+}
+
+eexit() {
+    local error_str="$@"
+
+    echo $error_str
+    exit 1
+}
 
-# Stage two: Collect the trivial stuff.
-cp $OPENMANAGE_CONFIGDIR/agent_config.json $backup_workspace
-cp -r $SPIDEROAK_ESCROW_KEYS_PATH $backup_workspace
-cp -r $SPIDEROAK_ESCROW_LAYERS_PATH $backup_workspace
+set_current_values() {
+    md5=($(md5sum $OPENMANAGE_ROOT/etc/agent_config.json))
+    echo "export AGENT_CONFIG_MD5=$md5" > $BACKUP_HASH_FILE
+    md5=($(echo "select datname, tup_inserted, tup_updated, tup_deleted from pg_stat_database where datname='openmanage';" | sudo -u postgres psql openmanage | md5sum))
+    echo "export DB_MD5=$md5" >> $BACKUP_HASH_FILE
+}
 
-# Stage three: collect the DB contents.
-su postgres -c "pg_dump openmanage" > $backup_workspace/db_dump.sql
+need_backup() {
+    if ! [ -a $BACKUP_HASH_FILE ] 
+    then
+        set_current_values
+        echo "Backup needed."
+        return 0
+    fi
+    . $BACKUP_HASH_FILE
+    md5=($(md5sum $OPENMANAGE_ROOT/etc/agent_config.json))
+    if [ $AGENT_CONFIG_MD5 != $md5 ]
+    then
+        echo "agent_config doesn't match last backup. Backup needed."
+        return 0
+    fi
+    md5=($(echo "select datname, tup_inserted, tup_updated, tup_deleted from pg_stat_database where datname='openmanage';" | sudo -u postgres psql openmanage | md5sum))
+    if [ $DB_MD5 != $md5 ]
+    then
+        echo "Database doesn't match last backup. Backup needed."
+        return 0
+    fi
+    return 1
+}
 
-pushd $HOME
-tar czf $HOME/omva-backup-$backup_date.tar.gz ./omva-backup
-rm -r $backup_workspace
-popd
+main() {
+    lock || eexit "Script is already running. Exiting..."
+    need_backup || eexit "Nothing has changed since last backup. Exiting..."
+    number=$(( ( RANDOM % 45 ) + 1 ))
+    echo "sleeping for $number minutes to spread out backups"
+    sleep ${number}m
+    set_current_values
+    python $OPENMANAGE_ROOT/bin/update_backup.py
+}
+main

bin/create_backup.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. /etc/default/openmanage
+
+CURRENT_DATE=$1
+
+BACKUP_BASE=$OPENMANAGE_ROOT/tmp_backup
+BACKUP_DIR=openmanage-backup-$CURRENT_DATE
+BACKUP_BZ2=openmanage-backup-$CURRENT_DATE.tar.bz2
+
+rm -rf $BACKUP_BASE
+mkdir $BACKUP_BASE
+cd $BACKUP_BASE
+mkdir $BACKUP_DIR
+
+cp -r $SPIDEROAK_ESCROW_LAYERS_PATH $BACKUP_DIR
+cp -r $SPIDEROAK_ESCROW_KEYS_PATH $BACKUP_DIR
+cp $OPENMANAGE_CONFIGDIR/agent_config.json $BACKUP_DIR
+sudo -u postgres pg_dump openmanage > $BACKUP_DIR/openmanage.sql
+
+tar cjf $BACKUP_BZ2 $BACKUP_DIR

bin/first_setup.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-if [ -e ~/.ran_firstsetup ]; then
+if [ -e /opt/openmanage/etc/.ran_firstsetup ]; then
+    echo "Keys already generated. Exiting"
     exit 0
 fi
 
@@ -15,10 +16,6 @@ if [ ! -f /var/lib/openmanage/keys/base.cfg ]; then
 $OPENMANAGE_ROOT/bin/make_keys.sh $OPENMANAGE_BRAND
 fi
 
-#sudo mkdir -p /etc/service/openmanage/supervise
-#sudo ln -s $OPENMANAGE_ROOT/etc/service/openmanage/run /etc/service/openmanage/run
-#sudo sv start openmanage
-
-touch ~/.ran_firstsetup
+touch /opt/openmanage/etc/.ran_firstsetup
 
 echo "finished"

bin/gather_logs.sh

@@ -6,13 +6,11 @@ LOG_BASE=/opt/openmanage/tmp_logs
 LOG_DIR=$LOG_BASE/openmanage-logs-$CURRENT_DATE
 LOG_BZ2=$LOG_BASE/openmanage-logs-$CURRENT_DATE.tar.bz2
 
+rm -rf $LOG_BASE
 mkdir $LOG_BASE
 mkdir $LOG_DIR
 
 cp -r /var/log/admin_console/ $LOG_DIR
 cp -r /var/log/omva/ $LOG_DIR
 
 tar cjf $LOG_BZ2 $LOG_DIR
-
-#rm -rf $LOG_DIR
-#rm -rf $LOG_BZ2

bin/resources/recreate_openmanage.sql

@@ -0,0 +1,10 @@
+SELECT
+    pg_terminate_backend (pg_stat_activity.procpid)
+FROM
+    pg_stat_activity
+WHERE
+    pg_stat_activity.datname = 'openmanage'
+    and procpid != pg_backend_pid()
+;
+drop database openmanage;
+create database openmanage;

bin/restore_backup.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+import os
+import datetime
+from hashlib import sha256
+import subprocess
+from binascii import a2b_base64
+import bcrypt
+import nacl.secret
+
+from netkes import common
+from netkes.account_mgr.accounts_api import Api
+
+config = common.read_config_file()
+
+api = Api.create(
+    config["api_root"],
+    config["api_user"],
+    config["api_password"],
+)
+
+def create_secret_box(password, username):
+    key = bcrypt.kdf(
+        password.encode('utf-8'),
+        username,                
+        nacl.secret.SecretBox.KEY_SIZE, 
+        100,
+    )
+
+    nonce = nacl.utils.random(nacl.secret.SecretBox.NONCE_SIZE)
+    return nacl.secret.SecretBox(key), nonce
+
+secret_box, nonce = create_secret_box(config['api_password'], config['api_user'])
+
+date = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
+filename = 'openmanage-backup-%s.tar.bz2' % date
+path = '/opt/openmanage/tmp_backup/%s' % filename
+
+backup = api.backup()
+
+data = a2b_base64(backup['data'])
+data = secret_box.decrypt(data)
+
+with open(path, 'w') as f:
+    f.write(data)
+
+subprocess.call(['/opt/openmanage/bin/restore_omva.sh', path])

bin/restore_omva.sh

@@ -6,29 +6,40 @@
 
 . /etc/default/openmanage
 
+set -x
+
 # Stage Zero: Sanity-check the tarball
-file $1 | grep 'gzip compressed data' 2>&1 1>/dev/null
+file $1 | grep 'bzip2 compressed data' 2>&1 1>/dev/null
 if [ $? != 0 ]; then
     echo "Backup argument $1 not showing as a tarball properly.  Aborting." >&2
     exit
 fi
 
 # Stage One: Unzip the tarball.
-tar xzf $1
-pushd omva-backup
+BACKUP_BASE=$OPENMANAGE_ROOT/tmp_backup
+
+cd $BACKUP_BASE
+
+tar xjfv $1
+pushd openmanage-backup*
 
 # Stage Two: move the configuration and keys back into place.
 mkdir -p $SPIDEROAK_ESCROW_KEYS_PATH
 mkdir -p $SPIDEROAK_ESCROW_LAYERS_PATH
 
-cp -r omva-backup/keys/* $SPIDEROAK_ESCROW_KEYS_PATH
-cp -r omva-backup/layers/* $SPIDEROAK_ESCROW_LAYERS_PATH
+cp -r keys/* $SPIDEROAK_ESCROW_KEYS_PATH
+cp -r layers/* $SPIDEROAK_ESCROW_LAYERS_PATH
 
-cp agent-config.json $OPENMANAGE_CONFIGDIR
+cp agent_config.json $OPENMANAGE_CONFIGDIR
 
 # Stage Three: Re-load the DB SQL.
-su postgres -c "psql -f db_dump.sql openmanage"
+sudo -u postgres psql -f /opt/openmanage/bin/resources/recreate_openmanage.sql
+sudo -u postgres psql --single-transaction --pset pager=off -f openmanage.sql openmanage 
+
+# We already have keys so we don't need to run first setup
+touch /opt/openmanage/etc/.ran_firstsetup
 
 # Clean up.
 popd
-rm -r omva-backup
+rm -r openmanage-backup*
+sudo sv restart admin_console

bin/run_restore_omva.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Backup script for the OpenManage Virtual Appliance.
+
+if [ "$EUID" -ne 0 ]
+  then echo "Please run as root"
+  exit
+fi
+
+. /etc/default/openmanage
+
+BACKUP_BASE=$OPENMANAGE_ROOT/tmp_backup
+
+rm -rf $BACKUP_BASE
+mkdir $BACKUP_BASE
+
+
+python $OPENMANAGE_ROOT/bin/restore_backup.py

bin/set_version.py

@@ -0,0 +1,18 @@
+import sys
+from netkes import common
+from netkes.account_mgr.accounts_api import Api
+
+config = common.read_config_file()
+
+if len(sys.argv) != 2:
+    print "version is required"
+    print "python set_version.py 1.2.3"
+    sys.exit(0)
+
+if config['api_password']:
+    api = Api.create(
+        config["api_root"],
+        config["api_user"],
+        config["api_password"],
+    )
+    api.update_enterprise_settings(dict(api_version=sys.argv[1]))

bin/update_backup.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import os
+import datetime
+from hashlib import sha256
+import subprocess
+from binascii import b2a_base64
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'omva.settings'
+
+from openmanage.views import create_secret_box
+from netkes import common
+from netkes.account_mgr.accounts_api import Api
+
+config = common.read_config_file()
+
+# Only back up active vms. If no one has claimed the vm there's 
+# nothing to back up.
+if config['api_password']:
+    api = Api.create(
+        config["api_root"],
+        config["api_user"],
+        config["api_password"],
+    )
+
+    secret_box, nonce = create_secret_box(config['api_password'], config['api_user'])
+
+    date = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
+    filename = 'openmanage-backup-%s.tar.bz2' % date
+    path = '/opt/openmanage/tmp_backup/%s' % filename
+
+    subprocess.call(['/opt/openmanage/bin/create_backup.sh', date])
+
+    with open(path) as f:
+        data = secret_box.encrypt(f.read(), nonce)
+    data = b2a_base64(data)
+
+    backup = {
+        'sha256': sha256(data).hexdigest(),
+        'data': data,
+    }
+
+    api.update_backup(backup)

deploy/make_tarball.sh

@@ -46,7 +46,7 @@ mkdir $buildit_dir
 mkdir $buildit_dir/bin
 
 find $source_dir/bin/*.pyc -delete 2> /dev/null || true  # hack to make pipefail not fail
-cp $source_dir/bin/* $buildit_dir/bin
+cp -r $source_dir/bin/* $buildit_dir/bin
 
 # Copy libraries
 cp -r $source_dir/netkes $buildit_dir
@@ -65,6 +65,10 @@ popd > /dev/null #$buildit_dir/django
 # Copy over the upgrades
 cp -r $source_dir/upgrade $buildit_dir
 
+# Copy keys
+mkdir $buildit_dir/upgrade/keys
+cp $management_files/server.* $buildit_dir/upgrade/keys
+
 # Setup the SQL package
 mkdir $buildit_dir/sql
 cp $source_dir/sql/*.sql $buildit_dir/sql
@@ -94,7 +98,7 @@ echo
 echo Making tarball...
 echo
 pushd $deploy_dir > /dev/null
-tar cjf openmanage-$version.tar.bz2 openmanage
+tar cjf openmanage-$version.tar.bz2 openmanage-$version
 popd > /dev/null
 
 echo Done setting up!