manage

#!/bin/sh

##H Usage: manage ACTION [ARG] [SECURITY-STRING]
##H
##H Available actions:
##H   help        show this help
##H   version     get current version of the service
##H   status      show current service's status
##H   sysboot     start server from crond if not running
##H   restart     (re)start the service
##H   start       (re)start the service
##H   stop        stop the service
##H   cron ARG    run cronjob named ARG (dataframe|models|verify)
##H

if [ $(id -un)  = cmsweb ]; then
  echo "ERROR: please use another account" 1>&2
  exit 1
fi

echo_e=-e
case $(uname) in Darwin )
  md5sum() { md5 -r ${1+"$@"}; }
  echo_e=
  ;;
esac

ME=$(basename $(dirname $0))
TOP=$(cd $(dirname $0)/../../.. && pwd)
ROOT=$(cd $(dirname $0)/../.. && pwd)
LOGDIR=$TOP/logs/$ME
STATEDIR=$TOP/state/$ME
COLOR_OK="\\033[0;32m"
COLOR_WARN="\\033[0;31m"
COLOR_NORMAL="\\033[0;39m"

. $ROOT/apps/wmarchive/etc/profile.d/init.sh

export WMARCHIVE_ROOT
export WMA_STATIC_ROOT=$WMARCHIVE_ROOT/data
export WMARCHIVE_PORT=8247
export PYTHONUNBUFFERED=1
export JAVA_JDK_ROOT
export JAVA_HOME=$JAVA_JDK_ROOT
if [ -z "$JAVA_HOME" ]; then
export JAVA_HOME=/usr
fi

# kerberos
export WMAKEYTAB=$STATEDIR/wma.keytab
principal=`klist -k $WMAKEYTAB | tail -1 | awk '{print $2}'`
kinit $principal -k -t $WMAKEYTAB

# detect hadoop distribution
if [ -d /usr/hdp/hadoop ]; then
    export HADOOP_HOME=/usr/hdp/hadoop
fi

# Go based config
#export WMARCHIVE_GOCONFIG=$ROOT/config/$ME/wmarch_go.json
export WMARCHIVE_GOCONFIG=$ROOT/auth/$ME/wmarch_go.json

# GRID
export X509_USER_PROXY=$STATEDIR/proxy/proxy.cert
export X509_USER_CERT=$X509_USER_PROXY
export X509_USER_KEY=$X509_USER_PROXY
export WMARCHIVE_CONFIG=$ROOT/config/$ME/wmarch_config.py
export WMARCHIVE_ERROR_CODES=$WMA_STATIC_ROOT/wmexceptions.json
export WMARCHIVE_PERF_METRICS=$WMA_STATIC_ROOT/maps/metrics.json
export MONGO_URI="mongodb://localhost:8230"
export AVRO_STORAGE=$STATEDIR/avro/data
export AVRO_STORAGE_MIGRATE=$STATEDIR/avro/migrate
# area on HDFS where we'll migrate the data
export HDFS_MIGRATE=/cms/wmarchive/avro
export AVRO_SCHEMA=$STATEDIR/avro/schemas/current.avsc
export AVRO_SCHEMA_HDFS=`cat $WMARCHIVE_CONFIG | grep ^data.long_storage_uri | awk '{split($0,a,"="); print a[2]}' | sed -e 's/^[[:space:]]*//' -e "s,',,g"`
# we'll clean-up data a one day old if their storage type is avroio
export CLEANUP_TSTAMP=1d
export CLEANUP_STYPE="avroio"

cd $STATEDIR
host=`hostname -s`

# Start service conditionally on crond restart.
sysboot()
{
  dostart=false
  if [ $(pgrep -u $(id -u) -f "[/]wmarch" | wc -l) = 0 ]; then
      dostart=true
  fi
  $dostart && start
}

createindex()
{
  if [ ! -f $WMARCHIVE_GOCONFIG ]; then
      mport=`echo $MONGO_URI | awk -F: '{print $3}'`
      mongo --port $mport < $WMARCHIVE_ROOT/data/js/create_wma_indexes.js
  fi
}

start()
{
  # Go based service
  if [ -f $WMARCHIVE_GOCONFIG ]; then
      echo "starting $ME with $WMARCHIVE_GOCONFIG"
      $WMARCHIVE_ROOT/bin/wmarchive -config $WMARCHIVE_GOCONFIG \
          </dev/null 2>&1 | rotatelogs $LOGDIR/wmarchive-%Y%m%d-`hostname -s`.log 86400 >/dev/null 2>&1 &
  else
      echo "starting $ME"
      # start service
      $WMARCHIVE_ROOT/bin/wmc-httpd -r -d $LOGDIR $WMARCHIVE_CONFIG &
      $WMARCHIVE_ROOT/bin/mongo2avro \
          --mongo=$MONGO_URI --dbname=fwjr --schema=$AVRO_SCHEMA --thr=240 \
          --odir=$AVRO_STORAGE/fwjr --mdir=$AVRO_STORAGE_MIGRATE/fwjr \
          --tstamp=$CLEANUP_TSTAMP --stype=$CLEANUP_STYPE --dtype=fwjr \
          </dev/null 2>&1 | rotatelogs $LOGDIR/mongo2avro-fwjr-%Y%m%d-`hostname -s`.log 86400 >/dev/null 2>&1 &
      $WMARCHIVE_ROOT/bin/mongo2avro \
          --mongo=$MONGO_URI --dbname=crab --schema=$AVRO_SCHEMA --thr=240 \
          --odir=$AVRO_STORAGE/crab --mdir=$AVRO_STORAGE_MIGRATE/crab \
          --tstamp=$CLEANUP_TSTAMP --stype=$CLEANUP_STYPE --dtype=crab \
          </dev/null 2>&1 | rotatelogs $LOGDIR/mongo2avro-crab-%Y%m%d-`hostname -s`.log 86400 >/dev/null 2>&1 &
  fi
}

# Stop the service.
stop()
{
  if [ -f $WMARCHIVE_GOCONFIG ]; then
      echo "stopping $ME with $WMARCHIVE_GOCONFIG"
      for pid in $(pgrep -u $(id -u) -f "[/]wmarch_go.json"); do
          kill -9 $pid
      done
  else
      echo "stopping $ME"
      for pid in $(pgrep -u $(id -u) -f "[/]wmarch_config.py"); do
          kill -9 $pid
      done
      for pid in $(pgrep -u $(id -u) -f "[/]mongo2avro"); do
          kill -9 $pid
      done
  fi
}

# Check if the server is running.
status()
{
  if [ -f $WMARCHIVE_GOCONFIG ]; then
      local pid=$(pgrep -u $(id -u) -f "[/]wmarch_go.json" | sort -n)
      if [ X"$pid" = X ]; then
        echo $echo_e "$ME $pat is ${COLOR_WARN}NOT RUNNING${COLOR_NORMAL}."
      else
        echo $echo_e "$ME $pat is ${COLOR_OK}RUNNING${COLOR_NORMAL}, PID" $pid
      fi
  else
      local pid=$(pgrep -u $(id -u) -f "[/]wmarch_config.py" | sort -n)
      if [ X"$pid" = X ]; then
        echo $echo_e "$ME $pat is ${COLOR_WARN}NOT RUNNING${COLOR_NORMAL}."
      else
        echo $echo_e "$ME $pat is ${COLOR_OK}RUNNING${COLOR_NORMAL}, PID" $pid
      fi
  fi
}

# Runs actions to be done through cronjobs
cron()
{
  if [ -f $WMARCHIVE_GOCONFIG ]; then
      return
  fi
  local action=$1
  case $action in
    migrate2hdfs )
      $WMARCHIVE_ROOT/bin/migrate2hdfs \
          --idir=$AVRO_STORAGE_MIGRATE/fwjr --odir=$HDFS_MIGRATE/fwjr --check --remove \
          --verbose &> $LOGDIR/migrate2hdfs-fwjr.log
      $WMARCHIVE_ROOT/bin/migrate2hdfs \
          --idir=$AVRO_STORAGE_MIGRATE/crab --odir=$HDFS_MIGRATE/crab --check --remove \
          --verbose &> $LOGDIR/migrate2hdfs-crab.log
      ;;
    aggregate-fwjr )
      # run hourly aggregation, results will appear in aggregated.hour MongoDB collection
      day=0 # use aggregation of the current day
      WMARCHIVE_PERF_DB="aggregated" WMARCHIVE_PERF_COLL="hour" WMARCHIVE_AMQ_BROKER=$STATEDIR/wma_broker.json \
      $WMARCHIVE_ROOT/bin/aggregate_performance 0 --precision=hour \
          --schema $AVRO_SCHEMA_HDFS --hdir=$HDFS_MIGRATE/fwjr \
          --use_myspark &> $LOGDIR/agg-hour-fwjr.log
      # run daily aggregation, results will appear in aggregated.day MongoDB collection
      WMARCHIVE_PERF_DB="aggregated" WMARCHIVE_PERF_COLL="day" \
      $WMARCHIVE_ROOT/bin/aggregate_performance 0 --precision=day \
          --schema $AVRO_SCHEMA_HDFS --hdir=$HDFS_MIGRATE/fwjr \
          --use_myspark &> $LOGDIR/agg-day-fwjr.log
      ;;
    aggregate-crab )
      # CRAB aggregation
      # run hourly aggregation, results will appear in aggregated.hour MongoDB collection
      day=0 # use aggregation of the current day
      day=1 # use aggregation of the previous day since crab injection is slow
      WMARCHIVE_PERF_DB="crab" WMARCHIVE_PERF_COLL="hour" WMARCHIVE_AMQ_BROKER=$STATEDIR/wma_broker.json \
      $WMARCHIVE_ROOT/bin/aggregate_performance $day --precision=hour \
          --schema $AVRO_SCHEMA_HDFS --hdir=$HDFS_MIGRATE/crab \
          --use_myspark &> $LOGDIR/agg-hour-crab.log
      # run daily aggregation, results will appear in aggregated.day MongoDB collection
      WMARCHIVE_PERF_DB="crab" WMARCHIVE_PERF_COLL="day" \
      $WMARCHIVE_ROOT/bin/aggregate_performance $day --precision=day \
          --schema $AVRO_SCHEMA_HDFS --hdir=$HDFS_MIGRATE/crab \
          --use_myspark &> $LOGDIR/agg-day-crab.log
      ;;
    cleanup )
      $WMARCHIVE_ROOT/bin/clean_performance --interval=3m &> $LOGDIR/cleanup.log
      ;;
    * )
      echo "You must specify the cron action to run: mongo2avro|mongocleanup "
      exit 1
      ;;
  esac
}

# Verify the security string.
check()
{
  CHECK=$(echo "$1" | md5sum | awk '{print $1}')
  if [ $CHECK != 94e261a5a70785552d34a65068819993 ]; then
    echo "$0: cannot complete operation, please check documentation." 1>&2
    exit 2;
  fi
}

# Main routine, perform action requested on command line.
case ${1:-status} in
  sysboot )
    sysboot
    ;;

  start | restart )
    check "$2"
    stop
    start
    createindex
    ;;

  status )
    status
    ;;

  stop )
    check "$2"
    stop
    ;;

  cron )
    check "$3"
    cron $2
    ;;

  help )
    perl -ne '/^##H/ && do { s/^##H ?//; print }' < $0
    ;;

  version )
    echo "$WMARCHIVE_VERSION"
    ;;

  * )
    echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
    exit 1
    ;;
esac