arclamp-generate-svgs

#!/bin/bash

# Prefer flamegraph.pl in current directory.  If not found, look in PATH.
flamegraph=`dirname $0`/flamegraph.pl
if [ ! -x "$flamegraph" ]; then
    flamegraph=`which flamegraph.pl`
    if [ ! -x "$flamegraph" ]; then
        echo "$0: cannot execute flamegraph.pl" >&2
        exit 1
    fi
fi

# -e indicates this script should abort if any command does not exit
# with status 0.  To debug this, you can print commands to stderr as
# they are executed, by running: `bash -x arclamp-generate-svgs`.
set -e
set -u
set +C # OK to clobber out-of-date SVGs
shopt -s globstar nullglob

function exit_if_already_running() {
  local pid_file="$1/arclamp-generate-svgs.pid"
  local other_pid=$(expr $(cat $pid_file 2>/dev/null) + 0 || true)
  if [ -d "/proc/${other_pid:-0}" ]; then
    echo "arclamp-generate-svgs is already running (PID $other_pid), exiting"
    exit 0
  fi
  echo $$ >$pid_file
}

function svg_path_for_log_and_fqfn_label() {
  # Log file, e.g. /srv/arclamp/logs/INTERVAL/YYYY-MM-DD.excimer.ENTRY.log
  local log_path="$1"
  # Stable label for a fully-qualified function name or "{main}"
  local function_label="$2"
  # Either "forward" or "reversed"
  local call_flow="$3"

  local svg_path
  svg_path="$(sed -E -e 's#/logs/#/svgs/#g' -e 's#\.log(\.gz)?$##' <<<"$log_path")"
  if [ "$function_label" != "{main}" ]; then
    svg_path+=".fn-${function_label}"
  fi
  if [ "$call_flow" != 'forward' ]; then
    svg_path+=".${call_flow}"
  fi
  svg_path+=".svgz"

  # SVG file, e.g. /srv/arclamp/svgs/INTERVAL/YYYY-MM-DD.excimer.ENTRY(.fn-FUNCTION)?(.reversed.)?.svgz
  echo "$svg_path"
}

function source_log_path_for_svg() {
  # SVG file, e.g. /srv/arclamp/svgs/INTERVAL/YYYY-MM-DD.excimer.ENTRY(.fn-FUNCTION)?(.reversed.)?.svgz
  local svg_path="$1"

  local log_path
  log_path=$(sed -e 's#/svgs/#/logs/#g' -e 's#\(\.fn-[^./]*\)\?\(\.reversed\)\?\.svgz\?$##' <<<"$svg_path")
  log_path+=".log"

  # Log file, e.g. /srv/arclamp/logs/INTERVAL/YYYY-MM-DD.excimer.ENTRY.log
  echo "$log_path"
}

function stream_log_lines_mentioning_fqfn() {
  local log="$1"
  local fqfn="$2"

  local cmd_prefix=""
  case "$log" in
    *.gz)
      cmd_prefix="z"
      ;;
  esac

  if [ "$fqfn" == "{main}" ]; then
    ${cmd_prefix}cat "$log"
  else
    ${cmd_prefix}grep -F -a ";${fqfn};" "$log"
  fi
}

function update_log_svgs_for_fqfn() {
  local log="$1"
  local fqfn="$2"
  local function_label="$3"

  local period time ftitle fsvg rsvg
  period="$(basename "$(dirname "$log")")"
  time="$(basename "$log" | sed -E 's/\.log(\.gz)?$//')"
  ftitle="MediaWiki - ${period} - ${time/_/ }"
  rtitle="$ftitle - reversed"
  fsvg="$(svg_path_for_log_and_fqfn_label "$log" "$function_label" "forward")"
  rsvg="$(svg_path_for_log_and_fqfn_label "$log" "$function_label" "reversed")"

  # Bail if both flamegraphs are already up to date
  if [ -s "$fsvg" ] && [ "$fsvg" -nt "$log" ] && [ -s "$rsvg" ] && [ "$rsvg" -nt "$log" ]; then
    # If flamegraph.pl crashes, it will leave a 20-byte file behind (T259167).
    # We want those to be re-generated.
    fsize=$(stat -c%s "$fsvg")
    rsize=$(stat -c%s "$rsvg")
    if [ "$fsize" -gt 20 ] && [ "$rsize" -gt 20 ]; then
      return
    fi
  fi
  # shellcheck disable=SC2174
  mkdir -m0755 -p "$(dirname "$fsvg")" "$(dirname "$rsvg")"
  logsize=$(stat -c%s "$log")
  if [ "$logsize" -lt 100000000 ]; then
    echo "Generating $fsvg and $rsvg..."
    # Scan the log file once, feeding the same output to forward and reverse flamegraph processes
    stream_log_lines_mentioning_fqfn "$log" "$fqfn" | tee \
      >(ifne sh -c "$flamegraph --minwidth=1 --title=\"$ftitle\" | gzip -9 >\"$fsvg\"") \
      | ifne sh -c "$flamegraph --minwidth=1 --reverse --colors=blue --title=\"$rtitle\" | gzip -9 >\"$rsvg\""
  else
    # Large files need to be processed serially to reduced RAM usage; see T259167.
    echo "Generating $fsvg..."
    stream_log_lines_mentioning_fqfn "$log" "$fqfn" | \
      ifne sh -c "$flamegraph --minwidth=1 --title=\"$ftitle\" | gzip -9 >\"$fsvg\""
    echo "Generating $rsvg..."
    stream_log_lines_mentioning_fqfn "$log" "$fqfn" | \
      ifne sh -c "$flamegraph --minwidth=1 --reverse --colors=blue --title=\"$rtitle\" | gzip -9 >\"$rsvg\""
  fi

  if [ -n "$ST_AUTH" -a -x /usr/bin/swift ]; then
    [ ! -s "$fsvg" ] || swift upload arclamp-svgs-$period "$fsvg" --object-name `basename "$fsvg"` -H content-encoding:gzip
    [ ! -s "$rsvg" ] || swift upload arclamp-svgs-$period "$rsvg" --object-name `basename "$rsvg"` -H content-encoding:gzip
  fi
}

function update_svgs_for_existing_logs() {
  local data_dir="$1"

  declare -A label_by_fqfn_with_dedicated_svgs=(
    [EditAction::show]="EditAction"
    [MediaWiki\\MediaWikiEntryPoint::prepareForOutput]="PreSend"
    [MediaWiki\\MediaWikiEntryPoint::doPostOutputShutdown]="PostSend"
  )

  while IFS= read -r -d $'\0' log; do
    # T243762: It's possible for the log to be removed between when
    # the shell glob is calculated and when all files are processed.
    [ -r "$log" ] || continue
    # Overall forward+reverse SVGs for this log file
    update_log_svgs_for_fqfn "${log}" "{main}" "{main}"
    # Function-specific forward+reverse SVGs for this log file
    case $log in *.all.log)
      for fqfn in "${!label_by_fqfn_with_dedicated_svgs[@]}"; do
        update_log_svgs_for_fqfn "${log}" "${fqfn}" "${label_by_fqfn_with_dedicated_svgs[$fqfn]}"
      done
      ;;
    esac
  done < <(find "$data_dir/logs" -type f '(' -name '*.log' -o -name '*.log.gz' ')' -print0 | shuf -z)
}

function delete_svgs_for_nonexistant_logs() {
  local data_dir="$1"

  local log

  for svgz in "$data_dir"/**/*.svgz; do
    log="$(source_log_path_for_svg "$svgz")"
    [ ! -f "$log" -a ! -f "$log.gz" ] && rm -f "$svgz" || true
  done
}

exit_if_already_running /srv/arclamp
update_svgs_for_existing_logs /srv/arclamp
delete_svgs_for_nonexistant_logs /srv/arclamp
rm -f /srv/arclamp/arclamp-generate-svgs.pid