#!/usr/bin/env bash
#
# JbTecWiz Support Centre -- generated fix script
#
# Fault : A process is killed by the OOM killer
# Fix   : Confirm the kill and find what consumed the memory
# Source: https://jbtecwiz.com/support/lnx-oom-killer
#
# Run as     : Shell
# Expect     : 30 minutes
# Risk       : low
# Reversible : yes
#
# WHEN THIS IS THE RIGHT FIX
#   The process runs directly on the host.
#
# HOW TO UNDO IT
#   swapoff /swapfile and remove the fstab line; systemctl revert
#   myservice.service.
#
# Walks the fix one step at a time and asks before each. Steps with no
# command are yours to do -- it prints those and waits. DRYRUN=1 prints
# without executing; UNATTENDED=1 does not ask.
#
# --------------------------------------------------------------------
# NO WARRANTY - USE AT YOUR OWN RISK
#
# This script is provided by JbTecWiz as-is and with no warranty of any
# kind, express or implied. You run it entirely at your own risk.
#
# JbTecWiz accepts no liability for any loss or damage arising from its
# use, including but not limited to data loss, downtime, or configuration
# changes that turn out to be wrong for your system.
#
# You are responsible for reading this script before running it, for
# satisfying yourself that it suits the machine in front of you, and for
# having a working backup first. Some steps cannot be undone.
# --------------------------------------------------------------------

set -uo pipefail
DRYRUN="${DRYRUN:-0}"
UNATTENDED="${UNATTENDED:-0}"
failed=0

rule() { printf "\n%s\n" "$(printf '-%.0s' $(seq 1 70))"; if [ $# -gt 0 ]; then echo "$1"; fi; }
prose() { echo "$1" | fold -s -w 74 | sed "s/^/  /"; }

# Returns 0 when the caller should run the command, 1 when it should not.
# A manual step always returns 1 -- there is nothing for the caller to run.
step() {  # step <n> <do> <why> <manual|cmd> [command lines...]
  local n="$1" dotext="$2" why="$3" mode="$4"; shift 4
  rule "  Step $n of 6"
  prose "$dotext"
  if [ -n "$why" ]; then echo; prose "$why"; fi
  if [ "$mode" = "manual" ]; then
    echo; echo "  -> Do this yourself, then press Enter to carry on."
    if [ "$UNATTENDED" = "0" ] && [ "$DRYRUN" = "0" ]; then read -r _; fi
    return 1
  fi
  echo; printf "    %s\n" "$@"; echo
  if [ "$DRYRUN" = "1" ]; then echo "  (dry run -- not executed)"; return 1; fi
  if [ "$UNATTENDED" = "0" ]; then
    read -r -p "  Run this step? [Y]es / [S]kip / [Q]uit " a
    case "$a" in
      [Qq]*) echo "  Stopped at your request."; exit 0 ;;
      [Ss]*) echo "  Skipped."; return 1 ;;
    esac
  fi
  return 0
}

rule
echo "  A process is killed by the OOM killer"
echo "  Confirm the kill and find what consumed the memory"
echo
echo "  Risk: low   Reversible   30 minutes"
echo
prose 'No warranty. Use at your own risk - JbTecWiz accepts no liability. Read it before you run it, and have a backup.'
rule
echo
if [ "$UNATTENDED" = "0" ] && [ "$DRYRUN" = "0" ]; then
  read -r -p "  Ready? [y/N] " go
  case "$go" in [Yy]*) ;; *) echo "  Nothing was changed."; exit 0;; esac
fi

if step 1 'Confirm it was the OOM killer and see what it chose.' 'The kernel prints a table of every process and its RSS at the moment of the kill, which is the best evidence you will get of what was actually using the memory.' cmd 'sudo dmesg -T | grep -i -A3 '\''out of memory'\''' 'sudo journalctl -k -g '\''Out of memory'\'' --no-pager'; then
  sudo dmesg -T | grep -i -A3 'out of memory'
  sudo journalctl -k -g 'Out of memory' --no-pager
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 1 failed. The rest of the fix may depend on it." >&2
  fi
fi

if step 2 'Look at current usage including how much is genuinely available.' 'The '\''available'\'' column is the one that matters -- '\''free'\'' looks alarmingly low on a healthy Linux box because the kernel uses spare memory for cache.' cmd 'free -h' 'ps aux --sort=-%mem | head -15'; then
  free -h
  ps aux --sort=-%mem | head -15
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 2 failed. The rest of the fix may depend on it." >&2
  fi
fi

if step 3 'Check whether there is any swap at all.' '' cmd 'swapon --show' 'cat /proc/sys/vm/swappiness'; then
  swapon --show
  cat /proc/sys/vm/swappiness
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 3 failed. The rest of the fix may depend on it." >&2
  fi
fi

if step 4 'Add swap if there is none. It does not prevent OOM but it turns a hard kill into slow degradation you can react to.' '' cmd 'sudo fallocate -l 4G /swapfile' 'sudo chmod 600 /swapfile' 'sudo mkswap /swapfile' 'sudo swapon /swapfile' 'echo '\''/swapfile none swap sw 0 0'\'' | sudo tee -a /etc/fstab'; then
  sudo fallocate -l 4G /swapfile
  sudo chmod 600 /swapfile
  sudo mkswap /swapfile
  sudo swapon /swapfile
  echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 4 failed. The rest of the fix may depend on it." >&2
  fi
fi

if step 5 'Protect the important process by lowering its OOM score, so something else is chosen instead.' 'Add OOMScoreAdjust=-500 under [Service]. This is a trade, not a fix -- something still gets killed.' cmd 'sudo systemctl edit myservice.service'; then
  sudo systemctl edit myservice.service
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 5 failed. The rest of the fix may depend on it." >&2
  fi
fi

step 6 'Deal with the real cause: fix the leak, cap the application'\''s own memory settings, or add RAM.' '' manual || true

rule "  Confirm it worked"
prose 'The workload runs without further OOM events.'
if [ "$DRYRUN" = "0" ]; then
  sudo journalctl -k -g 'Out of memory' --since '1 hour ago' --no-pager
fi

rule
if [ "$failed" -gt 0 ]; then
  echo "  Finished with $failed failed step(s)."
  echo "  Read the full write-up at https://jbtecwiz.com/support/lnx-oom-killer"
else
  echo "  Finished."
fi
echo
prose 'To undo: swapoff /swapfile and remove the fstab line; systemctl revert myservice.service.'
rule