#!/usr/bin/env bash
#
# JbTecWiz Support Centre -- generated fix script
#
# Fault : "No space left on device" but df shows free space
# Fix   : Release space held by a deleted-but-open file
# Source: https://jbtecwiz.com/support/lnx-enospc-inodes
#
# Run as     : Shell as root
# Expect     : 15 minutes
# Risk       : medium
# Reversible : yes
#
# WHEN THIS IS THE RIGHT FIX
#   Inodes are fine but space is not being returned. Classic after someone
#   deletes a huge log that a running daemon still has open.
#
# HOW TO UNDO IT
#   Nothing to undo once the descriptor is closed; the data is
#   unrecoverable either way since the file was already unlinked.
#
# Walks the fix one step at a time and asks before each. Steps with no
# command are yours to do -- it prints those and waits. DRYRUN=1 prints
# without executing; UNATTENDED=1 does not ask.
#
# --------------------------------------------------------------------
# NO WARRANTY - USE AT YOUR OWN RISK
#
# This script is provided by JbTecWiz as-is and with no warranty of any
# kind, express or implied. You run it entirely at your own risk.
#
# JbTecWiz accepts no liability for any loss or damage arising from its
# use, including but not limited to data loss, downtime, or configuration
# changes that turn out to be wrong for your system.
#
# You are responsible for reading this script before running it, for
# satisfying yourself that it suits the machine in front of you, and for
# having a working backup first. Some steps cannot be undone.
# --------------------------------------------------------------------

set -uo pipefail
DRYRUN="${DRYRUN:-0}"
UNATTENDED="${UNATTENDED:-0}"
failed=0

if [ "$(id -u)" -ne 0 ]; then
  echo "  This fix is documented as needing root. Re-run with sudo." >&2
  exit 3
fi

rule() { printf "\n%s\n" "$(printf '-%.0s' $(seq 1 70))"; if [ $# -gt 0 ]; then echo "$1"; fi; }
prose() { echo "$1" | fold -s -w 74 | sed "s/^/  /"; }

# Returns 0 when the caller should run the command, 1 when it should not.
# A manual step always returns 1 -- there is nothing for the caller to run.
step() {  # step <n> <do> <why> <manual|cmd> [command lines...]
  local n="$1" dotext="$2" why="$3" mode="$4"; shift 4
  rule "  Step $n of 6"
  prose "$dotext"
  if [ -n "$why" ]; then echo; prose "$why"; fi
  if [ "$mode" = "manual" ]; then
    echo; echo "  -> Do this yourself, then press Enter to carry on."
    if [ "$UNATTENDED" = "0" ] && [ "$DRYRUN" = "0" ]; then read -r _; fi
    return 1
  fi
  echo; printf "    %s\n" "$@"; echo
  if [ "$DRYRUN" = "1" ]; then echo "  (dry run -- not executed)"; return 1; fi
  if [ "$UNATTENDED" = "0" ]; then
    read -r -p "  Run this step? [Y]es / [S]kip / [Q]uit " a
    case "$a" in
      [Qq]*) echo "  Stopped at your request."; exit 0 ;;
      [Ss]*) echo "  Skipped."; return 1 ;;
    esac
  fi
  return 0
}

rule
echo "  "No space left on device" but df shows free space"
echo "  Release space held by a deleted-but-open file"
echo
echo "  Risk: medium   Reversible   15 minutes"
echo
prose 'No warranty. Use at your own risk - JbTecWiz accepts no liability. Read it before you run it, and have a backup.'
rule
echo
if [ "$UNATTENDED" = "0" ] && [ "$DRYRUN" = "0" ]; then
  read -r -p "  Ready? [y/N] " go
  case "$go" in [Yy]*) ;; *) echo "  Nothing was changed."; exit 0;; esac
fi

if step 1 'List deleted files still held open, largest first.' '+L1 selects files with a link count below 1 -- that is exactly the set that has been unlinked but not closed.' cmd 'sudo lsof -nP +L1 | sort -k7 -rn | head -20'; then
  sudo lsof -nP +L1 | sort -k7 -rn | head -20
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 1 failed. The rest of the fix may depend on it." >&2
  fi
fi

step 2 'Note the PID and the size. The space comes back when that file descriptor closes.' '' manual || true

if step 3 'Preferred: reload the service so it reopens its logs.' '' cmd 'sudo systemctl reload nginx'; then
  sudo systemctl reload nginx
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 3 failed. The rest of the fix may depend on it." >&2
  fi
fi

if step 4 'If reload is not supported, restart it.' '' cmd 'sudo systemctl restart nginx'; then
  sudo systemctl restart nginx
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 4 failed. The rest of the fix may depend on it." >&2
  fi
fi

if step 5 'As a last resort -- and only if you cannot restart the service -- truncate the descriptor in place. Replace PID and FD with the values from lsof.' 'This frees the blocks without stopping the process. It discards the file contents, so do not do it to something you still need to read.' cmd 'sudo truncate -s 0 /proc/12345/fd/4'; then
  sudo truncate -s 0 /proc/12345/fd/4
  if [ $? -ne 0 ]; then
    failed=$((failed+1))
    echo "  Step 5 failed. The rest of the fix may depend on it." >&2
  fi
fi

step 6 'Set up logrotate with copytruncate for whatever produced the file, so it does not recur.' '' manual || true

rule "  Confirm it worked"
prose 'df shows the space returned.'
if [ "$DRYRUN" = "0" ]; then
  df -h
fi

rule
if [ "$failed" -gt 0 ]; then
  echo "  Finished with $failed failed step(s)."
  echo "  Read the full write-up at https://jbtecwiz.com/support/lnx-enospc-inodes"
else
  echo "  Finished."
fi
echo
prose 'To undo: Nothing to undo once the descriptor is closed; the data is unrecoverable either way since the file was already unlinked.'
rule