#!/usr/bin/env bash # # JbTecWiz Support Centre -- generated fix script # # Fault : nginx 502 Bad Gateway # Fix : Deal with an upstream that is too slow or crashing # Source: https://jbtecwiz.com/support/lnx-web-502 # # Run as : Root shell # Expect : 40 minutes # Risk : medium # Reversible : yes # # WHEN THIS IS THE RIGHT FIX # The log says upstream timed out, or prematurely closed the connection. # # HOW TO UNDO IT # Restore the previous timeout values from the backup copies of the # configuration. # # Walks the fix one step at a time and asks before each. Steps with no # command are yours to do -- it prints those and waits. DRYRUN=1 prints # without executing; UNATTENDED=1 does not ask. # # -------------------------------------------------------------------- # NO WARRANTY - USE AT YOUR OWN RISK # # This script is provided by JbTecWiz as-is and with no warranty of any # kind, express or implied. You run it entirely at your own risk. # # JbTecWiz accepts no liability for any loss or damage arising from its # use, including but not limited to data loss, downtime, or configuration # changes that turn out to be wrong for your system. # # You are responsible for reading this script before running it, for # satisfying yourself that it suits the machine in front of you, and for # having a working backup first. Some steps cannot be undone. # -------------------------------------------------------------------- set -uo pipefail DRYRUN="${DRYRUN:-0}" UNATTENDED="${UNATTENDED:-0}" failed=0 if [ "$(id -u)" -ne 0 ]; then echo " This fix is documented as needing root. Re-run with sudo." >&2 exit 3 fi rule() { printf "\n%s\n" "$(printf '-%.0s' $(seq 1 70))"; if [ $# -gt 0 ]; then echo "$1"; fi; } prose() { echo "$1" | fold -s -w 74 | sed "s/^/ /"; } # Returns 0 when the caller should run the command, 1 when it should not. # A manual step always returns 1 -- there is nothing for the caller to run. step() { # step [command lines...] local n="$1" dotext="$2" why="$3" mode="$4"; shift 4 rule " Step $n of 5" prose "$dotext" if [ -n "$why" ]; then echo; prose "$why"; fi if [ "$mode" = "manual" ]; then echo; echo " -> Do this yourself, then press Enter to carry on." if [ "$UNATTENDED" = "0" ] && [ "$DRYRUN" = "0" ]; then read -r _; fi return 1 fi echo; printf " %s\n" "$@"; echo if [ "$DRYRUN" = "1" ]; then echo " (dry run -- not executed)"; return 1; fi if [ "$UNATTENDED" = "0" ]; then read -r -p " Run this step? [Y]es / [S]kip / [Q]uit " a case "$a" in [Qq]*) echo " Stopped at your request."; exit 0 ;; [Ss]*) echo " Skipped."; return 1 ;; esac fi return 0 } rule echo " nginx 502 Bad Gateway" echo " Deal with an upstream that is too slow or crashing" echo echo " Risk: medium Reversible 40 minutes" echo prose 'No warranty. Use at your own risk - JbTecWiz accepts no liability. Read it before you run it, and have a backup.' rule echo if [ "$UNATTENDED" = "0" ] && [ "$DRYRUN" = "0" ]; then read -r -p " Ready? [y/N] " go case "$go" in [Yy]*) ;; *) echo " Nothing was changed."; exit 0;; esac fi if step 1 'Find out whether it is slow or dying. A premature close means the worker exited mid-request.' 'A worker killed by the OOM killer produces exactly the same 502 as a slow query, and raising the timeout for it makes matters worse. One command separates them.' cmd 'sudo journalctl -u php8.2-fpm --since '\''1 hour ago'\'' | grep -iE '\''exited|signal|oom|segfault'\''' 'dmesg -T | grep -i '\''killed process'\'''; then sudo journalctl -u php8.2-fpm --since '1 hour ago' | grep -iE 'exited|signal|oom|segfault' dmesg -T | grep -i 'killed process' if [ $? -ne 0 ]; then failed=$((failed+1)) echo " Step 1 failed. The rest of the fix may depend on it." >&2 fi fi if step 2 'If it is being killed for memory, look at the pool'\''s memory limit and worker count rather than the timeout.' '' cmd 'grep -E '\''^pm|memory_limit'\'' /etc/php/8.2/fpm/pool.d/www.conf /etc/php/8.2/fpm/php.ini'; then grep -E '^pm|memory_limit' /etc/php/8.2/fpm/pool.d/www.conf /etc/php/8.2/fpm/php.ini if [ $? -ne 0 ]; then failed=$((failed+1)) echo " Step 2 failed. The rest of the fix may depend on it." >&2 fi fi if step 3 'If it is genuinely slow, raise the timeouts on both sides -- they must match or the shorter one wins.' '' cmd 'grep -rn '\''fastcgi_read_timeout\|proxy_read_timeout'\'' /etc/nginx/' 'grep -n '\''request_terminate_timeout\|max_execution_time'\'' /etc/php/8.2/fpm/pool.d/www.conf /etc/php/8.2/fpm/php.ini'; then grep -rn 'fastcgi_read_timeout\|proxy_read_timeout' /etc/nginx/ grep -n 'request_terminate_timeout\|max_execution_time' /etc/php/8.2/fpm/pool.d/www.conf /etc/php/8.2/fpm/php.ini if [ $? -ne 0 ]; then failed=$((failed+1)) echo " Step 3 failed. The rest of the fix may depend on it." >&2 fi fi if step 4 'Set them consistently, and treat a long-running request as something to fix rather than to accommodate.' '' cmd 'sudo sed -i '\''s/^;*request_terminate_timeout.*/request_terminate_timeout = 120/'\'' /etc/php/8.2/fpm/pool.d/www.conf' 'sudo systemctl restart php8.2-fpm'; then sudo sed -i 's/^;*request_terminate_timeout.*/request_terminate_timeout = 120/' /etc/php/8.2/fpm/pool.d/www.conf sudo systemctl restart php8.2-fpm if [ $? -ne 0 ]; then failed=$((failed+1)) echo " Step 4 failed. The rest of the fix may depend on it." >&2 fi fi if step 5 'Enable the slow log so the actual slow function is recorded.' '' cmd 'sudo sed -i '\''s|^;*slowlog.*|slowlog = /var/log/php-fpm-slow.log|; s/^;*request_slowlog_timeout.*/request_slowlog_timeout = 5s/'\'' /etc/php/8.2/fpm/pool.d/www.conf' 'sudo systemctl restart php8.2-fpm'; then sudo sed -i 's|^;*slowlog.*|slowlog = /var/log/php-fpm-slow.log|; s/^;*request_slowlog_timeout.*/request_slowlog_timeout = 5s/' /etc/php/8.2/fpm/pool.d/www.conf sudo systemctl restart php8.2-fpm if [ $? -ne 0 ]; then failed=$((failed+1)) echo " Step 5 failed. The rest of the fix may depend on it." >&2 fi fi rule " Confirm it worked" prose 'Requests complete and the slow log names anything still over the threshold.' if [ "$DRYRUN" = "0" ]; then sudo tail -20 /var/log/php-fpm-slow.log fi rule if [ "$failed" -gt 0 ]; then echo " Finished with $failed failed step(s)." echo " Read the full write-up at https://jbtecwiz.com/support/lnx-web-502" else echo " Finished." fi echo prose 'To undo: Restore the previous timeout values from the backup copies of the configuration.' rule