932 lines
26 KiB
Bash
932 lines
26 KiB
Bash
#!/bin/bash
|
|
|
|
diagLabel="[Diag]"
|
|
diagExcludeVhostsRegex='localhost|Example'
|
|
|
|
# Writes a section header to the diag log file and prints it to the info log.
|
|
# $1 (title): section title text.
|
|
function diagLogSection() {
|
|
local title="$1"
|
|
local padding
|
|
local logPath
|
|
|
|
logInfo "$diagLabel $title"
|
|
|
|
padding=$((72 - ${#title}))
|
|
if (( padding > 0 )); then
|
|
title="${title} $(printf '.%.s' $(seq 1 "$padding"))"
|
|
fi
|
|
|
|
logPath=$(dirname "$diagFile")
|
|
[[ -d "$logPath" ]] || mkdir -p "$logPath"
|
|
|
|
printf "[ %s ]\n" "$title" >> "$diagFile"
|
|
}
|
|
|
|
# Runs a command and appends its output (stdout and stderr) to the diag file.
|
|
# $@ (...): command and arguments to execute.
|
|
function diagCmd() {
|
|
echo "+ $*" >> "$diagFile"
|
|
"$@" >> "$diagFile" 2>&1 || true
|
|
echo >> "$diagFile"
|
|
}
|
|
|
|
# Runs a shell command via bash -lc and appends its output to the diag file; errors ignored.
|
|
# $@ (...): shell command string to execute.
|
|
function diagShTry() {
|
|
echo "+ $*" >> "$diagFile"
|
|
bash -lc "$*" >> "$diagFile" 2>&1 || true
|
|
echo >> "$diagFile"
|
|
}
|
|
|
|
# Runs a kubectl command via k3sRun and appends its output to the diag file; errors ignored.
|
|
# $@ (...): kubectl arguments to pass to k3sRun.
|
|
function diagK3sTry() {
|
|
echo "+ k3sRun $*" >> "$diagFile"
|
|
k3sRun "$@" >> "$diagFile" 2>&1 || true
|
|
echo >> "$diagFile"
|
|
}
|
|
|
|
# Converts $diagSince (e.g. 4h, 30m, 2d) to a human-readable date argument for the date command.
|
|
function diagSinceDateArg() {
|
|
local s="${diagSince:-4h}"
|
|
|
|
case "$s" in
|
|
*m)
|
|
printf '%s minutes ago' "${s%m}"
|
|
;;
|
|
*h)
|
|
printf '%s hours ago' "${s%h}"
|
|
;;
|
|
*d)
|
|
printf '%s days ago' "${s%d}"
|
|
;;
|
|
*)
|
|
printf '4 hours ago'
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Returns the Unix epoch timestamp corresponding to the $diagSince cutoff time.
|
|
function diagCutoffEpoch() {
|
|
date -d "$(diagSinceDateArg)" +%s 2>/dev/null || date -d "4 hours ago" +%s
|
|
}
|
|
|
|
# Returns the HH:MM:SS start time for sar based on $diagSince; falls back to 00:00:00 if in the past.
|
|
function diagSarStartTime() {
|
|
local t now
|
|
|
|
t="$(date -d "$(diagSinceDateArg)" +%H:%M:%S 2>/dev/null || echo "00:00:00")"
|
|
now="$(date +%H:%M:%S)"
|
|
|
|
if [[ "$t" > "$now" ]]; then
|
|
printf "00:00:00"
|
|
else
|
|
printf "%s" "$t"
|
|
fi
|
|
}
|
|
|
|
# Writes diagnostic metadata (timestamp, since, hostname, file path) to the diag file.
|
|
function diagMeta() {
|
|
diagLogSection "Diagnostic metadata"
|
|
|
|
{
|
|
echo "started_at: $(date -Iseconds)"
|
|
echo "since: $diagSince"
|
|
echo "hostname: [SERVER]"
|
|
echo "file: $diagFile"
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Replaces hostnames, user paths, and UIDs in the diag file with redacted placeholders.
|
|
function diagSanitizeReport() {
|
|
local hostLower
|
|
local hostFqdn
|
|
local hostFqdnLower
|
|
local sanitizeHosts
|
|
|
|
[[ -f "$diagFile" ]] || return 0
|
|
|
|
hostLower="$(printf '%s' "$hostName" | tr '[:upper:]' '[:lower:]')"
|
|
hostFqdn="$(hostname -f 2>/dev/null || true)"
|
|
hostFqdnLower="$(printf '%s' "$hostFqdn" | tr '[:upper:]' '[:lower:]')"
|
|
|
|
sanitizeHosts="$(
|
|
printf '%s\n' "$hostName" "$hostLower" "$hostFqdn" "$hostFqdnLower" | awk 'NF && !seen[$0]++'
|
|
)"
|
|
|
|
SANITIZE_HOSTS="$sanitizeHosts" perl -0pi -e '
|
|
my $hosts = $ENV{SANITIZE_HOSTS} // "";
|
|
for my $h (split /\n/, $hosts) {
|
|
next unless defined $h && length $h;
|
|
my $q = quotemeta($h);
|
|
s/$q/[SERVER]/g;
|
|
}
|
|
|
|
s#(/media/)[^/\s]+#$1[USER]#g;
|
|
s#(/run/user/)[0-9]+#$1[UID]#g;
|
|
' "$diagFile"
|
|
}
|
|
|
|
# Filters stdin lines, excluding those matching reserved vhost names ($diagExcludeVhostsRegex).
|
|
function diagGrepExcludeReserved() {
|
|
local regex="${diagExcludeVhostsRegex:-}"
|
|
|
|
if [[ -n "$regex" ]]; then
|
|
grep -Eiv "$regex"
|
|
else
|
|
cat
|
|
fi
|
|
}
|
|
|
|
# Appends filtered disk usage (df -h, excluding overlay/tmpfs/pod mounts) to the diag file.
|
|
function diagDfUsage() {
|
|
{
|
|
echo "+ df -h filtered"
|
|
df -h \
|
|
-x tmpfs \
|
|
-x devtmpfs \
|
|
-x overlay \
|
|
-x squashfs \
|
|
-x efivarfs \
|
|
2>/dev/null | awk '
|
|
NR == 1 { print; next }
|
|
$6 ~ "^/run/k3s/" { next }
|
|
$6 ~ "^/var/lib/kubelet/pods/" { next }
|
|
$6 ~ "^/run/user/" { next }
|
|
$6 ~ "^/media/" { next }
|
|
{ print }
|
|
'
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Appends filtered inode usage (df -i, excluding overlay/tmpfs/pod mounts) to the diag file.
|
|
function diagDfInodes() {
|
|
{
|
|
echo "+ df -i filtered"
|
|
df -i \
|
|
-x tmpfs \
|
|
-x devtmpfs \
|
|
-x overlay \
|
|
-x squashfs \
|
|
-x efivarfs \
|
|
2>/dev/null | awk '
|
|
NR == 1 { print; next }
|
|
$6 ~ "^/run/k3s/" { next }
|
|
$6 ~ "^/var/lib/kubelet/pods/" { next }
|
|
$6 ~ "^/run/user/" { next }
|
|
$6 ~ "^/media/" { next }
|
|
{ print }
|
|
'
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Appends iostat extended stats (1s interval, 5 samples, loop/sr devices excluded) to the diag file.
|
|
function diagIostat() {
|
|
{
|
|
echo "+ iostat -x -z 1 5 | filter loop/sr devices"
|
|
|
|
{
|
|
iostat -x -z 1 5 2>/dev/null || iostat -x 1 5 2>/dev/null
|
|
} | awk '
|
|
/^loop[0-9]+/ { next }
|
|
/^sr[0-9]+/ { next }
|
|
{ print }
|
|
'
|
|
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Runs a sar command and appends its output to the diag file; skips gracefully if data is unavailable.
|
|
# $1 (title): section title for the diag log.
|
|
# $2 (cmd): sar command string to execute.
|
|
function diagSarTry() {
|
|
local title="$1"
|
|
local cmd="$2"
|
|
local saFile
|
|
|
|
saFile="/var/log/sysstat/sa$(date +%d)"
|
|
|
|
diagLogSection "$title"
|
|
|
|
{
|
|
echo "+ $cmd"
|
|
|
|
if [[ ! -r "$saFile" ]]; then
|
|
echo "sar_data_available: no"
|
|
echo "sar_file: $saFile"
|
|
echo "hint: enable sysstat data collection if SAR trends are needed"
|
|
echo
|
|
return
|
|
fi
|
|
|
|
bash -lc "$cmd" || true
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Appends a filtered k3s node summary (conditions, capacity, allocated resources) to the diag file.
|
|
function diagK3sNodeSummary() {
|
|
{
|
|
echo "+ k3sRun describe node filtered"
|
|
|
|
echo
|
|
echo "Conditions:"
|
|
k3sRun describe node 2>/dev/null | awk '
|
|
/^Conditions:/ { p=1; next }
|
|
/^Addresses:/ { p=0 }
|
|
p {
|
|
if ($1 == "Type" ||
|
|
$1 == "----" ||
|
|
$1 == "MemoryPressure" ||
|
|
$1 == "DiskPressure" ||
|
|
$1 == "PIDPressure" ||
|
|
$1 == "Ready") {
|
|
print
|
|
}
|
|
}
|
|
' || true
|
|
|
|
echo
|
|
echo "Capacity/Allocatable:"
|
|
k3sRun describe node 2>/dev/null | awk '
|
|
/^Capacity:/ { p=1; print; next }
|
|
/^System Info:/ { p=0 }
|
|
p && $1 !~ /InternalIP|Hostname|Machine|UUID|Boot|ProviderID/ { print }
|
|
' || true
|
|
|
|
echo
|
|
echo "Allocated resources:"
|
|
k3sRun describe node 2>/dev/null | awk '
|
|
/^Allocated resources:/ { p=1; print; next }
|
|
/^Events:/ { p=0 }
|
|
p { print }
|
|
' || true
|
|
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Appends a MariaDB slave replication status summary to the diag file.
|
|
function diagMariaDBReplicaSummary() {
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
|
|
diagLogSection "MariaDB slave replication summary"
|
|
|
|
{
|
|
echo "+ mariadbSlaveRootQuery SHOW REPLICA STATUS"
|
|
echo
|
|
|
|
mariadbSlaveRootQuery "SHOW REPLICA STATUS\G" > "$tmp" 2>&1 || true
|
|
|
|
if grep -q 'Slave_IO_Running:' "$tmp"; then
|
|
awk -F': ' '
|
|
/^[[:space:]]*Slave_IO_State:/ { print "Slave_IO_State:", $2 }
|
|
/^[[:space:]]*Slave_IO_Running:/ { print "Slave_IO_Running:", $2 }
|
|
/^[[:space:]]*Slave_SQL_Running:/ { print "Slave_SQL_Running:", $2 }
|
|
/^[[:space:]]*Seconds_Behind_Master:/ { print "Seconds_Behind_Master:", $2 }
|
|
/^[[:space:]]*Last_IO_Errno:/ { print "Last_IO_Errno:", $2 }
|
|
/^[[:space:]]*Last_SQL_Errno:/ { print "Last_SQL_Errno:", $2 }
|
|
/^[[:space:]]*Relay_Log_Space:/ { print "Relay_Log_Space:", $2 }
|
|
/^[[:space:]]*Read_Master_Log_Pos:/ { print "Read_Master_Log_Pos:", $2 }
|
|
/^[[:space:]]*Exec_Master_Log_Pos:/ { print "Exec_Master_Log_Pos:", $2 }
|
|
/^[[:space:]]*Using_Gtid:/ { print "Using_Gtid:", $2 }
|
|
/^[[:space:]]*Parallel_Mode:/ { print "Parallel_Mode:", $2 }
|
|
/^[[:space:]]*Slave_SQL_Running_State:/ { print "Slave_SQL_Running_State:", $2 }
|
|
' "$tmp" || true
|
|
|
|
echo
|
|
rm -f "$tmp"
|
|
return
|
|
fi
|
|
|
|
: > "$tmp"
|
|
mariadbSlaveRootQuery "SHOW REPLICA STATUS;" > "$tmp" 2>&1 || true
|
|
|
|
if [[ ! -s "$tmp" ]]; then
|
|
echo "replication_status_available: no"
|
|
echo
|
|
rm -f "$tmp"
|
|
return
|
|
fi
|
|
|
|
awk -F'\t' '
|
|
NF >= 12 {
|
|
print "Slave_IO_State:", $1
|
|
print "Master_Host: [REDACTED]"
|
|
print "Master_User: [REDACTED]"
|
|
print "Master_Port:", $4
|
|
print "Master_Log_File:", $6
|
|
print "Read_Master_Log_Pos:", $7
|
|
print "Relay_Log_File:", $8
|
|
print "Relay_Log_Pos:", $9
|
|
print "Relay_Master_Log_File:", $10
|
|
print "Slave_IO_Running:", $11
|
|
print "Slave_SQL_Running:", $12
|
|
found=1
|
|
next
|
|
}
|
|
{
|
|
print
|
|
}
|
|
END {
|
|
if (!found) {
|
|
print "replication_status_parse: raw fallback"
|
|
}
|
|
}
|
|
' "$tmp" || true
|
|
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
|
|
rm -f "$tmp"
|
|
}
|
|
|
|
# Parses the MariaDB slow query log since $diagSince and appends a fingerprint summary to the diag file.
|
|
function diagMariaDBSlowSummary() {
|
|
local mariadbMasterSlowLog="$mariadbMasterPath/slow.log"
|
|
local cutoff
|
|
|
|
cutoff="$(diagCutoffEpoch)"
|
|
|
|
diagLogSection "MariaDB slow query summary only"
|
|
|
|
{
|
|
echo "+ summarize slow log: $mariadbMasterSlowLog since ${diagSince:-4h}"
|
|
echo
|
|
|
|
if [[ ! -r "$mariadbMasterSlowLog" ]]; then
|
|
echo "slow_log_readable: no"
|
|
echo
|
|
return
|
|
fi
|
|
|
|
perl -MTime::Local - "$mariadbMasterSlowLog" "$cutoff" <<'PERL'
|
|
use strict;
|
|
use warnings;
|
|
use Time::Local;
|
|
|
|
my ($file, $cutoff) = @ARGV;
|
|
|
|
open my $fh, '<', $file or do {
|
|
print "slow_log_readable: no\n";
|
|
exit 0;
|
|
};
|
|
|
|
my %fp;
|
|
my $total = 0;
|
|
my $max_qt = 0;
|
|
my $max_rows_examined = 0;
|
|
my $max_rows_sent = 0;
|
|
|
|
my ($active, $ts, $qt, $rows_sent, $rows_examined, $sql);
|
|
|
|
sub reset_entry {
|
|
$active = 0;
|
|
$ts = 0;
|
|
$qt = 0;
|
|
$rows_sent = 0;
|
|
$rows_examined = 0;
|
|
$sql = '';
|
|
}
|
|
|
|
sub fingerprint_sql {
|
|
my ($s) = @_;
|
|
|
|
$s =~ s/`[^`]*`/`X`/g;
|
|
$s =~ s/'(?:\\.|[^'\\])*'/'X'/g;
|
|
$s =~ s/"(?:\\.|[^"\\])*"/"X"/g;
|
|
$s =~ s/\b[0-9a-fA-F]{16,}\b/HEX/g;
|
|
$s =~ s/\b\d+(?:\.\d+)?\b/N/g;
|
|
$s =~ s/\s+/ /g;
|
|
$s =~ s/^\s+|\s+$//g;
|
|
|
|
return substr($s, 0, 220);
|
|
}
|
|
|
|
sub flush_entry {
|
|
return unless $active;
|
|
return if $ts < $cutoff;
|
|
|
|
$total++;
|
|
$max_qt = $qt if $qt > $max_qt;
|
|
$max_rows_examined = $rows_examined if $rows_examined > $max_rows_examined;
|
|
$max_rows_sent = $rows_sent if $rows_sent > $max_rows_sent;
|
|
|
|
my $f = fingerprint_sql($sql);
|
|
return unless length $f;
|
|
|
|
$fp{$f}{count}++;
|
|
$fp{$f}{total_time} += $qt;
|
|
$fp{$f}{total_rows_examined} += $rows_examined;
|
|
$fp{$f}{max_time} = $qt if !defined($fp{$f}{max_time}) || $qt > $fp{$f}{max_time};
|
|
}
|
|
|
|
reset_entry();
|
|
|
|
while (my $line = <$fh>) {
|
|
chomp $line;
|
|
|
|
if ($line =~ /^# Time:\s*(\d{2})(\d{2})(\d{2})\s+(\d{1,2}):(\d{2}):(\d{2})/) {
|
|
flush_entry();
|
|
reset_entry();
|
|
|
|
my ($yy, $mo, $dd, $hh, $mm, $ss) = ($1, $2, $3, $4, $5, $6);
|
|
my $yyyy = $yy >= 70 ? 1900 + $yy : 2000 + $yy;
|
|
|
|
$ts = timelocal($ss, $mm, $hh, $dd, $mo - 1, $yyyy);
|
|
$active = 1;
|
|
next;
|
|
}
|
|
|
|
next unless $active;
|
|
|
|
if ($line =~ /^# Query_time:\s*([0-9.]+).*Rows_sent:\s*([0-9]+).*Rows_examined:\s*([0-9]+)/) {
|
|
$qt = $1 + 0;
|
|
$rows_sent = $2 + 0;
|
|
$rows_examined = $3 + 0;
|
|
next;
|
|
}
|
|
|
|
next if $line =~ /^# User\@Host:/;
|
|
next if $line =~ /^use `/;
|
|
next if $line =~ /^SET timestamp=/;
|
|
next if $line =~ /^#/;
|
|
next if $line =~ /^\s*$/;
|
|
|
|
$sql .= ' ' . $line;
|
|
}
|
|
|
|
flush_entry();
|
|
|
|
print "slow_queries_count: $total\n";
|
|
print "max_query_time: $max_qt\n";
|
|
print "max_rows_examined: $max_rows_examined\n";
|
|
print "max_rows_sent: $max_rows_sent\n";
|
|
print "\n";
|
|
print "top_fingerprints:\n";
|
|
|
|
my $shown = 0;
|
|
for my $f (
|
|
sort {
|
|
$fp{$b}{total_time} <=> $fp{$a}{total_time}
|
|
||
|
|
$fp{$b}{total_rows_examined} <=> $fp{$a}{total_rows_examined}
|
|
} keys %fp
|
|
) {
|
|
last if $shown++ >= 10;
|
|
|
|
printf(
|
|
"- count=%d total_time=%.3f max_time=%.3f total_rows_examined=%d sql=%s\n",
|
|
$fp{$f}{count},
|
|
$fp{$f}{total_time},
|
|
$fp{$f}{max_time},
|
|
$fp{$f}{total_rows_examined},
|
|
$f
|
|
);
|
|
}
|
|
PERL
|
|
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Appends lsphp process count and RSS stats from an OLS pod to the diag file.
|
|
# $1 (pod): OLS pod name.
|
|
function diagOlsPodStats() {
|
|
local pod="$1"
|
|
|
|
diagLogSection "OLS $pod lsphp count and RSS"
|
|
|
|
{
|
|
echo "+ k3sRun exec pod/$pod -c openlitespeed -- sh -s <lsphp stats>"
|
|
|
|
k3sRun exec -i "pod/$pod" -c openlitespeed -- sh -s <<'SH' || true
|
|
tmp="$(mktemp)"
|
|
|
|
echo "hostname:"
|
|
hostname 2>/dev/null || true
|
|
echo
|
|
|
|
ps -eo user=,rss=,comm=,args= > "$tmp" 2>/dev/null || ps aux > "$tmp" 2>/dev/null || true
|
|
|
|
echo "lsphp count:"
|
|
awk '
|
|
/lsphp/ && $0 !~ /awk|grep/ { c++ }
|
|
END { print c + 0 }
|
|
' "$tmp"
|
|
echo
|
|
|
|
echo "total lsphp RSS KiB:"
|
|
awk '
|
|
/lsphp/ && $0 !~ /awk|grep/ { sum += $2 }
|
|
END { print sum + 0 }
|
|
' "$tmp"
|
|
echo
|
|
|
|
echo "total lsphp RSS MiB:"
|
|
awk '
|
|
/lsphp/ && $0 !~ /awk|grep/ { sum += $2 }
|
|
END { printf "%.1f\n", sum / 1024 }
|
|
' "$tmp"
|
|
echo
|
|
|
|
echo "top RSS lsphp:"
|
|
awk '
|
|
/lsphp/ && $0 !~ /awk|grep/ {
|
|
print
|
|
}
|
|
' "$tmp" | sort -k2,2nr | head -10
|
|
echo
|
|
|
|
echo "lsphp by user:"
|
|
awk '
|
|
/lsphp/ && $0 !~ /awk|grep/ {
|
|
count[$1]++
|
|
rss[$1] += $2
|
|
}
|
|
END {
|
|
for (u in count) {
|
|
printf "%s count=%d rss_kib=%d rss_mib=%.1f\n", u, count[u], rss[u], rss[u] / 1024
|
|
}
|
|
}
|
|
' "$tmp" | sort -k3,3nr
|
|
echo
|
|
|
|
echo "process count:"
|
|
awk '
|
|
NF >= 3 {
|
|
comm=$3
|
|
count[comm]++
|
|
}
|
|
END {
|
|
for (c in count) {
|
|
print count[c], c
|
|
}
|
|
}
|
|
' "$tmp" | sort -nr | head -20
|
|
echo
|
|
|
|
rm -f "$tmp"
|
|
SH
|
|
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Parses the OLS error log since $diagSince and appends categorized error counters to the diag file.
|
|
function diagOlsErrorSummary() {
|
|
local openlitespeedErrorLog="$olsLogsPath/error.log"
|
|
local cutoff
|
|
|
|
cutoff="$(diagCutoffEpoch)"
|
|
|
|
diagLogSection "OLS error counters summary"
|
|
|
|
{
|
|
echo "+ summarize OLS error log: $openlitespeedErrorLog since ${diagSince:-4h}"
|
|
echo
|
|
|
|
if [[ ! -r "$openlitespeedErrorLog" ]]; then
|
|
echo "ols_error_log_readable: no"
|
|
echo
|
|
return
|
|
fi
|
|
|
|
perl -MTime::Local - "$openlitespeedErrorLog" "$cutoff" <<'PERL'
|
|
use strict;
|
|
use warnings;
|
|
use Time::Local;
|
|
|
|
my ($file, $cutoff) = @ARGV;
|
|
|
|
open my $fh, '<', $file or do {
|
|
print "ols_error_log_readable: no\n";
|
|
exit 0;
|
|
};
|
|
|
|
my %c = (
|
|
total_lines => 0,
|
|
warn_count => 0,
|
|
error_count => 0,
|
|
hostname_mapping_conflicts => 0,
|
|
inaccessible_vhost_root => 0,
|
|
no_request_delivery => 0,
|
|
connection_reset => 0,
|
|
memory_errors => 0,
|
|
too_many_open_files => 0,
|
|
permission_denied => 0,
|
|
);
|
|
|
|
while (my $line = <$fh>) {
|
|
my $ts = 0;
|
|
|
|
if ($line =~ /^(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})/) {
|
|
$ts = timelocal($6, $5, $4, $3, $2 - 1, $1);
|
|
}
|
|
|
|
next if $ts && $ts < $cutoff;
|
|
|
|
$c{total_lines}++;
|
|
$c{warn_count}++ if $line =~ /\[WARN\]/;
|
|
$c{error_count}++ if $line =~ /\[ERROR\]/;
|
|
$c{hostname_mapping_conflicts}++ if $line =~ /Hostname .* is mapped to virtual host .* can.t map to virtual host/;
|
|
$c{inaccessible_vhost_root}++ if $line =~ /Path for vhost root is not accessible/;
|
|
$c{no_request_delivery}++ if $line =~ /No request delivery notification has been received/;
|
|
$c{connection_reset}++ if $line =~ /Connection reset by peer/;
|
|
$c{memory_errors}++ if $line =~ /OOM|out of memory|Cannot allocate memory/i;
|
|
$c{too_many_open_files}++ if $line =~ /Too many open files/i;
|
|
$c{permission_denied}++ if $line =~ /Permission denied/i;
|
|
}
|
|
|
|
for my $k (
|
|
qw(
|
|
total_lines
|
|
warn_count
|
|
error_count
|
|
hostname_mapping_conflicts
|
|
inaccessible_vhost_root
|
|
no_request_delivery
|
|
connection_reset
|
|
memory_errors
|
|
too_many_open_files
|
|
permission_denied
|
|
)
|
|
) {
|
|
print "$k: $c{$k}\n";
|
|
}
|
|
PERL
|
|
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Fetches the OPcache status endpoint four times and appends key metrics to the diag file.
|
|
function diagOpcacheSummary() {
|
|
local i tmp
|
|
|
|
diagLogSection "OPcache summary"
|
|
|
|
{
|
|
echo "+ curl OPcache endpoint summary"
|
|
echo "attempts: 4"
|
|
echo
|
|
|
|
for i in 1 2 3 4; do
|
|
tmp="$(mktemp)"
|
|
|
|
curl -kfsS --max-time 10 "$diagOpcacheUrl" > "$tmp" 2>/dev/null || {
|
|
echo "attempt_$i: failed"
|
|
rm -f "$tmp"
|
|
continue
|
|
}
|
|
|
|
echo "attempt_$i:"
|
|
|
|
awk '
|
|
function val(line) {
|
|
sub(/^.*=>[[:space:]]*/, "", line)
|
|
if (line == "" || line == "=>") return "false"
|
|
return line
|
|
}
|
|
|
|
/\[memory_usage\]/ { ctx="memory"; next }
|
|
/\[interned_strings_usage\]/ { ctx="interned"; next }
|
|
/\[opcache_statistics\]/ { ctx="stats"; next }
|
|
/\[jit\]/ { ctx="jit"; next }
|
|
|
|
/\[opcache_enabled\]/ { print " opcache_enabled:", val($0) }
|
|
/\[cache_full\]/ { print " cache_full:", val($0) }
|
|
/\[restart_pending\]/ { print " restart_pending:", val($0) }
|
|
/\[restart_in_progress\]/ { print " restart_in_progress:", val($0) }
|
|
|
|
ctx=="memory" && /\[used_memory\]/ { print " memory_used:", val($0) }
|
|
ctx=="memory" && /\[free_memory\]/ { print " memory_free:", val($0) }
|
|
ctx=="memory" && /\[wasted_memory\]/ { print " memory_wasted:", val($0) }
|
|
ctx=="memory" && /\[current_wasted_percentage\]/ { print " memory_wasted_pct:", val($0) }
|
|
|
|
ctx=="interned" && /\[buffer_size\]/ { print " interned_buffer_size:", val($0) }
|
|
ctx=="interned" && /\[used_memory\]/ { print " interned_used:", val($0) }
|
|
ctx=="interned" && /\[free_memory\]/ { print " interned_free:", val($0) }
|
|
ctx=="interned" && /\[number_of_strings\]/ { print " interned_strings:", val($0) }
|
|
|
|
ctx=="stats" && /\[num_cached_scripts\]/ { print " num_cached_scripts:", val($0) }
|
|
ctx=="stats" && /\[num_cached_keys\]/ { print " num_cached_keys:", val($0) }
|
|
ctx=="stats" && /\[max_cached_keys\]/ { print " max_cached_keys:", val($0) }
|
|
ctx=="stats" && /\[hits\]/ { print " hits:", val($0) }
|
|
ctx=="stats" && /\[misses\]/ { print " misses:", val($0) }
|
|
ctx=="stats" && /\[oom_restarts\]/ { print " oom_restarts:", val($0) }
|
|
ctx=="stats" && /\[hash_restarts\]/ { print " hash_restarts:", val($0) }
|
|
ctx=="stats" && /\[manual_restarts\]/ { print " manual_restarts:", val($0) }
|
|
ctx=="stats" && /\[opcache_hit_rate\]/ { print " opcache_hit_rate:", val($0) }
|
|
' "$tmp" || true
|
|
|
|
rm -f "$tmp"
|
|
echo
|
|
done
|
|
} >> "$diagFile" 2>&1
|
|
}
|
|
|
|
# Collects system-level diagnostics (uptime, memory, vmstat, iostat, PSI, sar, disk usage).
|
|
function diagSystem() {
|
|
diagLogSection "uptime"
|
|
diagCmd uptime
|
|
|
|
diagLogSection "free -m"
|
|
diagCmd free -m
|
|
|
|
diagLogSection "vmstat 1 10"
|
|
diagCmd vmstat 1 10
|
|
|
|
diagLogSection "mpstat 1 10"
|
|
diagCmd mpstat 1 10
|
|
|
|
diagLogSection "iostat -x -z 1 5 filtered"
|
|
diagIostat
|
|
|
|
diagLogSection "cat /proc/pressure/cpu"
|
|
diagCmd cat /proc/pressure/cpu
|
|
|
|
diagLogSection "cat /proc/pressure/memory"
|
|
diagCmd cat /proc/pressure/memory
|
|
|
|
diagLogSection "cat /proc/pressure/io"
|
|
diagCmd cat /proc/pressure/io
|
|
|
|
diagSarTry "sar queue last ${diagSince:-4h}" "sar -q -s '$(diagSarStartTime)'"
|
|
diagSarTry "sar cpu last ${diagSince:-4h}" "sar -u -s '$(diagSarStartTime)'"
|
|
diagSarTry "sar memory last ${diagSince:-4h}" "sar -r -s '$(diagSarStartTime)'"
|
|
diagSarTry "sar swap last ${diagSince:-4h}" "sar -W -s '$(diagSarStartTime)'"
|
|
|
|
diagLogSection "Filesystem pressure quick checks"
|
|
diagDfUsage
|
|
|
|
diagLogSection "Filesystem inode quick checks"
|
|
diagDfInodes
|
|
}
|
|
|
|
# Collects k3s diagnostics (pod top/list, warning events, restart summary, node conditions).
|
|
function diagK3s() {
|
|
diagLogSection "Top pod"
|
|
diagK3sTry top pod
|
|
|
|
diagLogSection "Get pod"
|
|
diagK3sTry get pod
|
|
|
|
diagLogSection "Get warning events recent"
|
|
diagK3sTry get events --field-selector type!=Normal --sort-by=.lastTimestamp
|
|
|
|
diagLogSection "Pod restart summary"
|
|
diagK3sTry get pod -o custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,RESTARTS:.status.containerStatuses[*].restartCount,STATE:.status.containerStatuses[*].state.waiting.reason,LAST_STATE:.status.containerStatuses[*].lastState.terminated.reason
|
|
|
|
diagLogSection "Node conditions and allocated resources"
|
|
diagK3sNodeSummary
|
|
}
|
|
|
|
# Collects MariaDB diagnostics (master/slave global status, replication summary, slow queries).
|
|
function diagMariaDB() {
|
|
diagLogSection "MariaDB master global status"
|
|
{
|
|
echo "+ mariadbMasterRootQuery"
|
|
mariadbMasterRootQuery "
|
|
SHOW GLOBAL STATUS WHERE Variable_name IN (
|
|
'Uptime',
|
|
'Threads_running',
|
|
'Threads_connected',
|
|
'Max_used_connections',
|
|
'Connections',
|
|
'Aborted_connects',
|
|
'Slow_queries',
|
|
'Questions',
|
|
'Queries',
|
|
'Created_tmp_tables',
|
|
'Created_tmp_disk_tables',
|
|
'Innodb_buffer_pool_reads',
|
|
'Innodb_buffer_pool_read_requests',
|
|
'Innodb_data_reads',
|
|
'Innodb_data_writes',
|
|
'Innodb_log_waits',
|
|
'Open_tables',
|
|
'Opened_tables',
|
|
'Table_open_cache_hits',
|
|
'Table_open_cache_misses',
|
|
'Table_open_cache_overflows'
|
|
);
|
|
" || true
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
|
|
diagLogSection "MariaDB slave global status"
|
|
{
|
|
echo "+ mariadbSlaveRootQuery"
|
|
mariadbSlaveRootQuery "
|
|
SHOW GLOBAL STATUS WHERE Variable_name IN (
|
|
'Uptime',
|
|
'Threads_running',
|
|
'Threads_connected',
|
|
'Max_used_connections',
|
|
'Connections',
|
|
'Aborted_connects',
|
|
'Slow_queries',
|
|
'Questions',
|
|
'Queries',
|
|
'Created_tmp_tables',
|
|
'Created_tmp_disk_tables',
|
|
'Innodb_buffer_pool_reads',
|
|
'Innodb_buffer_pool_read_requests',
|
|
'Innodb_data_reads',
|
|
'Innodb_data_writes',
|
|
'Innodb_log_waits',
|
|
'Slave_running',
|
|
'Slave_received_heartbeats',
|
|
'Slave_heartbeat_period',
|
|
'Slave_open_temp_tables'
|
|
);
|
|
" || true
|
|
echo
|
|
} >> "$diagFile" 2>&1
|
|
|
|
diagMariaDBReplicaSummary
|
|
diagMariaDBSlowSummary
|
|
}
|
|
|
|
# Collects OpenLiteSpeed diagnostics (pod list, per-pod lsphp stats, error summary, OPcache).
|
|
function diagOpenLiteSpeed() {
|
|
local podList pod
|
|
|
|
diagLogSection "OpenLiteSpeed pods"
|
|
diagK3sTry get pod -l app=openlitespeed
|
|
|
|
podList="$(k3sPodListApp openlitespeed 2>/dev/null || true)"
|
|
while read -r pod; do
|
|
[[ -z "$pod" ]] && continue
|
|
diagOlsPodStats "$pod"
|
|
done < <(awk 'NF' <<< "$podList")
|
|
|
|
diagOlsErrorSummary
|
|
diagOpcacheSummary
|
|
}
|
|
|
|
# Generates a full diagnostic report and writes it to $diagFile.
|
|
# [$1] (diagFile): output file path (defaults to $diagFile).
|
|
function diagReport() {
|
|
local diagFile="${1:-$diagFile}"
|
|
local aiModel="${1:-short}"
|
|
|
|
export LC_ALL=C
|
|
export LANG=C
|
|
|
|
local reportPath
|
|
reportPath=$(dirname "$diagFile")
|
|
[[ -d "$reportPath" ]] || mkdir -p "$reportPath"
|
|
|
|
if [[ "$postfixHost" ]]; then
|
|
diagExcludeVhostsRegex="$diagExcludeVhostsRegex|${postfixHost//./\\.}"
|
|
fi
|
|
|
|
: > "$diagFile"
|
|
diagMeta
|
|
diagSystem
|
|
diagK3s
|
|
diagMariaDB
|
|
diagOpenLiteSpeed
|
|
diagSanitizeReport
|
|
}
|
|
|
|
# Uploads the diag report file to the remote API.
|
|
# [$1] (diagFile): report file path (defaults to $diagFile).
|
|
# [$2] (aiModelId): AI endpoint ID to use (defaults to 1).
|
|
function diagSend() {
|
|
local diagFile="${1:-$diagFile}"
|
|
local aiModelId="${2:-1}"
|
|
|
|
logInfo "$diagLabel Sending data..."
|
|
curl -fsS -X POST "$diagApiUrl" -F "source_type=worker" -F "source_id=$hostUuid" -F "generated_time=$(date +%s)" -F "report_file=@$diagFile" -F "ai_endpoint_id=$aiModelId"
|
|
logSuccess "$diagLabel Data sent successfully"
|
|
}
|
|
|
|
# Generates the full diagnostic report and sends it to the remote API.
|
|
# [$1] (aiModel): model name: full or short (defaults to short).
|
|
function diagRun() {
|
|
local aiModel="${1:-short}"
|
|
local aiModelId=2
|
|
|
|
case "$aiModel" in
|
|
full)
|
|
aiModelId=1
|
|
;;
|
|
*)
|
|
aiModelId=2
|
|
;;
|
|
esac
|
|
|
|
diagReport "$diagFile"
|
|
diagSend "$diagFile" "$aiModelId"
|
|
}
|