sodew-bash
This commit is contained in:
@@ -0,0 +1,931 @@
|
||||
#!/bin/bash
|
||||
|
||||
diagLabel="[Diag]"
|
||||
diagExcludeVhostsRegex='localhost|Example'
|
||||
|
||||
# Writes a section header to the diag log file and prints it to the info log.
|
||||
# $1 (title): section title text.
|
||||
function diagLogSection() {
|
||||
local title="$1"
|
||||
local padding
|
||||
local logPath
|
||||
|
||||
logInfo "$diagLabel $title"
|
||||
|
||||
padding=$((72 - ${#title}))
|
||||
if (( padding > 0 )); then
|
||||
title="${title} $(printf '.%.s' $(seq 1 "$padding"))"
|
||||
fi
|
||||
|
||||
logPath=$(dirname "$diagFile")
|
||||
[[ -d "$logPath" ]] || mkdir -p "$logPath"
|
||||
|
||||
printf "[ %s ]\n" "$title" >> "$diagFile"
|
||||
}
|
||||
|
||||
# Runs a command and appends its output (stdout and stderr) to the diag file.
|
||||
# $@ (...): command and arguments to execute.
|
||||
function diagCmd() {
|
||||
echo "+ $*" >> "$diagFile"
|
||||
"$@" >> "$diagFile" 2>&1 || true
|
||||
echo >> "$diagFile"
|
||||
}
|
||||
|
||||
# Runs a shell command via bash -lc and appends its output to the diag file; errors ignored.
|
||||
# $@ (...): shell command string to execute.
|
||||
function diagShTry() {
|
||||
echo "+ $*" >> "$diagFile"
|
||||
bash -lc "$*" >> "$diagFile" 2>&1 || true
|
||||
echo >> "$diagFile"
|
||||
}
|
||||
|
||||
# Runs a kubectl command via k3sRun and appends its output to the diag file; errors ignored.
|
||||
# $@ (...): kubectl arguments to pass to k3sRun.
|
||||
function diagK3sTry() {
|
||||
echo "+ k3sRun $*" >> "$diagFile"
|
||||
k3sRun "$@" >> "$diagFile" 2>&1 || true
|
||||
echo >> "$diagFile"
|
||||
}
|
||||
|
||||
# Converts $diagSince (e.g. 4h, 30m, 2d) to a human-readable date argument for the date command.
|
||||
function diagSinceDateArg() {
|
||||
local s="${diagSince:-4h}"
|
||||
|
||||
case "$s" in
|
||||
*m)
|
||||
printf '%s minutes ago' "${s%m}"
|
||||
;;
|
||||
*h)
|
||||
printf '%s hours ago' "${s%h}"
|
||||
;;
|
||||
*d)
|
||||
printf '%s days ago' "${s%d}"
|
||||
;;
|
||||
*)
|
||||
printf '4 hours ago'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Returns the Unix epoch timestamp corresponding to the $diagSince cutoff time.
|
||||
function diagCutoffEpoch() {
|
||||
date -d "$(diagSinceDateArg)" +%s 2>/dev/null || date -d "4 hours ago" +%s
|
||||
}
|
||||
|
||||
# Returns the HH:MM:SS start time for sar based on $diagSince; falls back to 00:00:00 if in the past.
|
||||
function diagSarStartTime() {
|
||||
local t now
|
||||
|
||||
t="$(date -d "$(diagSinceDateArg)" +%H:%M:%S 2>/dev/null || echo "00:00:00")"
|
||||
now="$(date +%H:%M:%S)"
|
||||
|
||||
if [[ "$t" > "$now" ]]; then
|
||||
printf "00:00:00"
|
||||
else
|
||||
printf "%s" "$t"
|
||||
fi
|
||||
}
|
||||
|
||||
# Writes diagnostic metadata (timestamp, since, hostname, file path) to the diag file.
|
||||
function diagMeta() {
|
||||
diagLogSection "Diagnostic metadata"
|
||||
|
||||
{
|
||||
echo "started_at: $(date -Iseconds)"
|
||||
echo "since: $diagSince"
|
||||
echo "hostname: [SERVER]"
|
||||
echo "file: $diagFile"
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Replaces hostnames, user paths, and UIDs in the diag file with redacted placeholders.
|
||||
function diagSanitizeReport() {
|
||||
local hostLower
|
||||
local hostFqdn
|
||||
local hostFqdnLower
|
||||
local sanitizeHosts
|
||||
|
||||
[[ -f "$diagFile" ]] || return 0
|
||||
|
||||
hostLower="$(printf '%s' "$hostName" | tr '[:upper:]' '[:lower:]')"
|
||||
hostFqdn="$(hostname -f 2>/dev/null || true)"
|
||||
hostFqdnLower="$(printf '%s' "$hostFqdn" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
sanitizeHosts="$(
|
||||
printf '%s\n' "$hostName" "$hostLower" "$hostFqdn" "$hostFqdnLower" | awk 'NF && !seen[$0]++'
|
||||
)"
|
||||
|
||||
SANITIZE_HOSTS="$sanitizeHosts" perl -0pi -e '
|
||||
my $hosts = $ENV{SANITIZE_HOSTS} // "";
|
||||
for my $h (split /\n/, $hosts) {
|
||||
next unless defined $h && length $h;
|
||||
my $q = quotemeta($h);
|
||||
s/$q/[SERVER]/g;
|
||||
}
|
||||
|
||||
s#(/media/)[^/\s]+#$1[USER]#g;
|
||||
s#(/run/user/)[0-9]+#$1[UID]#g;
|
||||
' "$diagFile"
|
||||
}
|
||||
|
||||
# Filters stdin lines, excluding those matching reserved vhost names ($diagExcludeVhostsRegex).
|
||||
function diagGrepExcludeReserved() {
|
||||
local regex="${diagExcludeVhostsRegex:-}"
|
||||
|
||||
if [[ -n "$regex" ]]; then
|
||||
grep -Eiv "$regex"
|
||||
else
|
||||
cat
|
||||
fi
|
||||
}
|
||||
|
||||
# Appends filtered disk usage (df -h, excluding overlay/tmpfs/pod mounts) to the diag file.
|
||||
function diagDfUsage() {
|
||||
{
|
||||
echo "+ df -h filtered"
|
||||
df -h \
|
||||
-x tmpfs \
|
||||
-x devtmpfs \
|
||||
-x overlay \
|
||||
-x squashfs \
|
||||
-x efivarfs \
|
||||
2>/dev/null | awk '
|
||||
NR == 1 { print; next }
|
||||
$6 ~ "^/run/k3s/" { next }
|
||||
$6 ~ "^/var/lib/kubelet/pods/" { next }
|
||||
$6 ~ "^/run/user/" { next }
|
||||
$6 ~ "^/media/" { next }
|
||||
{ print }
|
||||
'
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Appends filtered inode usage (df -i, excluding overlay/tmpfs/pod mounts) to the diag file.
|
||||
function diagDfInodes() {
|
||||
{
|
||||
echo "+ df -i filtered"
|
||||
df -i \
|
||||
-x tmpfs \
|
||||
-x devtmpfs \
|
||||
-x overlay \
|
||||
-x squashfs \
|
||||
-x efivarfs \
|
||||
2>/dev/null | awk '
|
||||
NR == 1 { print; next }
|
||||
$6 ~ "^/run/k3s/" { next }
|
||||
$6 ~ "^/var/lib/kubelet/pods/" { next }
|
||||
$6 ~ "^/run/user/" { next }
|
||||
$6 ~ "^/media/" { next }
|
||||
{ print }
|
||||
'
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Appends iostat extended stats (1s interval, 5 samples, loop/sr devices excluded) to the diag file.
|
||||
function diagIostat() {
|
||||
{
|
||||
echo "+ iostat -x -z 1 5 | filter loop/sr devices"
|
||||
|
||||
{
|
||||
iostat -x -z 1 5 2>/dev/null || iostat -x 1 5 2>/dev/null
|
||||
} | awk '
|
||||
/^loop[0-9]+/ { next }
|
||||
/^sr[0-9]+/ { next }
|
||||
{ print }
|
||||
'
|
||||
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Runs a sar command and appends its output to the diag file; skips gracefully if data is unavailable.
|
||||
# $1 (title): section title for the diag log.
|
||||
# $2 (cmd): sar command string to execute.
|
||||
function diagSarTry() {
|
||||
local title="$1"
|
||||
local cmd="$2"
|
||||
local saFile
|
||||
|
||||
saFile="/var/log/sysstat/sa$(date +%d)"
|
||||
|
||||
diagLogSection "$title"
|
||||
|
||||
{
|
||||
echo "+ $cmd"
|
||||
|
||||
if [[ ! -r "$saFile" ]]; then
|
||||
echo "sar_data_available: no"
|
||||
echo "sar_file: $saFile"
|
||||
echo "hint: enable sysstat data collection if SAR trends are needed"
|
||||
echo
|
||||
return
|
||||
fi
|
||||
|
||||
bash -lc "$cmd" || true
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Appends a filtered k3s node summary (conditions, capacity, allocated resources) to the diag file.
|
||||
function diagK3sNodeSummary() {
|
||||
{
|
||||
echo "+ k3sRun describe node filtered"
|
||||
|
||||
echo
|
||||
echo "Conditions:"
|
||||
k3sRun describe node 2>/dev/null | awk '
|
||||
/^Conditions:/ { p=1; next }
|
||||
/^Addresses:/ { p=0 }
|
||||
p {
|
||||
if ($1 == "Type" ||
|
||||
$1 == "----" ||
|
||||
$1 == "MemoryPressure" ||
|
||||
$1 == "DiskPressure" ||
|
||||
$1 == "PIDPressure" ||
|
||||
$1 == "Ready") {
|
||||
print
|
||||
}
|
||||
}
|
||||
' || true
|
||||
|
||||
echo
|
||||
echo "Capacity/Allocatable:"
|
||||
k3sRun describe node 2>/dev/null | awk '
|
||||
/^Capacity:/ { p=1; print; next }
|
||||
/^System Info:/ { p=0 }
|
||||
p && $1 !~ /InternalIP|Hostname|Machine|UUID|Boot|ProviderID/ { print }
|
||||
' || true
|
||||
|
||||
echo
|
||||
echo "Allocated resources:"
|
||||
k3sRun describe node 2>/dev/null | awk '
|
||||
/^Allocated resources:/ { p=1; print; next }
|
||||
/^Events:/ { p=0 }
|
||||
p { print }
|
||||
' || true
|
||||
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Appends a MariaDB slave replication status summary to the diag file.
|
||||
function diagMariaDBReplicaSummary() {
|
||||
local tmp
|
||||
|
||||
tmp="$(mktemp)"
|
||||
|
||||
diagLogSection "MariaDB slave replication summary"
|
||||
|
||||
{
|
||||
echo "+ mariadbSlaveRootQuery SHOW REPLICA STATUS"
|
||||
echo
|
||||
|
||||
mariadbSlaveRootQuery "SHOW REPLICA STATUS\G" > "$tmp" 2>&1 || true
|
||||
|
||||
if grep -q 'Slave_IO_Running:' "$tmp"; then
|
||||
awk -F': ' '
|
||||
/^[[:space:]]*Slave_IO_State:/ { print "Slave_IO_State:", $2 }
|
||||
/^[[:space:]]*Slave_IO_Running:/ { print "Slave_IO_Running:", $2 }
|
||||
/^[[:space:]]*Slave_SQL_Running:/ { print "Slave_SQL_Running:", $2 }
|
||||
/^[[:space:]]*Seconds_Behind_Master:/ { print "Seconds_Behind_Master:", $2 }
|
||||
/^[[:space:]]*Last_IO_Errno:/ { print "Last_IO_Errno:", $2 }
|
||||
/^[[:space:]]*Last_SQL_Errno:/ { print "Last_SQL_Errno:", $2 }
|
||||
/^[[:space:]]*Relay_Log_Space:/ { print "Relay_Log_Space:", $2 }
|
||||
/^[[:space:]]*Read_Master_Log_Pos:/ { print "Read_Master_Log_Pos:", $2 }
|
||||
/^[[:space:]]*Exec_Master_Log_Pos:/ { print "Exec_Master_Log_Pos:", $2 }
|
||||
/^[[:space:]]*Using_Gtid:/ { print "Using_Gtid:", $2 }
|
||||
/^[[:space:]]*Parallel_Mode:/ { print "Parallel_Mode:", $2 }
|
||||
/^[[:space:]]*Slave_SQL_Running_State:/ { print "Slave_SQL_Running_State:", $2 }
|
||||
' "$tmp" || true
|
||||
|
||||
echo
|
||||
rm -f "$tmp"
|
||||
return
|
||||
fi
|
||||
|
||||
: > "$tmp"
|
||||
mariadbSlaveRootQuery "SHOW REPLICA STATUS;" > "$tmp" 2>&1 || true
|
||||
|
||||
if [[ ! -s "$tmp" ]]; then
|
||||
echo "replication_status_available: no"
|
||||
echo
|
||||
rm -f "$tmp"
|
||||
return
|
||||
fi
|
||||
|
||||
awk -F'\t' '
|
||||
NF >= 12 {
|
||||
print "Slave_IO_State:", $1
|
||||
print "Master_Host: [REDACTED]"
|
||||
print "Master_User: [REDACTED]"
|
||||
print "Master_Port:", $4
|
||||
print "Master_Log_File:", $6
|
||||
print "Read_Master_Log_Pos:", $7
|
||||
print "Relay_Log_File:", $8
|
||||
print "Relay_Log_Pos:", $9
|
||||
print "Relay_Master_Log_File:", $10
|
||||
print "Slave_IO_Running:", $11
|
||||
print "Slave_SQL_Running:", $12
|
||||
found=1
|
||||
next
|
||||
}
|
||||
{
|
||||
print
|
||||
}
|
||||
END {
|
||||
if (!found) {
|
||||
print "replication_status_parse: raw fallback"
|
||||
}
|
||||
}
|
||||
' "$tmp" || true
|
||||
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
# Parses the MariaDB slow query log since $diagSince and appends a fingerprint summary to the diag file.
|
||||
function diagMariaDBSlowSummary() {
|
||||
local mariadbMasterSlowLog="$mariadbMasterPath/slow.log"
|
||||
local cutoff
|
||||
|
||||
cutoff="$(diagCutoffEpoch)"
|
||||
|
||||
diagLogSection "MariaDB slow query summary only"
|
||||
|
||||
{
|
||||
echo "+ summarize slow log: $mariadbMasterSlowLog since ${diagSince:-4h}"
|
||||
echo
|
||||
|
||||
if [[ ! -r "$mariadbMasterSlowLog" ]]; then
|
||||
echo "slow_log_readable: no"
|
||||
echo
|
||||
return
|
||||
fi
|
||||
|
||||
perl -MTime::Local - "$mariadbMasterSlowLog" "$cutoff" <<'PERL'
|
||||
use strict;
|
||||
use warnings;
|
||||
use Time::Local;
|
||||
|
||||
my ($file, $cutoff) = @ARGV;
|
||||
|
||||
open my $fh, '<', $file or do {
|
||||
print "slow_log_readable: no\n";
|
||||
exit 0;
|
||||
};
|
||||
|
||||
my %fp;
|
||||
my $total = 0;
|
||||
my $max_qt = 0;
|
||||
my $max_rows_examined = 0;
|
||||
my $max_rows_sent = 0;
|
||||
|
||||
my ($active, $ts, $qt, $rows_sent, $rows_examined, $sql);
|
||||
|
||||
sub reset_entry {
|
||||
$active = 0;
|
||||
$ts = 0;
|
||||
$qt = 0;
|
||||
$rows_sent = 0;
|
||||
$rows_examined = 0;
|
||||
$sql = '';
|
||||
}
|
||||
|
||||
sub fingerprint_sql {
|
||||
my ($s) = @_;
|
||||
|
||||
$s =~ s/`[^`]*`/`X`/g;
|
||||
$s =~ s/'(?:\\.|[^'\\])*'/'X'/g;
|
||||
$s =~ s/"(?:\\.|[^"\\])*"/"X"/g;
|
||||
$s =~ s/\b[0-9a-fA-F]{16,}\b/HEX/g;
|
||||
$s =~ s/\b\d+(?:\.\d+)?\b/N/g;
|
||||
$s =~ s/\s+/ /g;
|
||||
$s =~ s/^\s+|\s+$//g;
|
||||
|
||||
return substr($s, 0, 220);
|
||||
}
|
||||
|
||||
sub flush_entry {
|
||||
return unless $active;
|
||||
return if $ts < $cutoff;
|
||||
|
||||
$total++;
|
||||
$max_qt = $qt if $qt > $max_qt;
|
||||
$max_rows_examined = $rows_examined if $rows_examined > $max_rows_examined;
|
||||
$max_rows_sent = $rows_sent if $rows_sent > $max_rows_sent;
|
||||
|
||||
my $f = fingerprint_sql($sql);
|
||||
return unless length $f;
|
||||
|
||||
$fp{$f}{count}++;
|
||||
$fp{$f}{total_time} += $qt;
|
||||
$fp{$f}{total_rows_examined} += $rows_examined;
|
||||
$fp{$f}{max_time} = $qt if !defined($fp{$f}{max_time}) || $qt > $fp{$f}{max_time};
|
||||
}
|
||||
|
||||
reset_entry();
|
||||
|
||||
while (my $line = <$fh>) {
|
||||
chomp $line;
|
||||
|
||||
if ($line =~ /^# Time:\s*(\d{2})(\d{2})(\d{2})\s+(\d{1,2}):(\d{2}):(\d{2})/) {
|
||||
flush_entry();
|
||||
reset_entry();
|
||||
|
||||
my ($yy, $mo, $dd, $hh, $mm, $ss) = ($1, $2, $3, $4, $5, $6);
|
||||
my $yyyy = $yy >= 70 ? 1900 + $yy : 2000 + $yy;
|
||||
|
||||
$ts = timelocal($ss, $mm, $hh, $dd, $mo - 1, $yyyy);
|
||||
$active = 1;
|
||||
next;
|
||||
}
|
||||
|
||||
next unless $active;
|
||||
|
||||
if ($line =~ /^# Query_time:\s*([0-9.]+).*Rows_sent:\s*([0-9]+).*Rows_examined:\s*([0-9]+)/) {
|
||||
$qt = $1 + 0;
|
||||
$rows_sent = $2 + 0;
|
||||
$rows_examined = $3 + 0;
|
||||
next;
|
||||
}
|
||||
|
||||
next if $line =~ /^# User\@Host:/;
|
||||
next if $line =~ /^use `/;
|
||||
next if $line =~ /^SET timestamp=/;
|
||||
next if $line =~ /^#/;
|
||||
next if $line =~ /^\s*$/;
|
||||
|
||||
$sql .= ' ' . $line;
|
||||
}
|
||||
|
||||
flush_entry();
|
||||
|
||||
print "slow_queries_count: $total\n";
|
||||
print "max_query_time: $max_qt\n";
|
||||
print "max_rows_examined: $max_rows_examined\n";
|
||||
print "max_rows_sent: $max_rows_sent\n";
|
||||
print "\n";
|
||||
print "top_fingerprints:\n";
|
||||
|
||||
my $shown = 0;
|
||||
for my $f (
|
||||
sort {
|
||||
$fp{$b}{total_time} <=> $fp{$a}{total_time}
|
||||
||
|
||||
$fp{$b}{total_rows_examined} <=> $fp{$a}{total_rows_examined}
|
||||
} keys %fp
|
||||
) {
|
||||
last if $shown++ >= 10;
|
||||
|
||||
printf(
|
||||
"- count=%d total_time=%.3f max_time=%.3f total_rows_examined=%d sql=%s\n",
|
||||
$fp{$f}{count},
|
||||
$fp{$f}{total_time},
|
||||
$fp{$f}{max_time},
|
||||
$fp{$f}{total_rows_examined},
|
||||
$f
|
||||
);
|
||||
}
|
||||
PERL
|
||||
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Appends lsphp process count and RSS stats from an OLS pod to the diag file.
|
||||
# $1 (pod): OLS pod name.
|
||||
function diagOlsPodStats() {
|
||||
local pod="$1"
|
||||
|
||||
diagLogSection "OLS $pod lsphp count and RSS"
|
||||
|
||||
{
|
||||
echo "+ k3sRun exec pod/$pod -c openlitespeed -- sh -s <lsphp stats>"
|
||||
|
||||
k3sRun exec -i "pod/$pod" -c openlitespeed -- sh -s <<'SH' || true
|
||||
tmp="$(mktemp)"
|
||||
|
||||
echo "hostname:"
|
||||
hostname 2>/dev/null || true
|
||||
echo
|
||||
|
||||
ps -eo user=,rss=,comm=,args= > "$tmp" 2>/dev/null || ps aux > "$tmp" 2>/dev/null || true
|
||||
|
||||
echo "lsphp count:"
|
||||
awk '
|
||||
/lsphp/ && $0 !~ /awk|grep/ { c++ }
|
||||
END { print c + 0 }
|
||||
' "$tmp"
|
||||
echo
|
||||
|
||||
echo "total lsphp RSS KiB:"
|
||||
awk '
|
||||
/lsphp/ && $0 !~ /awk|grep/ { sum += $2 }
|
||||
END { print sum + 0 }
|
||||
' "$tmp"
|
||||
echo
|
||||
|
||||
echo "total lsphp RSS MiB:"
|
||||
awk '
|
||||
/lsphp/ && $0 !~ /awk|grep/ { sum += $2 }
|
||||
END { printf "%.1f\n", sum / 1024 }
|
||||
' "$tmp"
|
||||
echo
|
||||
|
||||
echo "top RSS lsphp:"
|
||||
awk '
|
||||
/lsphp/ && $0 !~ /awk|grep/ {
|
||||
print
|
||||
}
|
||||
' "$tmp" | sort -k2,2nr | head -10
|
||||
echo
|
||||
|
||||
echo "lsphp by user:"
|
||||
awk '
|
||||
/lsphp/ && $0 !~ /awk|grep/ {
|
||||
count[$1]++
|
||||
rss[$1] += $2
|
||||
}
|
||||
END {
|
||||
for (u in count) {
|
||||
printf "%s count=%d rss_kib=%d rss_mib=%.1f\n", u, count[u], rss[u], rss[u] / 1024
|
||||
}
|
||||
}
|
||||
' "$tmp" | sort -k3,3nr
|
||||
echo
|
||||
|
||||
echo "process count:"
|
||||
awk '
|
||||
NF >= 3 {
|
||||
comm=$3
|
||||
count[comm]++
|
||||
}
|
||||
END {
|
||||
for (c in count) {
|
||||
print count[c], c
|
||||
}
|
||||
}
|
||||
' "$tmp" | sort -nr | head -20
|
||||
echo
|
||||
|
||||
rm -f "$tmp"
|
||||
SH
|
||||
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Parses the OLS error log since $diagSince and appends categorized error counters to the diag file.
|
||||
function diagOlsErrorSummary() {
|
||||
local openlitespeedErrorLog="$olsLogsPath/error.log"
|
||||
local cutoff
|
||||
|
||||
cutoff="$(diagCutoffEpoch)"
|
||||
|
||||
diagLogSection "OLS error counters summary"
|
||||
|
||||
{
|
||||
echo "+ summarize OLS error log: $openlitespeedErrorLog since ${diagSince:-4h}"
|
||||
echo
|
||||
|
||||
if [[ ! -r "$openlitespeedErrorLog" ]]; then
|
||||
echo "ols_error_log_readable: no"
|
||||
echo
|
||||
return
|
||||
fi
|
||||
|
||||
perl -MTime::Local - "$openlitespeedErrorLog" "$cutoff" <<'PERL'
|
||||
use strict;
|
||||
use warnings;
|
||||
use Time::Local;
|
||||
|
||||
my ($file, $cutoff) = @ARGV;
|
||||
|
||||
open my $fh, '<', $file or do {
|
||||
print "ols_error_log_readable: no\n";
|
||||
exit 0;
|
||||
};
|
||||
|
||||
my %c = (
|
||||
total_lines => 0,
|
||||
warn_count => 0,
|
||||
error_count => 0,
|
||||
hostname_mapping_conflicts => 0,
|
||||
inaccessible_vhost_root => 0,
|
||||
no_request_delivery => 0,
|
||||
connection_reset => 0,
|
||||
memory_errors => 0,
|
||||
too_many_open_files => 0,
|
||||
permission_denied => 0,
|
||||
);
|
||||
|
||||
while (my $line = <$fh>) {
|
||||
my $ts = 0;
|
||||
|
||||
if ($line =~ /^(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})/) {
|
||||
$ts = timelocal($6, $5, $4, $3, $2 - 1, $1);
|
||||
}
|
||||
|
||||
next if $ts && $ts < $cutoff;
|
||||
|
||||
$c{total_lines}++;
|
||||
$c{warn_count}++ if $line =~ /\[WARN\]/;
|
||||
$c{error_count}++ if $line =~ /\[ERROR\]/;
|
||||
$c{hostname_mapping_conflicts}++ if $line =~ /Hostname .* is mapped to virtual host .* can.t map to virtual host/;
|
||||
$c{inaccessible_vhost_root}++ if $line =~ /Path for vhost root is not accessible/;
|
||||
$c{no_request_delivery}++ if $line =~ /No request delivery notification has been received/;
|
||||
$c{connection_reset}++ if $line =~ /Connection reset by peer/;
|
||||
$c{memory_errors}++ if $line =~ /OOM|out of memory|Cannot allocate memory/i;
|
||||
$c{too_many_open_files}++ if $line =~ /Too many open files/i;
|
||||
$c{permission_denied}++ if $line =~ /Permission denied/i;
|
||||
}
|
||||
|
||||
for my $k (
|
||||
qw(
|
||||
total_lines
|
||||
warn_count
|
||||
error_count
|
||||
hostname_mapping_conflicts
|
||||
inaccessible_vhost_root
|
||||
no_request_delivery
|
||||
connection_reset
|
||||
memory_errors
|
||||
too_many_open_files
|
||||
permission_denied
|
||||
)
|
||||
) {
|
||||
print "$k: $c{$k}\n";
|
||||
}
|
||||
PERL
|
||||
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Fetches the OPcache status endpoint four times and appends key metrics to the diag file.
|
||||
function diagOpcacheSummary() {
|
||||
local i tmp
|
||||
|
||||
diagLogSection "OPcache summary"
|
||||
|
||||
{
|
||||
echo "+ curl OPcache endpoint summary"
|
||||
echo "attempts: 4"
|
||||
echo
|
||||
|
||||
for i in 1 2 3 4; do
|
||||
tmp="$(mktemp)"
|
||||
|
||||
curl -kfsS --max-time 10 "$diagOpcacheUrl" > "$tmp" 2>/dev/null || {
|
||||
echo "attempt_$i: failed"
|
||||
rm -f "$tmp"
|
||||
continue
|
||||
}
|
||||
|
||||
echo "attempt_$i:"
|
||||
|
||||
awk '
|
||||
function val(line) {
|
||||
sub(/^.*=>[[:space:]]*/, "", line)
|
||||
if (line == "" || line == "=>") return "false"
|
||||
return line
|
||||
}
|
||||
|
||||
/\[memory_usage\]/ { ctx="memory"; next }
|
||||
/\[interned_strings_usage\]/ { ctx="interned"; next }
|
||||
/\[opcache_statistics\]/ { ctx="stats"; next }
|
||||
/\[jit\]/ { ctx="jit"; next }
|
||||
|
||||
/\[opcache_enabled\]/ { print " opcache_enabled:", val($0) }
|
||||
/\[cache_full\]/ { print " cache_full:", val($0) }
|
||||
/\[restart_pending\]/ { print " restart_pending:", val($0) }
|
||||
/\[restart_in_progress\]/ { print " restart_in_progress:", val($0) }
|
||||
|
||||
ctx=="memory" && /\[used_memory\]/ { print " memory_used:", val($0) }
|
||||
ctx=="memory" && /\[free_memory\]/ { print " memory_free:", val($0) }
|
||||
ctx=="memory" && /\[wasted_memory\]/ { print " memory_wasted:", val($0) }
|
||||
ctx=="memory" && /\[current_wasted_percentage\]/ { print " memory_wasted_pct:", val($0) }
|
||||
|
||||
ctx=="interned" && /\[buffer_size\]/ { print " interned_buffer_size:", val($0) }
|
||||
ctx=="interned" && /\[used_memory\]/ { print " interned_used:", val($0) }
|
||||
ctx=="interned" && /\[free_memory\]/ { print " interned_free:", val($0) }
|
||||
ctx=="interned" && /\[number_of_strings\]/ { print " interned_strings:", val($0) }
|
||||
|
||||
ctx=="stats" && /\[num_cached_scripts\]/ { print " num_cached_scripts:", val($0) }
|
||||
ctx=="stats" && /\[num_cached_keys\]/ { print " num_cached_keys:", val($0) }
|
||||
ctx=="stats" && /\[max_cached_keys\]/ { print " max_cached_keys:", val($0) }
|
||||
ctx=="stats" && /\[hits\]/ { print " hits:", val($0) }
|
||||
ctx=="stats" && /\[misses\]/ { print " misses:", val($0) }
|
||||
ctx=="stats" && /\[oom_restarts\]/ { print " oom_restarts:", val($0) }
|
||||
ctx=="stats" && /\[hash_restarts\]/ { print " hash_restarts:", val($0) }
|
||||
ctx=="stats" && /\[manual_restarts\]/ { print " manual_restarts:", val($0) }
|
||||
ctx=="stats" && /\[opcache_hit_rate\]/ { print " opcache_hit_rate:", val($0) }
|
||||
' "$tmp" || true
|
||||
|
||||
rm -f "$tmp"
|
||||
echo
|
||||
done
|
||||
} >> "$diagFile" 2>&1
|
||||
}
|
||||
|
||||
# Collects system-level diagnostics (uptime, memory, vmstat, iostat, PSI, sar, disk usage).
|
||||
function diagSystem() {
|
||||
diagLogSection "uptime"
|
||||
diagCmd uptime
|
||||
|
||||
diagLogSection "free -m"
|
||||
diagCmd free -m
|
||||
|
||||
diagLogSection "vmstat 1 10"
|
||||
diagCmd vmstat 1 10
|
||||
|
||||
diagLogSection "mpstat 1 10"
|
||||
diagCmd mpstat 1 10
|
||||
|
||||
diagLogSection "iostat -x -z 1 5 filtered"
|
||||
diagIostat
|
||||
|
||||
diagLogSection "cat /proc/pressure/cpu"
|
||||
diagCmd cat /proc/pressure/cpu
|
||||
|
||||
diagLogSection "cat /proc/pressure/memory"
|
||||
diagCmd cat /proc/pressure/memory
|
||||
|
||||
diagLogSection "cat /proc/pressure/io"
|
||||
diagCmd cat /proc/pressure/io
|
||||
|
||||
diagSarTry "sar queue last ${diagSince:-4h}" "sar -q -s '$(diagSarStartTime)'"
|
||||
diagSarTry "sar cpu last ${diagSince:-4h}" "sar -u -s '$(diagSarStartTime)'"
|
||||
diagSarTry "sar memory last ${diagSince:-4h}" "sar -r -s '$(diagSarStartTime)'"
|
||||
diagSarTry "sar swap last ${diagSince:-4h}" "sar -W -s '$(diagSarStartTime)'"
|
||||
|
||||
diagLogSection "Filesystem pressure quick checks"
|
||||
diagDfUsage
|
||||
|
||||
diagLogSection "Filesystem inode quick checks"
|
||||
diagDfInodes
|
||||
}
|
||||
|
||||
# Collects k3s diagnostics (pod top/list, warning events, restart summary, node conditions).
|
||||
function diagK3s() {
|
||||
diagLogSection "Top pod"
|
||||
diagK3sTry top pod
|
||||
|
||||
diagLogSection "Get pod"
|
||||
diagK3sTry get pod
|
||||
|
||||
diagLogSection "Get warning events recent"
|
||||
diagK3sTry get events --field-selector type!=Normal --sort-by=.lastTimestamp
|
||||
|
||||
diagLogSection "Pod restart summary"
|
||||
diagK3sTry get pod -o custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,RESTARTS:.status.containerStatuses[*].restartCount,STATE:.status.containerStatuses[*].state.waiting.reason,LAST_STATE:.status.containerStatuses[*].lastState.terminated.reason
|
||||
|
||||
diagLogSection "Node conditions and allocated resources"
|
||||
diagK3sNodeSummary
|
||||
}
|
||||
|
||||
# Collects MariaDB diagnostics (master/slave global status, replication summary, slow queries).
|
||||
function diagMariaDB() {
|
||||
diagLogSection "MariaDB master global status"
|
||||
{
|
||||
echo "+ mariadbMasterRootQuery"
|
||||
mariadbMasterRootQuery "
|
||||
SHOW GLOBAL STATUS WHERE Variable_name IN (
|
||||
'Uptime',
|
||||
'Threads_running',
|
||||
'Threads_connected',
|
||||
'Max_used_connections',
|
||||
'Connections',
|
||||
'Aborted_connects',
|
||||
'Slow_queries',
|
||||
'Questions',
|
||||
'Queries',
|
||||
'Created_tmp_tables',
|
||||
'Created_tmp_disk_tables',
|
||||
'Innodb_buffer_pool_reads',
|
||||
'Innodb_buffer_pool_read_requests',
|
||||
'Innodb_data_reads',
|
||||
'Innodb_data_writes',
|
||||
'Innodb_log_waits',
|
||||
'Open_tables',
|
||||
'Opened_tables',
|
||||
'Table_open_cache_hits',
|
||||
'Table_open_cache_misses',
|
||||
'Table_open_cache_overflows'
|
||||
);
|
||||
" || true
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
|
||||
diagLogSection "MariaDB slave global status"
|
||||
{
|
||||
echo "+ mariadbSlaveRootQuery"
|
||||
mariadbSlaveRootQuery "
|
||||
SHOW GLOBAL STATUS WHERE Variable_name IN (
|
||||
'Uptime',
|
||||
'Threads_running',
|
||||
'Threads_connected',
|
||||
'Max_used_connections',
|
||||
'Connections',
|
||||
'Aborted_connects',
|
||||
'Slow_queries',
|
||||
'Questions',
|
||||
'Queries',
|
||||
'Created_tmp_tables',
|
||||
'Created_tmp_disk_tables',
|
||||
'Innodb_buffer_pool_reads',
|
||||
'Innodb_buffer_pool_read_requests',
|
||||
'Innodb_data_reads',
|
||||
'Innodb_data_writes',
|
||||
'Innodb_log_waits',
|
||||
'Slave_running',
|
||||
'Slave_received_heartbeats',
|
||||
'Slave_heartbeat_period',
|
||||
'Slave_open_temp_tables'
|
||||
);
|
||||
" || true
|
||||
echo
|
||||
} >> "$diagFile" 2>&1
|
||||
|
||||
diagMariaDBReplicaSummary
|
||||
diagMariaDBSlowSummary
|
||||
}
|
||||
|
||||
# Collects OpenLiteSpeed diagnostics (pod list, per-pod lsphp stats, error summary, OPcache).
|
||||
function diagOpenLiteSpeed() {
|
||||
local podList pod
|
||||
|
||||
diagLogSection "OpenLiteSpeed pods"
|
||||
diagK3sTry get pod -l app=openlitespeed
|
||||
|
||||
podList="$(k3sPodListApp openlitespeed 2>/dev/null || true)"
|
||||
while read -r pod; do
|
||||
[[ -z "$pod" ]] && continue
|
||||
diagOlsPodStats "$pod"
|
||||
done < <(awk 'NF' <<< "$podList")
|
||||
|
||||
diagOlsErrorSummary
|
||||
diagOpcacheSummary
|
||||
}
|
||||
|
||||
# Generates a full diagnostic report and writes it to $diagFile.
|
||||
# [$1] (diagFile): output file path (defaults to $diagFile).
|
||||
function diagReport() {
|
||||
local diagFile="${1:-$diagFile}"
|
||||
local aiModel="${1:-short}"
|
||||
|
||||
export LC_ALL=C
|
||||
export LANG=C
|
||||
|
||||
local reportPath
|
||||
reportPath=$(dirname "$diagFile")
|
||||
[[ -d "$reportPath" ]] || mkdir -p "$reportPath"
|
||||
|
||||
if [[ "$postfixHost" ]]; then
|
||||
diagExcludeVhostsRegex="$diagExcludeVhostsRegex|${postfixHost//./\\.}"
|
||||
fi
|
||||
|
||||
: > "$diagFile"
|
||||
diagMeta
|
||||
diagSystem
|
||||
diagK3s
|
||||
diagMariaDB
|
||||
diagOpenLiteSpeed
|
||||
diagSanitizeReport
|
||||
}
|
||||
|
||||
# Uploads the diag report file to the remote API.
|
||||
# [$1] (diagFile): report file path (defaults to $diagFile).
|
||||
# [$2] (aiModelId): AI endpoint ID to use (defaults to 1).
|
||||
function diagSend() {
|
||||
local diagFile="${1:-$diagFile}"
|
||||
local aiModelId="${2:-1}"
|
||||
|
||||
logInfo "$diagLabel Sending data..."
|
||||
curl -fsS -X POST "$diagApiUrl" -F "source_type=worker" -F "source_id=$hostUuid" -F "generated_time=$(date +%s)" -F "report_file=@$diagFile" -F "ai_endpoint_id=$aiModelId"
|
||||
logSuccess "$diagLabel Data sent successfully"
|
||||
}
|
||||
|
||||
# Generates the full diagnostic report and sends it to the remote API.
|
||||
# [$1] (aiModel): model name: full or short (defaults to short).
|
||||
function diagRun() {
|
||||
local aiModel="${1:-short}"
|
||||
local aiModelId=2
|
||||
|
||||
case "$aiModel" in
|
||||
full)
|
||||
aiModelId=1
|
||||
;;
|
||||
*)
|
||||
aiModelId=2
|
||||
;;
|
||||
esac
|
||||
|
||||
diagReport "$diagFile"
|
||||
diagSend "$diagFile" "$aiModelId"
|
||||
}
|
||||
Reference in New Issue
Block a user