184 lines
8.2 KiB
Bash
184 lines
8.2 KiB
Bash
# Restarts the vmagent deployment; errors are intentionally ignored.
|
|
function metricRestart() {
|
|
k3sRun rollout restart deployment/"$metricKube"-vmagent || true
|
|
}
|
|
|
|
# Writes Prometheus metrics (build info, vhost counts, domain counts, pod memory) to $metricPromPath/kube.prom.
|
|
function metricKube() {
|
|
local fileProm="$metricPromPath/kube.prom"
|
|
local fileTmp="${fileProm}.tmp"
|
|
|
|
mkdir -p -- "$metricPromPath" || return 1
|
|
|
|
local vhostAllCount vhostUpCount
|
|
vhostAllCount=$(openlitespeedConfigVhostList all | grep -c . || true)
|
|
vhostUpCount=$(openlitespeedConfigVhostList up | grep -c . || true)
|
|
|
|
local appTimestamp
|
|
appTimestamp=$(date -d "$appDate ${appTime//-/:}" +%s)
|
|
|
|
local domainAllCount=-1 domainFreeCount=-1
|
|
[[ -f "$domainsList" ]] && domainAllCount=$(grep -cP '^(?!\s*$)' "$domainsList" || true)
|
|
[[ -f "$domainsList" ]] && domainFreeCount=$(grep -c '^#' "$domainsList" || true)
|
|
|
|
local masterMemReq=-1 masterMemLim=-1
|
|
local slaveMemReq=-1 slaveMemLim=-1
|
|
local olsMemReq=-1 olsMemLim=-1
|
|
local redisMemReq=-1 redisMemLim=-1
|
|
local _line _req _lim
|
|
|
|
_line=$(k3sRun get pod "${mariadbMasterKube}-0" \
|
|
-o jsonpath="{.spec.containers[?(@.name=='${mariadbMasterKube}')].resources.requests.memory} {.spec.containers[?(@.name=='${mariadbMasterKube}')].resources.limits.memory}" 2>/dev/null)
|
|
read -r _req _lim <<< "$_line"
|
|
masterMemReq=$(sizeToBytes "$_req"); masterMemLim=$(sizeToBytes "$_lim")
|
|
|
|
_line=$(k3sRun get pod "${mariadbSlaveKube}-0" \
|
|
-o jsonpath="{.spec.containers[?(@.name=='${mariadbSlaveKube}')].resources.requests.memory} {.spec.containers[?(@.name=='${mariadbSlaveKube}')].resources.limits.memory}" 2>/dev/null)
|
|
read -r _req _lim <<< "$_line"
|
|
slaveMemReq=$(sizeToBytes "$_req"); slaveMemLim=$(sizeToBytes "$_lim")
|
|
|
|
_line=$(k3sRun get pods -l "app=$olsKube" \
|
|
-o jsonpath="{.items[0].spec.containers[?(@.name=='${olsKube}')].resources.requests.memory} {.items[0].spec.containers[?(@.name=='${olsKube}')].resources.limits.memory}" 2>/dev/null)
|
|
read -r _req _lim <<< "$_line"
|
|
olsMemReq=$(sizeToBytes "$_req"); olsMemLim=$(sizeToBytes "$_lim")
|
|
|
|
_line=$(k3sRun get pod "${redisKube}-0" \
|
|
-o jsonpath="{.spec.containers[?(@.name=='${redisKube}')].resources.requests.memory} {.spec.containers[?(@.name=='${redisKube}')].resources.limits.memory}" 2>/dev/null)
|
|
read -r _req _lim <<< "$_line"
|
|
redisMemReq=$(sizeToBytes "$_req"); redisMemLim=$(sizeToBytes "$_lim")
|
|
|
|
cat > "$fileTmp" <<EOF
|
|
# HELP kube_build_info Build and version info
|
|
# TYPE kube_build_info gauge
|
|
kube_build_info{version="${appVersion}"} 1
|
|
|
|
# HELP kube_start_time Time snapshot
|
|
# TYPE kube_start_time gauge
|
|
kube_start_time $appTimestamp
|
|
|
|
# HELP kube_vhost_all_count Number of virtual hosts
|
|
# TYPE kube_vhost_all_count gauge
|
|
kube_vhost_all_count $vhostAllCount
|
|
|
|
# HELP kube_vhost_up_count Number of active virtual hosts
|
|
# TYPE kube_vhost_up_count gauge
|
|
kube_vhost_up_count $vhostUpCount
|
|
|
|
# HELP kube_domain_all_count Number of domains
|
|
# TYPE kube_domain_all_count gauge
|
|
kube_domain_all_count $domainAllCount
|
|
|
|
# HELP kube_domain_use_count Number of use domains
|
|
# TYPE kube_domain_use_count gauge
|
|
kube_domain_use_count $domainFreeCount
|
|
|
|
# HELP kube_pod_memory_request_bytes Pod memory request in bytes
|
|
# TYPE kube_pod_memory_request_bytes gauge
|
|
kube_pod_memory_request_bytes{component="mariadb-master"} $masterMemReq
|
|
kube_pod_memory_request_bytes{component="mariadb-slave"} $slaveMemReq
|
|
kube_pod_memory_request_bytes{component="openlitespeed"} $olsMemReq
|
|
kube_pod_memory_request_bytes{component="redis"} $redisMemReq
|
|
|
|
# HELP kube_pod_memory_limit_bytes Pod memory limit in bytes
|
|
# TYPE kube_pod_memory_limit_bytes gauge
|
|
kube_pod_memory_limit_bytes{component="mariadb-master"} $masterMemLim
|
|
kube_pod_memory_limit_bytes{component="mariadb-slave"} $slaveMemLim
|
|
kube_pod_memory_limit_bytes{component="openlitespeed"} $olsMemLim
|
|
kube_pod_memory_limit_bytes{component="redis"} $redisMemLim
|
|
EOF
|
|
mv "$fileTmp" "$fileProm"
|
|
}
|
|
|
|
# Collects lsphp CPU/memory/worker metrics per domain across OLS pods and writes to $metricPromPath/lsphp.prom.
|
|
function metricLsphp() {
|
|
local fileProm="$metricPromPath/lsphp.prom"
|
|
local fileTmp="${fileProm}.tmp"
|
|
|
|
mkdir -p -- "$metricPromPath" || return 1
|
|
|
|
local podList error
|
|
run podList error k3sPodListStatus "$olsKube" || return 1
|
|
|
|
local pod ready phase reason output cpu mem count domain
|
|
{
|
|
printf '# HELP lsphp_cpu_percent Total CPU percent used by lsphp workers per domain\n'
|
|
printf '# TYPE lsphp_cpu_percent gauge\n'
|
|
printf '# HELP lsphp_memory_percent Total memory percent used by lsphp workers per domain\n'
|
|
printf '# TYPE lsphp_memory_percent gauge\n'
|
|
printf '# HELP lsphp_worker_count Number of lsphp worker processes per domain\n'
|
|
printf '# TYPE lsphp_worker_count gauge\n'
|
|
|
|
while IFS='|' read -r pod ready phase reason; do
|
|
[[ "$ready" == "1" ]] || continue
|
|
# run output error openlitespeedPodExec "$pod" sh -c "ps aux | grep lsphp | grep -v grep | awk '{u=\$1;cpu[u]+=\$3;mem[u]+=\$4;count[u]++} END {for(u in cpu){name=u;cmd=\"find $olsPodPrivatePath -maxdepth 1 -uid \"u\" -type d 2>/dev/null\";if((cmd|getline dir)>0&&dir!=\"\"){n=split(dir,a,\"/\");name=a[n]};close(cmd);print cpu[u],mem[u],count[u],name}}'" || continue
|
|
run output error openlitespeedPodExec "$pod" sh -c "ps aux | awk '/lsphp/ && !/nobody/ {u=\$1;cpu[u]+=\$3;mem[u]+=\$4;count[u]++} END {for(u in cpu){name=u;cmd=\"find $olsPodPrivatePath -maxdepth 1 -uid \"u\" -type d 2>/dev/null\";if((cmd|getline dir)>0&&dir!=\"\"){n=split(dir,a,\"/\");name=a[n]};close(cmd);printf \"%.1f\\t%.1f\\t%d\\t%s\\n\",cpu[u],mem[u],count[u],name}}'" || continue
|
|
|
|
while IFS=$'\t' read -r cpu mem count domain; do
|
|
[[ -n "$domain" ]] || continue
|
|
printf 'lsphp_cpu_percent{domain="%s",pod="%s"} %s\n' "$domain" "$pod" "$cpu"
|
|
printf 'lsphp_memory_percent{domain="%s",pod="%s"} %s\n' "$domain" "$pod" "$mem"
|
|
printf 'lsphp_worker_count{domain="%s",pod="%s"} %s\n' "$domain" "$pod" "$count"
|
|
done <<< "$output"
|
|
done < <(awk 'NF' <<< "$podList")
|
|
} > "$fileTmp"
|
|
|
|
mv "$fileTmp" "$fileProm"
|
|
}
|
|
|
|
# Collects disk usage per site and sends metrics to the API.
|
|
function metricSites() {
|
|
local metricSitesApiUrl="https://api.sodew.ai/api/metrics/sites"
|
|
local batchId batchTime metricsJson dataJson payload httpCode
|
|
batchId="$(uuidgen | tr '[:upper:]' '[:lower:]')"
|
|
batchTime="$(date +%s)"
|
|
metricsJson='{
|
|
"site_disk_usage_mb": {"type":"number"}
|
|
}'
|
|
dataJson='{}'
|
|
|
|
local error vhostList
|
|
run vhostList error openlitespeedConfigVhostList || {
|
|
appError "Error retrieving vhost list: $error"
|
|
return 1
|
|
}
|
|
while IFS= read -r domain <&3; do
|
|
local diskUsage domainJson
|
|
diskUsage="$(pathSize "$olsVhostsPath/$domain" "mb" || true)"
|
|
[[ -n "$diskUsage" ]] || continue
|
|
|
|
domainJson="$(
|
|
jq -n --arg diskUsage "$diskUsage" \
|
|
'{
|
|
site_disk_usage_mb: { value: (if ($diskUsage | length) > 0 then ($diskUsage | tonumber) else null end) },
|
|
}'
|
|
)"
|
|
dataJson="$(jq --arg domain "$domain" --argjson row "$domainJson" '. + {($domain): $row}' <<< "$dataJson")"
|
|
done 3< <(awk 'NF' <<< "$vhostList")
|
|
|
|
payload="$(
|
|
jq -n \
|
|
--arg source_type "worker" \
|
|
--arg source_id "$hostUuid" \
|
|
--arg batch_id "$batchId" \
|
|
--argjson batch_time "$batchTime" \
|
|
--argjson metrics "$metricsJson" \
|
|
--argjson data "$dataJson" \
|
|
'{
|
|
source_type: $source_type,
|
|
source_id: $source_id,
|
|
batch_id: $batch_id,
|
|
batch_time: $batch_time,
|
|
metrics: $metrics,
|
|
data: $data
|
|
}'
|
|
)"
|
|
|
|
# Sending data...
|
|
httpCode="$(curl -sS -o /tmp/metrics_sites_response.txt -w '%{http_code}' -X POST "$metricSitesApiUrl" -H 'Content-Type: application/json' --data-binary "$payload")"
|
|
if [[ "$httpCode" != "204" ]]; then
|
|
appError "Ingest failed, HTTP $httpCode"
|
|
cat /tmp/metrics_sites_response.txt >&2 || true
|
|
return 1
|
|
fi
|
|
}
|