374 lines
14 KiB
Bash
374 lines
14 KiB
Bash
redisLabel="[Redis]"
|
|
|
|
# Prepares Kubernetes secrets for Redis.
|
|
function cmdRedisPreparation() {
|
|
printSection "Preparation..."
|
|
|
|
local error
|
|
runError error k3sRun delete secret "$redisKube-secret" --ignore-not-found || {
|
|
printDotText "preparation" "$labelFail"
|
|
printDanger "Delete old Secret: $error"
|
|
return 1
|
|
}
|
|
|
|
runError error k3sRun create secret generic "$redisKube-secret" --from-literal=root-password="$redisRootPass" || {
|
|
printDotText "preparation" "$labelFail"
|
|
printDanger "Create new Secret: $error"
|
|
return 1
|
|
}
|
|
|
|
fileBackup "$redisPath/$redisFileUsersAcl"
|
|
|
|
printDotText "preparation" "$labelDone"
|
|
}
|
|
|
|
# Renders the Redis Kubernetes YAML manifest via Helm.
|
|
function cmdRedisYamlRender() {
|
|
local templateFile="templates/$redisKube.yaml"
|
|
|
|
printSection "Render YAML file | Helm"
|
|
printDotText "Template" "$appAssetsPath/k3s/$templateFile"
|
|
[[ -f "$appAssetsPath/k3s/$templateFile" ]] || {
|
|
printDanger "Template file not found"
|
|
return 1
|
|
}
|
|
|
|
local profileCpuFile="$appAssetsPath/k3s/profiles/cpu-$kubeCpuProfile.yaml"
|
|
printDotText "CPU profile" "$profileCpuFile"
|
|
[[ -f "$profileCpuFile" ]] || {
|
|
printDanger "CPU profile file not found"
|
|
return 1
|
|
}
|
|
|
|
local profileMemoryFile="$appAssetsPath/k3s/profiles/memory-$kubeMemoryProfile.yaml"
|
|
printDotText "Memory profile" "$profileMemoryFile"
|
|
[[ -f "$profileMemoryFile" ]] || {
|
|
printDanger "Memory profile file not found"
|
|
return 1
|
|
}
|
|
|
|
local varsFile
|
|
varsFile=$(mktemp "/tmp/$redisKube.vars.XXXXXX.yaml") || {
|
|
printDotText "render" "$labelFail"
|
|
printDanger "Create temp variables file"
|
|
return 1
|
|
}
|
|
printDotText "Variables" "$varsFile"
|
|
trap 'rm -f -- "$varsFile"' RETURN
|
|
cat > "$varsFile" <<EOF
|
|
redisHelm: true
|
|
namespace: $k3sNamespace
|
|
redis: $redisKube
|
|
redisSentinel: $redisSentinelKube
|
|
redisPath: $redisPath
|
|
redisFileUsersAcl: $redisFileUsersAcl
|
|
EOF
|
|
|
|
printDotText "Result" "$redisYaml"
|
|
mkdir -p -- "$(dirname -- "$redisYaml")" || return 1
|
|
fileBackup "$redisYaml"
|
|
helm template stack "$appAssetsPath/k3s" -f "$varsFile" -f "$profileCpuFile" -f "$profileMemoryFile" --show-only "$templateFile" > "$redisYaml" || {
|
|
printDotText "render" "$labelFail"
|
|
printDanger "Render failed"
|
|
return 1
|
|
}
|
|
|
|
printDotText "render" "$labelDone"
|
|
}
|
|
|
|
# Updates Redis Kubernetes resources (limits, config, etc.) without re-initialization.
|
|
function cmdRedisUpdate() {
|
|
cmdRedisYamlRender || return 1
|
|
cmdK3sYamlApplyEx "$redisYaml" || return 1
|
|
}
|
|
|
|
# Installs Redis into Kubernetes.
|
|
function cmdRedisInstall() {
|
|
cmdRedisPreparation || return 1
|
|
cmdRedisYamlRender || return 1
|
|
cmdK3sYamlApplyEx "$redisYaml" || return 1
|
|
}
|
|
|
|
# Uninstalls Redis Kubernetes resources.
|
|
function cmdRedisUninstall() {
|
|
"$k3sCmd" kubectl delete -f "$redisYaml"
|
|
}
|
|
|
|
# Prints replication INFO for all Redis and Sentinel pods.
|
|
function cmdRedisInfo() {
|
|
local output error podList ver pid
|
|
|
|
if ! run podList error k3sPodListStatus "$redisKube"; then
|
|
printDanger "Get pods: $error"
|
|
elif [[ -z "$podList" ]]; then
|
|
printDanger "Pods not found"
|
|
else
|
|
local ready reason
|
|
while IFS='|' read -r pod ready phase reason; do
|
|
printSection "$pod"
|
|
|
|
if [[ "$ready" != "1" ]]; then
|
|
printDotText "Status" "$fontGray$reason $fontRed$phase$fontReset"
|
|
else
|
|
printDotText "Status" "$fontGreen$phase$fontReset"
|
|
|
|
if ! run output error redisPodExecCli "$pod" INFO server; then
|
|
printDotText "Redis version" "$labelUnknown"
|
|
printDanger "$error"
|
|
else
|
|
ver=$(printf '%s' "$output" | grep redis_version | cut -d ':' -f2 | tr -d '[:space:]')
|
|
printDotText "Redis version" "$ver"
|
|
fi
|
|
|
|
if ! run output error redisPodExecCli "$pod" INFO replication; then
|
|
printDotText "Redis version" "$labelUnknown"
|
|
printDanger "$error"
|
|
else
|
|
|
|
local role slaves masterHost masterLinkStatus slaveLag
|
|
role=$(printf '%s' "$output" | grep -i "role" | cut -d: -f2 | tr -d '\r\n')
|
|
if [[ "$role" == "master" ]]; then
|
|
printDotText "Replica role" "$fontGreen$role$fontReset"
|
|
|
|
slaves=$(printf '%s' "$output" | grep -i "connected_slaves" | cut -d: -f2 | tr -d '\r\n')
|
|
if [[ "$slaves" -eq 0 ]]; then
|
|
printDotText "Slaves" "$fontRed$slaves$fontReset"
|
|
else
|
|
printDotText "Slaves" "$fontGreen$slaves$fontReset"
|
|
fi
|
|
elif [[ "$role" == "slave" ]]; then
|
|
printDotText "Replica role" "$fontGreen$role$fontReset"
|
|
|
|
masterHost=$(printf '%s' "$output" | grep -i "master_host" | cut -d: -f2 | tr -d '\r\n')
|
|
masterLinkStatus=$(printf '%s' "$output" | grep -i "master_link_status" | cut -d: -f2 | tr -d '\r\n')
|
|
if [[ -z "$masterHost" || "$masterLinkStatus" != "up" ]]; then
|
|
[[ -z "$masterHost" ]] && printDotText "Master" "${fontRed}Not connect to master (master_host)$fontReset"
|
|
[[ "$masterLinkStatus" != "up" ]] && printDotText "Master" "${fontRed}Not connect to master (master_link_status)$fontReset"
|
|
continue
|
|
fi
|
|
printDotText "Master" "$fontGreen$masterHost$fontReset"
|
|
|
|
slaveLag=$(printf '%s' "$output" | grep -i "master_last_io_seconds_ago" | cut -d: -f2 | tr -d '\r\n')
|
|
[[ "$slaveLag" -ge 3 ]] && printDotText "Replication delay" "$fontYallow${slaveLag}s$fontReset"
|
|
else
|
|
printDotText "Replica role" "$fontRed$role$fontReset"
|
|
fi
|
|
fi
|
|
|
|
|
|
|
|
fi
|
|
done < <(awk 'NF' <<< "$podList")
|
|
fi
|
|
|
|
if ! run podList error k3sPodListStatus "$redisSentinelKube"; then
|
|
printDanger "Get pods: $error"
|
|
elif [[ -z "$podList" ]]; then
|
|
printDanger "Pods not found"
|
|
else
|
|
local masterInfo slaveInfo
|
|
local masterName masterIP masterPort masterNumOtherSentinels masterQuorum
|
|
local masterSig replicaSig refPod refMasterSig refReplicaSig
|
|
local activeReplicaCount mismatch=0
|
|
local ready reason
|
|
|
|
while IFS='|' read -r pod ready phase reason; do
|
|
printSection "$pod"
|
|
|
|
if [[ "$ready" != "1" ]]; then
|
|
printDotText "Status" "$fontGray$reason $fontRed$phase$fontReset"
|
|
else
|
|
printDotText "Status" "$fontGreen$phase$fontReset"
|
|
|
|
if ! run output error redisPodExecCli "$pod" INFO server; then
|
|
printDotText "RedisSentinel version" "$labelUnknown"
|
|
printDanger "$error"
|
|
else
|
|
ver=$(printf '%s' "$output" | grep redis_version | cut -d ':' -f2 | tr -d '[:space:]')
|
|
printDotText "RedisSentinel version" "$ver"
|
|
fi
|
|
|
|
run masterInfo error redisPodExecCli "$pod" SENTINEL masters || {
|
|
printDotText "Get sentinel masters" "$labelFail"
|
|
printDanger "$error"
|
|
mismatch=1
|
|
continue
|
|
}
|
|
|
|
local -A masterData=()
|
|
while read -r key && read -r value; do
|
|
masterData["$key"]="$value"
|
|
done <<< "$masterInfo"
|
|
masterName="${masterData[name]}"
|
|
masterIP="${masterData[ip]}"
|
|
masterPort="${masterData[port]}"
|
|
masterNumOtherSentinels="${masterData[num-other-sentinels]:-0}"
|
|
masterQuorum="${masterData[quorum]:-0}"
|
|
|
|
run slaveInfo error redisPodExecCli "$pod" --raw sentinel replicas "$masterName" || {
|
|
printDotText "Get sentinel replicas" "$labelFail"
|
|
printDanger "$error"
|
|
mismatch=1
|
|
continue
|
|
}
|
|
|
|
replicaSig="$(redisSentinelParseReplicas "$slaveInfo" | awk 'NF' | sort)"
|
|
activeReplicaCount="$(awk 'NF {c++} END {print c+0}' <<< "$replicaSig")"
|
|
masterSig="${masterName}|${masterIP}|${masterPort}|${activeReplicaCount}|${masterNumOtherSentinels}|${masterQuorum}"
|
|
|
|
refPod="" refMasterSig="" refReplicaSig=""
|
|
if [[ -z "$refPod" ]]; then
|
|
refPod="$pod"
|
|
refMasterSig="$masterSig"
|
|
refReplicaSig="$replicaSig"
|
|
fi
|
|
|
|
if [[ "$masterSig" != "$refMasterSig" || "$replicaSig" != "$refReplicaSig" ]]; then
|
|
mismatch=1
|
|
printDotText "Topology" "mismatch vs $fontRed$refPod$fontReset"
|
|
# else
|
|
# printDotText "Topology" "matches $fontGreen$refPod$fontReset"
|
|
fi
|
|
|
|
printDotText " ┌ Replica" "$masterName"
|
|
if (( masterQuorum < 1 )); then
|
|
printDotText " ├ Quorum" "$fontRed$masterQuorum$fontReset"
|
|
else
|
|
printDotText " ├ Quorum" "$fontGreen$masterQuorum$fontReset"
|
|
fi
|
|
printDotText " ├ Other sentinels" "$masterNumOtherSentinels"
|
|
|
|
# printDotText "Master" "$masterIP:$masterPort"
|
|
printDotText " ├ Master" "$masterIP"
|
|
if (( activeReplicaCount < 1 )); then
|
|
printDotText " └ Slave count" "$fontRed$activeReplicaCount$fontReset"
|
|
else
|
|
printDotText " └ Slave count" "$fontGreen$activeReplicaCount$fontReset"
|
|
fi
|
|
while IFS=$'\t' read -r slaveName slaveIP slavePort slaveMaster slaveRunId; do
|
|
[[ -z "$slaveName" ]] && continue
|
|
printText " ┊"
|
|
printDotText " ├ Slave" "$slaveIP"
|
|
printDotText " ├ Master" "$slaveMaster"
|
|
printDotText " └ RunID" "$slaveRunId"
|
|
done <<< "$replicaSig"
|
|
fi
|
|
done < <(awk 'NF' <<< "$podList")
|
|
fi
|
|
|
|
if ! run podList error k3sPodListStatus "$redisKube-haproxy"; then
|
|
printDanger "Get pods: $error"
|
|
elif [[ -z "$podList" ]]; then
|
|
printDanger "Pods not found"
|
|
else
|
|
local ready reason
|
|
while IFS='|' read -r pod ready phase reason; do
|
|
printSection "$pod"
|
|
|
|
if [[ "$ready" != "1" ]]; then
|
|
printDotText "Status" "$fontGray$reason $fontRed$phase$fontReset"
|
|
else
|
|
printDotText "Status" "$fontGreen$phase$fontReset"
|
|
|
|
if ! run output error k3sRun exec "pod/$pod" -- haproxy -v; then
|
|
printDotText "HAProxy version" "$labelUnknown"
|
|
printDanger "$error"
|
|
else
|
|
ver=$(awk 'NR==1{print $3}' <<< "$output")
|
|
printDotText "HAProxy version" "$ver"
|
|
fi
|
|
|
|
local role
|
|
if ! run output error redisExecCli -h redis-master -p 6379 ROLE; then
|
|
printDotText "Replica role" "$labelFail"
|
|
printDanger "$error"
|
|
else
|
|
role=$(printf '%s' "$output" | head -n1 | tr -d '\r\n')
|
|
if [[ "$role" != "master" ]]; then
|
|
printDotText "Replica role" "$fontRed$role$fontReset"
|
|
else
|
|
printDotText "Replica role" "$fontGreen$role$fontReset"
|
|
fi
|
|
|
|
fi
|
|
fi
|
|
done < <(awk 'NF' <<< "$podList")
|
|
fi
|
|
}
|
|
|
|
# Checks replication role/health on each Redis pod and Sentinel topology consistency.
|
|
function cmdRedisStatus() {
|
|
local output error podList
|
|
|
|
if ! run podList error k3sPodList "$redisKube"; then
|
|
printDanger "Get pods: $error"
|
|
elif [[ -z "$podList" ]]; then
|
|
printDanger "Pods not found"
|
|
else
|
|
while read -r pod; do
|
|
printSection "$pod"
|
|
if run output error redisPodExecCli "$pod" INFO replication; then
|
|
printText "$output"
|
|
else
|
|
printDanger "$pod | $error"
|
|
fi
|
|
done < <(awk 'NF' <<< "$podList")
|
|
fi
|
|
|
|
if ! run podList error k3sPodList "$redisSentinelKube"; then
|
|
printDanger "Get pods: $error"
|
|
elif [[ -z "$podList" ]]; then
|
|
printDanger "Pods not found"
|
|
else
|
|
while read -r pod; do
|
|
printSection "$pod"
|
|
if run output error redisPodExecCli "$pod" INFO sentinel; then
|
|
printText "$output"
|
|
else
|
|
printDanger "$pod | $error"
|
|
fi
|
|
done < <(awk 'NF' <<< "$podList")
|
|
fi
|
|
}
|
|
|
|
# Lists all Redis ACL users.
|
|
function cmdRedisUser() {
|
|
local output error
|
|
|
|
printRow
|
|
|
|
if ! run output error redisUserListGet; then
|
|
printDanger "$error"
|
|
elif [[ -z "$output" ]]; then
|
|
printText "$labelNull"
|
|
else
|
|
printText "$output"
|
|
fi
|
|
}
|
|
|
|
# Flushes all keys from the Redis database.
|
|
function cmdRedisDatabaseFlush() {
|
|
local output error
|
|
|
|
printRow
|
|
|
|
if run output error redisDatabaseFlush; then
|
|
printText "$output"
|
|
else
|
|
printDanger "$error"
|
|
fi
|
|
}
|
|
|
|
# Regenerates and applies the Redis root password across all nodes.
|
|
function cmdRedisRootPassUpdate() {
|
|
local output error
|
|
|
|
printRow
|
|
|
|
if run output error redisRootPassUpdate; then
|
|
printText "$output"
|
|
else
|
|
printDanger "$error"
|
|
fi
|
|
}
|