130 lines
5.1 KiB
Python
130 lines
5.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Enable durable local Redis writes and bound cache/queue memory without restart."""
|
|
|
|
import datetime
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
|
|
|
|
def redis(*args):
|
|
result = subprocess.run(
|
|
["redis-cli", "--json", *args], check=True, capture_output=True, text=True, timeout=10
|
|
)
|
|
if args[0] == "INFO":
|
|
# redis-cli versions differ: INFO may remain raw with --json enabled.
|
|
return json.loads(result.stdout) if result.stdout.lstrip().startswith('"') else result.stdout
|
|
value = json.loads(result.stdout)
|
|
if isinstance(value, dict) and "error" in value:
|
|
raise RuntimeError("Redis configuration command failed")
|
|
return value
|
|
|
|
|
|
def main():
|
|
if os.geteuid() != 0:
|
|
raise SystemExit("Run with sudo on CORP.")
|
|
with open("/run/lock/eusrr-deploy.lock", "a") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
configure_redis()
|
|
|
|
|
|
def info_fields(section):
|
|
info = redis("INFO", section)
|
|
if not isinstance(info, str):
|
|
raise RuntimeError("Unsupported Redis INFO response")
|
|
return dict(line.split(":", 1) for line in info.splitlines() if ":" in line)
|
|
|
|
|
|
def configure_redis():
|
|
config = Path("/etc/redis/redis.conf")
|
|
if config.is_symlink() or not config.is_file():
|
|
raise SystemExit("Unexpected Redis configuration path.")
|
|
old = redis(
|
|
"CONFIG", "GET", "appendonly", "appendfsync", "maxmemory", "maxmemory-policy"
|
|
)
|
|
wanted = {
|
|
"appendonly": "yes",
|
|
"appendfsync": "everysec",
|
|
"maxmemory": "268435456",
|
|
"maxmemory-policy": "noeviction",
|
|
}
|
|
if not isinstance(old, dict) or set(old) != set(wanted):
|
|
raise SystemExit("Unsupported redis-cli response format.")
|
|
binds = redis("CONFIG", "GET", "bind")["bind"].split()
|
|
if not binds or any(
|
|
address.lstrip("-") not in {"127.0.0.1", "::1"} for address in binds
|
|
):
|
|
raise SystemExit("Redis is not loopback-only; review networking first.")
|
|
memory = info_fields("memory")
|
|
used_memory = int(memory.get("used_memory", "-1"))
|
|
if used_memory < 0 or used_memory > int(wanted["maxmemory"]) * 3 // 4:
|
|
raise SystemExit("Redis is already too close to the proposed memory limit; size it before applying.")
|
|
directory = redis("CONFIG", "GET", "dir").get("dir")
|
|
if not directory or not Path(directory).is_absolute():
|
|
raise SystemExit("Cannot verify the Redis persistence directory.")
|
|
if shutil.disk_usage(directory).free < max(512 * 1024 * 1024, used_memory * 3):
|
|
raise SystemExit("Insufficient free space for initial AOF and its rewrite reserve.")
|
|
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
|
|
backup = Path("/var/lib/eusrr-deploy") / f"redis-backup-{stamp}"
|
|
backup.mkdir(mode=0o700)
|
|
shutil.copy2(config, backup / "redis.conf")
|
|
try:
|
|
for key, value in wanted.items():
|
|
if redis("CONFIG", "SET", key, value) != "OK":
|
|
raise RuntimeError("Redis rejected configuration change")
|
|
for _ in range(30):
|
|
fields = info_fields("persistence")
|
|
if (
|
|
fields.get("aof_rewrite_in_progress") == "0"
|
|
and fields.get("aof_rewrite_scheduled") == "0"
|
|
):
|
|
if (
|
|
fields.get("aof_enabled") != "1"
|
|
or fields.get("aof_last_bgrewrite_status") != "ok"
|
|
):
|
|
raise RuntimeError("Initial AOF snapshot failed")
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise RuntimeError("Initial AOF snapshot did not complete in time")
|
|
if redis("CONFIG", "REWRITE") != "OK":
|
|
raise RuntimeError("Could not persist Redis configuration")
|
|
observed = redis("CONFIG", "GET", *wanted)
|
|
if observed != wanted:
|
|
raise RuntimeError("Redis settings verification failed")
|
|
except BaseException as failure:
|
|
recovery_errors = []
|
|
for key, value in old.items():
|
|
try:
|
|
if redis("CONFIG", "SET", key, value) != "OK":
|
|
raise RuntimeError("Redis rejected rollback")
|
|
except BaseException:
|
|
recovery_errors.append(key)
|
|
try:
|
|
if redis("CONFIG", "REWRITE") != "OK":
|
|
raise RuntimeError("Redis rejected rollback persistence")
|
|
except BaseException:
|
|
recovery_errors.append("configuration persistence")
|
|
try:
|
|
if redis("CONFIG", "GET", *old) != old:
|
|
raise RuntimeError("Redis rollback verification failed")
|
|
except BaseException:
|
|
recovery_errors.append("verification")
|
|
if recovery_errors:
|
|
raise RuntimeError(
|
|
f"Redis recovery incomplete ({', '.join(recovery_errors)}); inspect root backup {backup}"
|
|
) from failure
|
|
raise
|
|
print(
|
|
"Redis AOF/everysec ready; maxmemory=256MiB, noeviction. No keys deleted, no restart."
|
|
)
|
|
print(f"Configuration backup: {backup}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|