Initial: Roko Backup Script v2 — Python rewrite with streaming progress & log rotation
Current deployed version (roko-backup.py) after refactoring from bash to Python. Generated via OpenCode (kimi-k2.7-code). Includes: - rclone streaming progress (log every 5% or 30s) - RotatingFileHandler (max 1MB, 5 backups) - LockFile for concurrent run prevention - Yandex Cloud upload with NO_PROXY bypass - Also includes comparison versions from mimo-v2.5-pro and kimi-k2.7-code
This commit is contained in:
Roko
@@ -0,0 +1,741 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Roko Backup Script — refactored Python rewrite.
|
||||
|
||||
Collect -> archive -> upload -> verify -> clean -> report.
|
||||
Runs from system crontab, standalone Python 3.10+.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, ContextManager
|
||||
|
||||
log = logging.getLogger("roko-backup")
|
||||
|
||||
|
||||
# ─── Exceptions ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class BackupError(Exception):
|
||||
"""Base exception for backup pipeline failures."""
|
||||
|
||||
|
||||
class LockError(BackupError):
|
||||
"""Raised when the lock file cannot be acquired."""
|
||||
|
||||
|
||||
# ─── Config ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
"""All paths and limits for the backup pipeline."""
|
||||
|
||||
home: Path = field(default_factory=lambda: Path.home())
|
||||
report_file: Path = field(
|
||||
default_factory=lambda: Path.home()
|
||||
/ ".openclaw"
|
||||
/ "workspace"
|
||||
/ "roko-backup-report.json"
|
||||
)
|
||||
archive_dir: Path = field(default_factory=lambda: Path.home() / "backups")
|
||||
log_file: Path = field(default_factory=lambda: Path.home() / "backups" / "roko-backup.log")
|
||||
rclone_remote: str = "yabackup:for-backup/roko"
|
||||
max_cloud_backups: int = 14
|
||||
max_local_backups: int = 3
|
||||
max_log_size: int = 1_048_576 # 1 MiB
|
||||
max_log_backups: int = 5
|
||||
lock_path: Path = Path("/tmp/roko-backup.lock")
|
||||
|
||||
@property
|
||||
def workspace(self) -> Path:
|
||||
"""Path to the OpenCode workspace directory."""
|
||||
return self.home / ".openclaw" / "workspace"
|
||||
|
||||
@property
|
||||
def config_file(self) -> Path:
|
||||
"""Path to the main OpenCode config file."""
|
||||
return self.home / ".openclaw" / "openclaw.json"
|
||||
|
||||
@property
|
||||
def scripts_dir(self) -> Path:
|
||||
"""Path to the OpenCode scripts directory."""
|
||||
return self.home / ".openclaw" / "scripts"
|
||||
|
||||
@property
|
||||
def media_dir(self) -> Path:
|
||||
"""Path to the OpenCode media directory."""
|
||||
return self.home / ".openclaw" / "media"
|
||||
|
||||
@property
|
||||
def plugin_skills_dir(self) -> Path:
|
||||
"""Path to the OpenCode plugin-skills directory."""
|
||||
return self.home / ".openclaw" / "plugin-skills"
|
||||
|
||||
@property
|
||||
def agent_db(self) -> Path:
|
||||
"""Path to the main agent SQLite database."""
|
||||
return self.home / ".openclaw" / "agents" / "main" / "agent" / "openclaw-agent.sqlite"
|
||||
|
||||
@property
|
||||
def rclone_env(self) -> dict[str, str]:
|
||||
"""Environment variables for rclone subprocesses."""
|
||||
return {
|
||||
**os.environ,
|
||||
"NO_PROXY": "storage.yandexcloud.net,yandexcloud.net",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> Config:
|
||||
"""Build config from environment variables with built-in defaults."""
|
||||
home = Path.home()
|
||||
return cls(
|
||||
home=home,
|
||||
report_file=Path(
|
||||
os.environ.get(
|
||||
"ROKO_REPORT_FILE", str(home / ".openclaw/workspace/roko-backup-report.json")
|
||||
)
|
||||
),
|
||||
archive_dir=Path(os.environ.get("ROKO_ARCHIVE_DIR", str(home / "backups"))),
|
||||
log_file=Path(os.environ.get("ROKO_LOG_FILE", str(home / "backups/roko-backup.log"))),
|
||||
rclone_remote=os.environ.get("ROKO_RCLONE_REMOTE", "yabackup:for-backup/roko"),
|
||||
max_cloud_backups=int(os.environ.get("ROKO_MAX_CLOUD", "14")),
|
||||
max_local_backups=int(os.environ.get("ROKO_MAX_LOCAL", "3")),
|
||||
max_log_size=int(os.environ.get("ROKO_MAX_LOG_SIZE", str(1_048_576))),
|
||||
max_log_backups=int(os.environ.get("ROKO_MAX_LOG_BACKUPS", "5")),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: Path) -> Config:
|
||||
"""Load config from a JSON file, falling back to defaults."""
|
||||
data: dict[str, Any] = {}
|
||||
if path.exists():
|
||||
data = json.loads(path.read_text())
|
||||
home = Path(data.get("home", str(Path.home())))
|
||||
return cls(
|
||||
home=home,
|
||||
report_file=Path(
|
||||
data.get(
|
||||
"report_file", str(home / ".openclaw/workspace/roko-backup-report.json")
|
||||
)
|
||||
),
|
||||
archive_dir=Path(data.get("archive_dir", str(home / "backups"))),
|
||||
log_file=Path(data.get("log_file", str(home / "backups/roko-backup.log"))),
|
||||
rclone_remote=data.get("rclone_remote", "yabackup:for-backup/roko"),
|
||||
max_cloud_backups=data.get("max_cloud_backups", 14),
|
||||
max_local_backups=data.get("max_local_backups", 3),
|
||||
max_log_size=data.get("max_log_size", 1_048_576),
|
||||
max_log_backups=data.get("max_log_backups", 5),
|
||||
)
|
||||
|
||||
|
||||
# ─── LockFile ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class LockFile(ContextManager["LockFile"]):
|
||||
"""Prevents concurrent backup runs via a PID file."""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self._acquired = False
|
||||
|
||||
def acquire(self) -> None:
|
||||
"""Acquire the lock. Raises LockError if another live process holds it."""
|
||||
if self.path.exists():
|
||||
try:
|
||||
pid = int(self.path.read_text().strip())
|
||||
os.kill(pid, 0)
|
||||
raise LockError(f"Another backup is already running (lock: {self.path})")
|
||||
except (ValueError, ProcessLookupError, PermissionError, OSError):
|
||||
log.warning("Stale lock file found (PID dead), removing")
|
||||
self.path.unlink(missing_ok=True)
|
||||
|
||||
self.path.write_text(str(os.getpid()))
|
||||
self._acquired = True
|
||||
signal.signal(signal.SIGTERM, self._signal_handler)
|
||||
signal.signal(signal.SIGINT, self._signal_handler)
|
||||
|
||||
def release(self) -> None:
|
||||
"""Release the lock and remove the file."""
|
||||
if self._acquired:
|
||||
self.path.unlink(missing_ok=True)
|
||||
self._acquired = False
|
||||
|
||||
def _signal_handler(self, signum: int, _frame: Any) -> None:
|
||||
self.release()
|
||||
sys.exit(128 + signum)
|
||||
|
||||
def __enter__(self) -> LockFile:
|
||||
self.acquire()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
||||
self.release()
|
||||
|
||||
|
||||
# ─── Collector ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class Collector:
|
||||
"""Gathers files to back up into a staging directory."""
|
||||
|
||||
EXCLUDE_PATTERNS = (
|
||||
"node_modules",
|
||||
"__pycache__",
|
||||
"*.pyc",
|
||||
".git",
|
||||
"roko-pulse.jsonl",
|
||||
"cache",
|
||||
"*.tmp",
|
||||
".cache",
|
||||
)
|
||||
|
||||
def __init__(self, config: Config, exclude_patterns: tuple[str, ...] | None = None) -> None:
|
||||
self.config = config
|
||||
self.exclude_patterns = exclude_patterns or self.EXCLUDE_PATTERNS
|
||||
|
||||
def collect(self, staging: Path) -> tuple[list[tuple[Path, Path]], int]:
|
||||
"""Copy workspace/config/scripts/media/plugin-skills/agent-db into staging.
|
||||
|
||||
Returns:
|
||||
A tuple of (list of (source, dest) pairs, total bytes collected).
|
||||
"""
|
||||
collected: list[tuple[Path, Path]] = []
|
||||
total_bytes = 0
|
||||
ignore = shutil.ignore_patterns(*self.exclude_patterns)
|
||||
|
||||
targets: list[tuple[Path, str]] = [
|
||||
(self.config.workspace, "openclaw/workspace"),
|
||||
(self.config.config_file, "openclaw/openclaw.json"),
|
||||
(self.config.scripts_dir, "openclaw/scripts"),
|
||||
(self.config.media_dir, "openclaw/media"),
|
||||
(self.config.plugin_skills_dir, "openclaw/plugin-skills"),
|
||||
(self.config.agent_db, "openclaw/openclaw-agent.sqlite"),
|
||||
]
|
||||
|
||||
for src, dest_rel in targets:
|
||||
dest = staging / dest_rel
|
||||
if not src.exists():
|
||||
log.warning("Source not found, skipping: %s", src)
|
||||
continue
|
||||
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
if src.is_dir():
|
||||
shutil.copytree(src, dest, ignore=ignore, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(src, dest)
|
||||
except Exception as exc:
|
||||
log.warning("Failed to copy %s: %s", src, exc)
|
||||
continue
|
||||
|
||||
size = self._tree_size(dest)
|
||||
collected.append((src, dest))
|
||||
total_bytes += size
|
||||
log.info("Collected %s -> %s (%d bytes)", src.name, dest_rel, size)
|
||||
|
||||
return collected, total_bytes
|
||||
|
||||
@staticmethod
|
||||
def _tree_size(path: Path) -> int:
|
||||
"""Return total size in bytes for a file or directory."""
|
||||
if path.is_file():
|
||||
return path.stat().st_size
|
||||
return sum(f.stat().st_size for f in path.rglob("*") if f.is_file())
|
||||
|
||||
|
||||
# ─── Archiver ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class Archiver:
|
||||
"""Creates a tar.gz archive from a staging directory."""
|
||||
|
||||
def __init__(self, config: Config) -> None:
|
||||
self.config = config
|
||||
|
||||
def archive(self, staging: Path, backup_name: str) -> tuple[Path, int]:
|
||||
"""Create tar.gz archive. Returns (archive_path, size_bytes)."""
|
||||
self.config.archive_dir.mkdir(parents=True, exist_ok=True)
|
||||
archive_path = self.config.archive_dir / f"{backup_name}.tar.gz"
|
||||
|
||||
log.info("Creating archive: %s", archive_path)
|
||||
with tarfile.open(archive_path, mode="w:gz") as tar:
|
||||
tar.add(staging, arcname=".")
|
||||
|
||||
size = archive_path.stat().st_size
|
||||
log.info("Archive created: %s (%d bytes)", archive_path.name, size)
|
||||
return archive_path, size
|
||||
|
||||
|
||||
# ─── Uploader ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class Uploader:
|
||||
"""Uploads archive via rclone to Yandex Object Storage with retry."""
|
||||
|
||||
UPLOAD_PROGRESS_INTERVAL = 30 # seconds between progress log lines
|
||||
PROGRESS_PERCENT_STEP = 5 # log every N percent
|
||||
|
||||
BACKOFF_SCHEDULE = (10, 30, 60)
|
||||
|
||||
_PROGRESS_RE = re.compile(
|
||||
r"Transferred:\s*([\d.]+)\s*(\w*)\s*/\s*([\d.]+)\s*(\w*),\s*([\d.]+)%"
|
||||
)
|
||||
|
||||
def __init__(self, config: Config) -> None:
|
||||
self.config = config
|
||||
|
||||
def upload(self, archive_path: Path) -> bool:
|
||||
"""Upload archive to remote. Returns True on success."""
|
||||
max_attempts = len(self.BACKOFF_SCHEDULE) + 1
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
log.info("Upload attempt %d/%d", attempt, max_attempts)
|
||||
if self._upload_once(archive_path):
|
||||
log.info("Upload succeeded")
|
||||
return True
|
||||
|
||||
if attempt < max_attempts:
|
||||
delay = self.BACKOFF_SCHEDULE[min(attempt - 1, len(self.BACKOFF_SCHEDULE) - 1)]
|
||||
log.info("Retrying in %ds...", delay)
|
||||
time.sleep(delay)
|
||||
|
||||
log.error("Upload failed after %d attempts", max_attempts)
|
||||
return False
|
||||
|
||||
def _upload_once(self, archive_path: Path) -> bool:
|
||||
"""Run a single upload attempt with streaming progress."""
|
||||
cmd = [
|
||||
"rclone",
|
||||
"copy",
|
||||
str(archive_path),
|
||||
f"{self.config.rclone_remote}/",
|
||||
"--progress",
|
||||
]
|
||||
log.debug("Running: %s", " ".join(cmd))
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
env=self.config.rclone_env,
|
||||
)
|
||||
|
||||
last_logged_percent = -1
|
||||
last_log_time = 0.0
|
||||
final_stats: dict[str, str | float] = {}
|
||||
|
||||
if process.stderr is None:
|
||||
log.warning("rclone stderr not available")
|
||||
process.wait()
|
||||
return process.returncode == 0
|
||||
|
||||
for line in process.stderr:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
match = self._PROGRESS_RE.search(line)
|
||||
if match:
|
||||
current, cur_unit, total, tot_unit, percent = match.groups()
|
||||
pct = float(percent)
|
||||
should_log = (
|
||||
pct - last_logged_percent >= self.PROGRESS_PERCENT_STEP
|
||||
or time.monotonic() - last_log_time >= self.UPLOAD_PROGRESS_INTERVAL
|
||||
)
|
||||
if should_log:
|
||||
log.info(
|
||||
"Upload: %.0f%% (%s %s / %s %s)",
|
||||
pct,
|
||||
current,
|
||||
cur_unit or "B",
|
||||
total,
|
||||
tot_unit or "B",
|
||||
)
|
||||
last_logged_percent = pct
|
||||
last_log_time = time.monotonic()
|
||||
final_stats = {
|
||||
"current": current,
|
||||
"current_unit": cur_unit or "B",
|
||||
"total": total,
|
||||
"total_unit": tot_unit or "B",
|
||||
"percent": pct,
|
||||
}
|
||||
elif "Transferred:" in line and "Elapsed time:" in line:
|
||||
final_stats["summary"] = line
|
||||
|
||||
process.wait()
|
||||
if process.returncode == 0:
|
||||
if final_stats:
|
||||
summary = final_stats.get("summary")
|
||||
if summary:
|
||||
log.info("Upload stats: %s", summary)
|
||||
else:
|
||||
log.info(
|
||||
"Upload complete: %.0f%% (%s %s / %s %s)",
|
||||
final_stats.get("percent", 0.0),
|
||||
final_stats.get("current", "0"),
|
||||
final_stats.get("current_unit", "B"),
|
||||
final_stats.get("total", "0"),
|
||||
final_stats.get("total_unit", "B"),
|
||||
)
|
||||
return True
|
||||
|
||||
log.warning("Upload failed (rc=%d)", process.returncode)
|
||||
return False
|
||||
|
||||
|
||||
# ─── Verifier ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class Verifier:
|
||||
"""Confirms the uploaded file exists in the remote."""
|
||||
|
||||
def __init__(self, config: Config) -> None:
|
||||
self.config = config
|
||||
|
||||
def verify(self, backup_name: str) -> tuple[bool, str]:
|
||||
"""Check that backup_name.tar.gz is listed in the remote.
|
||||
|
||||
Returns:
|
||||
A tuple of (success, details).
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["rclone", "ls", self.config.rclone_remote],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
env=self.config.rclone_env,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
return False, f"rclone ls failed (rc={result.returncode}): {result.stderr[:200]}"
|
||||
|
||||
target = f"{backup_name}.tar.gz"
|
||||
if target in result.stdout:
|
||||
return True, f"Verified: {target} found in remote"
|
||||
|
||||
return False, f"Not found in remote: {target}"
|
||||
|
||||
|
||||
# ─── Cleaner ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class Cleaner:
|
||||
"""Manages retention of local and cloud backups."""
|
||||
|
||||
def __init__(self, config: Config) -> None:
|
||||
self.config = config
|
||||
|
||||
def clean_local(self) -> int:
|
||||
"""Delete local archives beyond MAX_LOCAL_BACKUPS. Returns count deleted."""
|
||||
archives = sorted(
|
||||
self.config.archive_dir.glob("roko-*.tar.gz"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
deleted = 0
|
||||
for old in archives[self.config.max_local_backups :]:
|
||||
log.info("Deleting local: %s", old.name)
|
||||
old.unlink()
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
def clean_cloud(self) -> int:
|
||||
"""Delete cloud archives beyond MAX_CLOUD_BACKUPS. Returns count deleted."""
|
||||
result = subprocess.run(
|
||||
["rclone", "ls", self.config.rclone_remote],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
env=self.config.rclone_env,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
log.warning("Cannot list cloud backups for cleanup: %s", result.stderr[:200])
|
||||
return 0
|
||||
|
||||
entries: list[tuple[int, str]] = []
|
||||
for line in result.stdout.strip().splitlines():
|
||||
parts = line.split(None, 1)
|
||||
if len(parts) == 2:
|
||||
try:
|
||||
entries.append((int(parts[0]), parts[1]))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
entries.sort(key=lambda e: e[1])
|
||||
deleted = 0
|
||||
to_remove = entries[: max(0, len(entries) - self.config.max_cloud_backups)]
|
||||
|
||||
for _size, name in to_remove:
|
||||
log.info("Deleting cloud: %s", name)
|
||||
subprocess.run(
|
||||
["rclone", "delete", f"{self.config.rclone_remote}/{name}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
env=self.config.rclone_env,
|
||||
)
|
||||
deleted += 1
|
||||
|
||||
return deleted
|
||||
|
||||
|
||||
# ─── Reporter ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class Reporter:
|
||||
"""Writes JSON backup report to the workspace."""
|
||||
|
||||
def __init__(self, config: Config) -> None:
|
||||
self.config = config
|
||||
|
||||
def write(
|
||||
self,
|
||||
status: str,
|
||||
name: str | None = None,
|
||||
size_bytes: int | None = None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""Write the report JSON file."""
|
||||
report = {
|
||||
"timestamp": datetime.datetime.now().astimezone().isoformat(),
|
||||
"status": status,
|
||||
"name": name,
|
||||
"size_bytes": size_bytes,
|
||||
"error": error,
|
||||
}
|
||||
self.config.report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.config.report_file.write_text(json.dumps(report, indent=2) + "\n")
|
||||
log.info("Report written: %s (%s)", self.config.report_file, status)
|
||||
|
||||
|
||||
# ─── Pipeline ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class BackupPipeline:
|
||||
"""Orchestrates the full backup: collect -> archive -> upload -> verify -> clean -> report."""
|
||||
|
||||
def __init__(self, config: Config, *, dry_run: bool = False) -> None:
|
||||
self.config = config
|
||||
self.dry_run = dry_run
|
||||
self.collector = Collector(config)
|
||||
self.archiver = Archiver(config)
|
||||
self.uploader = Uploader(config)
|
||||
self.verifier = Verifier(config)
|
||||
self.cleaner = Cleaner(config)
|
||||
self.reporter = Reporter(config)
|
||||
|
||||
def run(self) -> int:
|
||||
"""Execute the pipeline. Returns exit code."""
|
||||
backup_name = f"roko-{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}"
|
||||
log.info("Starting backup: %s", backup_name)
|
||||
|
||||
free = shutil.disk_usage(self.config.archive_dir).free
|
||||
log.info("Free disk: %s", self._fmt_bytes(free))
|
||||
|
||||
archive_path: Path | None = None
|
||||
archive_size: int | None = None
|
||||
|
||||
with self._managed_staging() as staging:
|
||||
# 1. Collect
|
||||
t0 = time.monotonic()
|
||||
collected, total_bytes = self.collector.collect(staging)
|
||||
log.info(
|
||||
"Collection done in %.1fs: %d items, %s",
|
||||
time.monotonic() - t0,
|
||||
len(collected),
|
||||
self._fmt_bytes(total_bytes),
|
||||
)
|
||||
|
||||
if self.dry_run:
|
||||
log.info(
|
||||
"[DRY RUN] Would archive and upload %d items (%s)",
|
||||
len(collected),
|
||||
self._fmt_bytes(total_bytes),
|
||||
)
|
||||
for src, dest in collected:
|
||||
log.info(" %s -> %s", src, dest.relative_to(staging))
|
||||
self.reporter.write("dry-run", name=backup_name, size_bytes=total_bytes)
|
||||
return 0
|
||||
|
||||
# 2. Archive
|
||||
t0 = time.monotonic()
|
||||
archive_path, archive_size = self.archiver.archive(staging, backup_name)
|
||||
log.info(
|
||||
"Archiving done in %.1fs: %s",
|
||||
time.monotonic() - t0,
|
||||
self._fmt_bytes(archive_size),
|
||||
)
|
||||
|
||||
# 3. Upload
|
||||
if archive_path is None or archive_size is None:
|
||||
msg = "Archive was not created"
|
||||
log.error(msg)
|
||||
self.reporter.write("error", name=backup_name, error=msg)
|
||||
return 3
|
||||
|
||||
t0 = time.monotonic()
|
||||
if not self.uploader.upload(archive_path):
|
||||
msg = "Upload failed after retries"
|
||||
log.error(msg)
|
||||
self.reporter.write("error", name=backup_name, size_bytes=archive_size, error=msg)
|
||||
return 3
|
||||
log.info("Upload done in %.1fs", time.monotonic() - t0)
|
||||
|
||||
# 4. Verify
|
||||
ok, detail = self.verifier.verify(backup_name)
|
||||
if not ok:
|
||||
log.error("Verification failed: %s", detail)
|
||||
self.reporter.write("error", name=backup_name, size_bytes=archive_size, error=detail)
|
||||
return 3
|
||||
log.info("Verification: %s", detail)
|
||||
|
||||
# 5. Clean
|
||||
local_deleted = self.cleaner.clean_local()
|
||||
cloud_deleted = self.cleaner.clean_cloud()
|
||||
log.info("Cleanup: %d local, %d cloud removed", local_deleted, cloud_deleted)
|
||||
|
||||
# 6. Report
|
||||
self.reporter.write("success", name=backup_name, size_bytes=archive_size)
|
||||
log.info("Backup complete: %s (%s)", backup_name, self._fmt_bytes(archive_size))
|
||||
return 0
|
||||
|
||||
@contextmanager
|
||||
def _managed_staging(self) -> Generator[Path, None, None]:
|
||||
"""Create and always clean up a temporary staging directory."""
|
||||
tmp_dir = tempfile.mkdtemp(prefix="roko-backup-")
|
||||
staging = Path(tmp_dir)
|
||||
try:
|
||||
yield staging
|
||||
finally:
|
||||
try:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
except Exception as exc:
|
||||
log.warning("Failed to remove staging directory %s: %s", staging, exc)
|
||||
|
||||
@staticmethod
|
||||
def _fmt_bytes(n: int) -> str:
|
||||
"""Human-readable byte size."""
|
||||
value = float(n)
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if abs(value) < 1024:
|
||||
return f"{value:.1f} {unit}"
|
||||
value /= 1024
|
||||
return f"{value:.1f} TB"
|
||||
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
"""Parse command-line arguments."""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="roko-backup.py",
|
||||
description="Roko backup: collect -> archive -> upload -> verify -> clean -> report",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Config file path (default: env vars / built-in defaults)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Collect and print what would be backed up, don't archive/upload",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Detailed logging output (DEBUG level, also to stdout)",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def setup_logging(
|
||||
verbose: bool,
|
||||
log_file: Path,
|
||||
max_size: int,
|
||||
max_backups: int,
|
||||
) -> None:
|
||||
"""Configure rotating file logging and optional stdout output."""
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s %(levelname)-5s %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
str(log_file),
|
||||
maxBytes=max_size,
|
||||
backupCount=max_backups,
|
||||
)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
root = logging.getLogger()
|
||||
root.setLevel(level)
|
||||
root.addHandler(file_handler)
|
||||
|
||||
if verbose:
|
||||
stream_handler = logging.StreamHandler(sys.stdout)
|
||||
stream_handler.setFormatter(formatter)
|
||||
root.addHandler(stream_handler)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Entry point."""
|
||||
args = parse_args(argv)
|
||||
|
||||
if args.config:
|
||||
config = Config.from_file(args.config)
|
||||
else:
|
||||
config = Config.from_env()
|
||||
|
||||
setup_logging(
|
||||
args.verbose,
|
||||
config.log_file,
|
||||
config.max_log_size,
|
||||
config.max_log_backups,
|
||||
)
|
||||
|
||||
if not shutil.which("rclone"):
|
||||
log.error("rclone not found in PATH")
|
||||
return 1
|
||||
|
||||
try:
|
||||
with LockFile(config.lock_path):
|
||||
pipeline = BackupPipeline(config, dry_run=args.dry_run)
|
||||
return pipeline.run()
|
||||
except LockError as exc:
|
||||
log.error("%s", exc)
|
||||
return 2
|
||||
except Exception as exc:
|
||||
log.exception("Unexpected error: %s", exc)
|
||||
Reporter(config).write("error", error=str(exc))
|
||||
return 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user