Make distributable: config.toml support, blanked Matrix defaults, env creds fallback
- ~/.config/coe-mirror/config.toml (tomllib, Python 3.11+ required) - Precedence uniform for every key: env var > config file > hardcoded default - API_BASE, OWNERS, PCLOUD_TARGET, PENDING_DIR, matrix homeserver/room resolved via _resolve() helper - Matrix homeserver/room now default to empty; _send_matrix short-circuits when either is blank so forks don't spam the original author's room - _fetch_creds falls back to COE_MIRROR_MATRIX_USER/PASSWORD env vars when the Infisical creds binary is unavailable (required for external distribution) - CLI subcommands: config init (writes example TOML, won't overwrite), config path Bare invocation and 'run' still work unchanged for the systemd timer - README rewritten as a public quick-start with the precedence table and Matrix as opt-in - Tests: 3 new for config precedence, 1 for Matrix silent no-op when blank, 2 for creds env fallback (26 total, all pass) DevTrack #1285
This commit is contained in:
n0mad1k
+167
-28
@@ -1,8 +1,12 @@
|
||||
"""Mirror repos from git.churchofmalware.org Forgejo instance to pCloud.
|
||||
"""Mirror repos from a Forgejo instance to a local target dir (e.g. pCloud).
|
||||
|
||||
Fetches all repos owned by each user in OWNERS and stores per-commit zip snapshots
|
||||
under `PCLOUD_TARGET/<owner>/`. Runs unattended via systemd timer twice daily.
|
||||
Local staging when pCloud unavailable.
|
||||
under `PCLOUD_TARGET/<owner>/`. Runs unattended via systemd timer.
|
||||
Local staging when the target is unavailable.
|
||||
|
||||
Configuration precedence (for each key): env var > config file > hardcoded default.
|
||||
Config file: `~/.config/coe-mirror/config.toml` (see `python coe_mirror.py config init`).
|
||||
Requires Python 3.11+ (for stdlib `tomllib`).
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -14,6 +18,7 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
import zipfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
@@ -23,11 +28,59 @@ from urllib.parse import quote
|
||||
|
||||
import requests
|
||||
|
||||
# Module constants (overridable via env vars for testability)
|
||||
API_BASE = "https://git.churchofmalware.org"
|
||||
OWNERS = ["Nightmare_Eclipse", "shai_hulud", "bikini_mirror", "ek0mssavi0r"]
|
||||
PCLOUD_TARGET = Path.home() / "pCloudDrive" / "Hacking" / "ChurchOfMalware"
|
||||
PENDING_DIR = Path.home() / ".local" / "share" / "coe-mirror" / "pending"
|
||||
# --- Config file (optional) ------------------------------------------------
|
||||
CONFIG_PATH = Path(
|
||||
os.getenv("COE_MIRROR_CONFIG")
|
||||
or (Path(os.getenv("XDG_CONFIG_HOME") or Path.home() / ".config") / "coe-mirror" / "config.toml")
|
||||
)
|
||||
|
||||
|
||||
def _load_config_file() -> dict:
|
||||
"""Load config.toml if present. Never raises; malformed file → empty dict + stderr warning."""
|
||||
try:
|
||||
if CONFIG_PATH.is_file():
|
||||
with open(CONFIG_PATH, "rb") as f:
|
||||
return tomllib.load(f)
|
||||
except Exception as exc:
|
||||
print(f"coe-mirror: warning: could not parse {CONFIG_PATH}: {exc}", file=sys.stderr)
|
||||
return {}
|
||||
|
||||
|
||||
_CFG = _load_config_file()
|
||||
_CFG_MATRIX = _CFG.get("matrix", {}) if isinstance(_CFG.get("matrix"), dict) else {}
|
||||
|
||||
|
||||
def _resolve(env_key: str, cfg_value, default):
|
||||
"""env var > config value > hardcoded default. Empty env/config values are ignored."""
|
||||
env_val = os.getenv(env_key)
|
||||
if env_val:
|
||||
return env_val
|
||||
if cfg_value not in (None, ""):
|
||||
return cfg_value
|
||||
return default
|
||||
|
||||
|
||||
# --- Module constants (resolved via env > config > default) ---------------
|
||||
API_BASE: str = _resolve("COE_MIRROR_API_BASE", _CFG.get("api_base"), "https://git.churchofmalware.org")
|
||||
OWNERS: List[str] = (
|
||||
[o.strip() for o in os.getenv("COE_MIRROR_OWNERS", "").split(",") if o.strip()]
|
||||
or (_CFG.get("owners") if isinstance(_CFG.get("owners"), list) else None)
|
||||
or ["Nightmare_Eclipse", "shai_hulud", "bikini_mirror", "ek0mssavi0r"]
|
||||
)
|
||||
PCLOUD_TARGET: Path = Path(
|
||||
_resolve(
|
||||
"COE_MIRROR_PCLOUD_TARGET",
|
||||
_CFG.get("pcloud_target"),
|
||||
str(Path.home() / "pCloudDrive" / "Hacking" / "ChurchOfMalware"),
|
||||
)
|
||||
).expanduser()
|
||||
PENDING_DIR: Path = Path(
|
||||
_resolve(
|
||||
"COE_MIRROR_PENDING_DIR",
|
||||
_CFG.get("pending_dir"),
|
||||
str(Path.home() / ".local" / "share" / "coe-mirror" / "pending"),
|
||||
)
|
||||
).expanduser()
|
||||
LOG_DIR = Path.home() / ".local" / "share" / "coe-mirror"
|
||||
LOG_FILE = LOG_DIR / "coe-mirror.log"
|
||||
LOCK_NAME = "coe-mirror.lock"
|
||||
@@ -38,25 +91,15 @@ RETRY_BACKOFF = [0, 5, 30] # seconds between attempts; len = max attempts
|
||||
USER_AGENT = "coe-mirror/1.0"
|
||||
REPO_NAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$")
|
||||
|
||||
# Matrix configuration
|
||||
MATRIX_HOMESERVER = "https://m.example.org"
|
||||
MATRIX_ROOM = "!REDACTEDROOM:m.example.org" # General
|
||||
# Matrix configuration (fully opt-in; blank defaults => notifications silently skipped)
|
||||
MATRIX_HOMESERVER: str = _resolve("COE_MIRROR_MATRIX_HOMESERVER", _CFG_MATRIX.get("homeserver"), "")
|
||||
MATRIX_ROOM: str = _resolve("COE_MIRROR_MATRIX_ROOM", _CFG_MATRIX.get("room"), "")
|
||||
MATRIX_TOKEN_CACHE = LOG_DIR / "matrix-token.json"
|
||||
MATRIX_TIMEOUT = 8
|
||||
MATRIX_REFRESH_THRESHOLD_SECONDS = 1800 # refresh if <30 min left
|
||||
COE_MIRROR_BOT_USER_KEY = "COE_MIRROR_BOT_USER"
|
||||
COE_MIRROR_BOT_PASSWORD_KEY = "COE_MIRROR_BOT_PASSWORD"
|
||||
|
||||
# Allow env var overrides for testing and deployment flexibility
|
||||
if env_pcloud := os.getenv("COE_MIRROR_PCLOUD_TARGET"):
|
||||
PCLOUD_TARGET = Path(env_pcloud)
|
||||
if env_pending := os.getenv("COE_MIRROR_PENDING_DIR"):
|
||||
PENDING_DIR = Path(env_pending)
|
||||
if env_hs := os.getenv("COE_MIRROR_MATRIX_HOMESERVER"):
|
||||
MATRIX_HOMESERVER = env_hs
|
||||
if env_room := os.getenv("COE_MIRROR_MATRIX_ROOM"):
|
||||
MATRIX_ROOM = env_room
|
||||
|
||||
logger: logging.Logger = None # Initialized in setup_logging()
|
||||
|
||||
|
||||
@@ -362,7 +405,12 @@ def _write_token_cache(access_token: str, expires_at_iso: str) -> None:
|
||||
|
||||
|
||||
def _fetch_creds() -> Optional[Tuple[str, str]]:
|
||||
"""Fetch (user, password) from Infisical via creds. Return None on failure."""
|
||||
"""Fetch (user, password) for Matrix.
|
||||
|
||||
Order: `creds` binary (Infisical) → `COE_MIRROR_MATRIX_USER` / `COE_MIRROR_MATRIX_PASSWORD`
|
||||
env vars → None. Distributed users typically set the env vars in the systemd
|
||||
EnvironmentFile; the Infisical path is convenience for hosts that already have it.
|
||||
"""
|
||||
try:
|
||||
user = subprocess.run(
|
||||
["creds", "get", COE_MIRROR_BOT_USER_KEY, "matrix"],
|
||||
@@ -376,14 +424,20 @@ def _fetch_creds() -> Optional[Tuple[str, str]]:
|
||||
u, p = user.stdout.strip(), pw.stdout.strip()
|
||||
if u and p:
|
||||
return u, p
|
||||
logger.warning("creds returned non-zero for %s/%s",
|
||||
COE_MIRROR_BOT_USER_KEY, COE_MIRROR_BOT_PASSWORD_KEY)
|
||||
logger.debug("creds returned non-zero for %s/%s; trying env fallback",
|
||||
COE_MIRROR_BOT_USER_KEY, COE_MIRROR_BOT_PASSWORD_KEY)
|
||||
except FileNotFoundError:
|
||||
logger.warning("creds binary not found on PATH")
|
||||
logger.debug("creds binary not on PATH; trying env fallback")
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("creds timed out fetching Matrix credentials")
|
||||
logger.warning("creds timed out; trying env fallback")
|
||||
except Exception as exc:
|
||||
logger.warning("creds fetch failed: %s", exc)
|
||||
logger.warning("creds fetch failed: %s; trying env fallback", exc)
|
||||
|
||||
env_u = os.getenv("COE_MIRROR_MATRIX_USER", "").strip()
|
||||
env_p = os.getenv("COE_MIRROR_MATRIX_PASSWORD", "").strip()
|
||||
if env_u and env_p:
|
||||
return env_u, env_p
|
||||
logger.warning("no Matrix credentials available (creds + env both empty); notifications disabled")
|
||||
return None
|
||||
|
||||
|
||||
@@ -436,7 +490,11 @@ def _send_matrix(message: str) -> bool:
|
||||
|
||||
Non-fatal: any failure logs WARN and returns False. Never raises.
|
||||
Implements one 401 retry with token refresh.
|
||||
Silently skips (returns False) if MATRIX_HOMESERVER or MATRIX_ROOM is unconfigured.
|
||||
"""
|
||||
if not MATRIX_HOMESERVER or not MATRIX_ROOM:
|
||||
logger.debug("matrix notifications disabled (homeserver or room unset)")
|
||||
return False
|
||||
with requests.Session() as session:
|
||||
session.headers.update({"User-Agent": USER_AGENT})
|
||||
|
||||
@@ -605,5 +663,86 @@ def main() -> int:
|
||||
release_lock(lock_fd)
|
||||
|
||||
|
||||
EXAMPLE_CONFIG = """\
|
||||
# coe-mirror configuration
|
||||
# Precedence for every key: env var > this file > hardcoded default.
|
||||
# Env var names are COE_MIRROR_<UPPER_KEY>; matrix keys are prefixed COE_MIRROR_MATRIX_.
|
||||
|
||||
api_base = "https://git.churchofmalware.org"
|
||||
|
||||
# Where mirrored zips are stored. One subdirectory per owner is created here.
|
||||
pcloud_target = "~/pCloudDrive/Hacking/ChurchOfMalware"
|
||||
|
||||
# Optional: local staging when pcloud_target is unavailable.
|
||||
# pending_dir = "~/.local/share/coe-mirror/pending"
|
||||
|
||||
# Forgejo owners (users or orgs) to mirror. Edit this list to add/remove.
|
||||
owners = [
|
||||
"Nightmare_Eclipse",
|
||||
"shai_hulud",
|
||||
"bikini_mirror",
|
||||
"ek0mssavi0r",
|
||||
]
|
||||
|
||||
# Optional Matrix notifications. Leave both blank to disable.
|
||||
# User/password come from COE_MIRROR_MATRIX_USER / COE_MIRROR_MATRIX_PASSWORD env vars
|
||||
# (or the `creds` binary if you use Infisical).
|
||||
[matrix]
|
||||
homeserver = ""
|
||||
room = ""
|
||||
"""
|
||||
|
||||
|
||||
def cmd_config_init() -> int:
|
||||
"""Write an example config to CONFIG_PATH. Won't overwrite existing files."""
|
||||
if CONFIG_PATH.exists():
|
||||
print(f"coe-mirror: config already exists at {CONFIG_PATH}; not overwriting", file=sys.stderr)
|
||||
return 1
|
||||
CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
CONFIG_PATH.write_text(EXAMPLE_CONFIG)
|
||||
print(f"coe-mirror: wrote example config to {CONFIG_PATH}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_config_path() -> int:
|
||||
"""Print the config file path (resolved), regardless of whether it exists."""
|
||||
print(str(CONFIG_PATH))
|
||||
return 0
|
||||
|
||||
|
||||
USAGE = """\
|
||||
Usage: coe_mirror.py [command]
|
||||
|
||||
Commands:
|
||||
(no args) Run the mirror (default; used by systemd timer)
|
||||
run Same as no args
|
||||
config init Write an example config.toml to the standard path
|
||||
config path Print the config file path
|
||||
-h, --help Show this message
|
||||
|
||||
Config file: {path}
|
||||
Precedence: env var > config file > hardcoded default
|
||||
"""
|
||||
|
||||
|
||||
def dispatch(argv: List[str]) -> int:
|
||||
if not argv or argv[0] == "run":
|
||||
return main()
|
||||
if argv[0] in ("-h", "--help", "help"):
|
||||
print(USAGE.format(path=CONFIG_PATH))
|
||||
return 0
|
||||
if argv[0] == "config":
|
||||
sub = argv[1] if len(argv) > 1 else ""
|
||||
if sub == "init":
|
||||
return cmd_config_init()
|
||||
if sub == "path":
|
||||
return cmd_config_path()
|
||||
print(USAGE.format(path=CONFIG_PATH), file=sys.stderr)
|
||||
return 2
|
||||
print(f"coe-mirror: unknown command {argv[0]!r}", file=sys.stderr)
|
||||
print(USAGE.format(path=CONFIG_PATH), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
sys.exit(dispatch(sys.argv[1:]))
|
||||
|
||||
Reference in New Issue
Block a user