"""Mirror repos from git.churchofmalware.org Forgejo instance to pCloud. Fetches all repos owned by each user in OWNERS and stores per-commit zip snapshots under `PCLOUD_TARGET//`. Runs unattended via systemd timer twice daily. Local staging when pCloud unavailable. """ import json import logging import logging.handlers import os import re import shutil import subprocess import sys import time import zipfile from datetime import datetime, timedelta, timezone from pathlib import Path from fcntl import flock, LOCK_EX, LOCK_NB from typing import List, Optional, Tuple from urllib.parse import quote import requests # Module constants (overridable via env vars for testability) API_BASE = "https://git.churchofmalware.org" OWNERS = ["Nightmare_Eclipse", "shai_hulud", "bikini_mirror", "ek0mssavi0r"] PCLOUD_TARGET = Path.home() / "pCloudDrive" / "Hacking" / "ChurchOfMalware" PENDING_DIR = Path.home() / ".local" / "share" / "coe-mirror" / "pending" LOG_DIR = Path.home() / ".local" / "share" / "coe-mirror" LOG_FILE = LOG_DIR / "coe-mirror.log" LOCK_NAME = "coe-mirror.lock" SHORT_SHA_LEN = 8 HTTP_TIMEOUT = 30 DOWNLOAD_TIMEOUT = 60 RETRY_BACKOFF = [0, 5, 30] # seconds between attempts; len = max attempts USER_AGENT = "coe-mirror/1.0" REPO_NAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$") # Matrix configuration MATRIX_HOMESERVER = "https://m.mealeyfamily.com" MATRIX_ROOM = "!MBpPtYlQYMcngtpvJt:m.mealeyfamily.com" # General MATRIX_TOKEN_CACHE = LOG_DIR / "matrix-token.json" MATRIX_TIMEOUT = 8 MATRIX_REFRESH_THRESHOLD_SECONDS = 1800 # refresh if <30 min left COE_MIRROR_BOT_USER_KEY = "COE_MIRROR_BOT_USER" COE_MIRROR_BOT_PASSWORD_KEY = "COE_MIRROR_BOT_PASSWORD" # Allow env var overrides for testing and deployment flexibility if env_pcloud := os.getenv("COE_MIRROR_PCLOUD_TARGET"): PCLOUD_TARGET = Path(env_pcloud) if env_pending := os.getenv("COE_MIRROR_PENDING_DIR"): PENDING_DIR = Path(env_pending) if env_hs := os.getenv("COE_MIRROR_MATRIX_HOMESERVER"): MATRIX_HOMESERVER = env_hs if env_room := os.getenv("COE_MIRROR_MATRIX_ROOM"): MATRIX_ROOM = env_room logger: logging.Logger = None # Initialized in setup_logging() def _safe_repo_name(name: str) -> bool: """Validate repo name against whitelist to prevent path traversal.""" return bool(REPO_NAME_RE.fullmatch(name)) def setup_logging() -> None: """Configure logging to file and stderr with rotation.""" global logger LOG_DIR.mkdir(parents=True, exist_ok=True) PENDING_DIR.mkdir(parents=True, exist_ok=True) logger = logging.getLogger("coe-mirror") logger.setLevel(logging.DEBUG) # Rotating file handler file_handler = logging.handlers.RotatingFileHandler( LOG_FILE, maxBytes=5 * 1024 * 1024, backupCount=3 ) file_handler.setLevel(logging.DEBUG) # Stderr handler for systemd journal stderr_handler = logging.StreamHandler(sys.stderr) stderr_handler.setLevel(logging.DEBUG) # Formatter formatter = logging.Formatter( "%(asctime)s %(levelname)s %(name)s: %(message)s" ) file_handler.setFormatter(formatter) stderr_handler.setFormatter(formatter) logger.addHandler(file_handler) logger.addHandler(stderr_handler) def acquire_lock() -> Optional[int]: """Acquire exclusive lock. Return lock fd on success, None if contended.""" xdg_runtime = os.getenv("XDG_RUNTIME_DIR") lock_dir = Path(xdg_runtime) if xdg_runtime else Path("/tmp") lock_file = lock_dir / LOCK_NAME try: fd = os.open(str(lock_file), os.O_CREAT | os.O_WRONLY, 0o600) try: flock(fd, LOCK_EX | LOCK_NB) return fd except OSError: os.close(fd) return None except OSError as exc: logger.error("Failed to open lock file: %s", exc) return None def release_lock(lock_fd: int) -> None: """Release and close the lock file descriptor.""" try: os.close(lock_fd) except OSError: pass def probe_pcloud() -> bool: """Probe pCloud health. Write tiny file, read back, delete. Return True if healthy.""" try: probe_file = PCLOUD_TARGET / ".write_probe" PCLOUD_TARGET.mkdir(parents=True, exist_ok=True) probe_file.write_text("probe") # fsync on file descriptor, not stat with open(probe_file, "r") as f: os.fsync(f.fileno()) content = probe_file.read_text() probe_file.unlink() if content != "probe": logger.warning("pCloud write verification failed (content mismatch); pcloud unavailable; queuing locally") return False return True except Exception as exc: logger.warning("pCloud probe failed: %s; pcloud unavailable; queuing locally", exc) return False def place_into_pcloud(src: Path, dst: Path) -> bool: """Copy src → dst with size verification; unlink src only on success. Returns True on success, False on any failure. Never raises. """ # Defense in depth: if src and dst are the same file, it's a no-op success if src.resolve() == dst.resolve(): logger.debug("src and dst are the same file, no-op: %s", src) return True try: shutil.copy2(src, dst) src_size = src.stat().st_size dst_size = dst.stat().st_size if src_size != dst_size: logger.warning( "size mismatch placing %s → %s (src=%d dst=%d)", src, dst, src_size, dst_size ) try: dst.unlink() except OSError: pass return False src.unlink() return True except Exception as exc: logger.warning("placement failed %s → %s: %s", src, dst, exc) return False def fetch_repos(session: requests.Session, owner: str) -> Optional[list]: """Fetch list of repos owned by `owner`. Return list of repo dicts or None on fatal error.""" url = f"{API_BASE}/api/v1/users/{owner}/repos" for attempt, backoff in enumerate(RETRY_BACKOFF): if attempt > 0: logger.info("Retrying repo list fetch (attempt %d), backoff %ds", attempt + 1, backoff) time.sleep(backoff) try: resp = session.get(url, timeout=HTTP_TIMEOUT) if resp.status_code == 429: logger.debug("Got 429 on repo list, will retry") continue if resp.status_code >= 500: logger.debug("Got %d on repo list, will retry", resp.status_code) continue resp.raise_for_status() return resp.json() except requests.RequestException as exc: logger.debug("Attempt %d: %s", attempt + 1, exc) logger.error("Failed to fetch repos after %d attempts", len(RETRY_BACKOFF)) return None def fetch_branch_sha(session: requests.Session, owner: str, repo_name: str, default_branch: str) -> Optional[str]: """Fetch commit SHA for repo's default branch. Return full hex SHA or None on error.""" url = f"{API_BASE}/api/v1/repos/{owner}/{quote(repo_name, safe='')}/branches/{quote(default_branch, safe='')}" for attempt, backoff in enumerate(RETRY_BACKOFF): if attempt > 0: logger.debug("Retrying branch fetch for %s (attempt %d), backoff %ds", repo_name, attempt + 1, backoff) time.sleep(backoff) try: resp = session.get(url, timeout=HTTP_TIMEOUT) if resp.status_code == 429: logger.debug("Got 429 on branch %s, will retry", repo_name) continue if resp.status_code >= 500: logger.debug("Got %d on branch %s, will retry", resp.status_code, repo_name) continue resp.raise_for_status() data = resp.json() return data.get("commit", {}).get("id") except requests.RequestException as exc: logger.debug("Attempt %d for %s: %s", attempt + 1, repo_name, exc) logger.error("Failed to fetch branch SHA for %s after %d attempts", repo_name, len(RETRY_BACKOFF)) return None def download_repo_zip(session: requests.Session, owner: str, repo_name: str, full_sha: str) -> Optional[Path]: """Download repo zip for given SHA. Return Path to downloaded file (in PENDING_DIR) or None. Pending filename is `{owner}__{repo}_{sha8}.zip` so owners with identical repo names don't collide in the shared PENDING_DIR. flush_pending() strips the owner prefix when routing to the per-owner pcloud subdir. """ # Defense in depth: validate repo_name even though main() should have done so if not _safe_repo_name(repo_name) or not _safe_repo_name(owner): logger.error("refusing unsafe owner/repo name in download_repo_zip: %r/%r", owner, repo_name) return None filename = f"{owner}__{repo_name}_{full_sha[:SHORT_SHA_LEN]}.zip" partial_path = PENDING_DIR / f"{filename}.partial" final_path = PENDING_DIR / filename # URL-encode repo_name and full_sha for safety (defense in depth after main-level validation) url = f"{API_BASE}/{owner}/{quote(repo_name, safe='')}/archive/{quote(full_sha, safe='')}.zip" for attempt, backoff in enumerate(RETRY_BACKOFF): if attempt > 0: logger.debug("Retrying download for %s (attempt %d), backoff %ds", filename, attempt + 1, backoff) time.sleep(backoff) try: resp = session.get(url, timeout=DOWNLOAD_TIMEOUT, stream=True) if resp.status_code == 429: logger.debug("Got 429 on download %s, will retry", filename) continue if resp.status_code >= 500: logger.debug("Got %d on download %s, will retry", resp.status_code, filename) continue resp.raise_for_status() # Stream to temp file partial_path.parent.mkdir(parents=True, exist_ok=True) with open(partial_path, "wb") as f: for chunk in resp.iter_content(chunk_size=65536): if chunk: f.write(chunk) # Validate zip if not zipfile.is_zipfile(partial_path): logger.error("Downloaded file is not a valid zip: %s", filename) partial_path.unlink() return None # Rename to final partial_path.rename(final_path) logger.info("Downloaded %s", filename) return final_path except requests.RequestException as exc: logger.debug("Attempt %d for %s: %s", attempt + 1, filename, exc) except Exception as exc: logger.error("Error downloading %s: %s", filename, exc) if partial_path.exists(): partial_path.unlink() return None logger.error("Failed to download %s after %d attempts", filename, len(RETRY_BACKOFF)) if partial_path.exists(): partial_path.unlink() return None def flush_pending(pcloud_healthy: bool) -> list: """Move all *.zip files from PENDING_DIR to PCLOUD_TARGET// if healthy. Pending filenames are `{owner}__{repo}_{sha8}.zip`. The owner prefix is stripped when placing into pcloud (final name is `{repo}_{sha8}.zip` inside owner subdir). Return list of (owner, repo, short_sha, final_location) tuples for files moved. """ moved = [] if not pcloud_healthy or not PENDING_DIR.exists(): return moved for pending_file in PENDING_DIR.glob("*.zip"): # Expect `{owner}__{repo}_{sha8}.zip` if "__" not in pending_file.name: logger.warning("skipping pending file without owner prefix: %s", pending_file.name) continue owner, rest = pending_file.name.split("__", 1) if not _safe_repo_name(owner): logger.warning("refusing unsafe owner prefix in pending file: %r", owner) continue owner_dir = PCLOUD_TARGET / owner owner_dir.mkdir(parents=True, exist_ok=True) dst = owner_dir / rest if place_into_pcloud(pending_file, dst): # Extract repo name and short sha from remainder (`{repo}_{sha8}.zip`) name_parts = Path(rest).stem.rsplit("_", 1) if len(name_parts) == 2: repo, short_sha = name_parts moved.append((owner, repo, short_sha, str(dst))) logger.info("Flushed %s to pCloud", pending_file.name) return moved def _read_token_cache() -> Optional[dict]: """Return cached token dict or None on any failure. Never raises.""" try: if not MATRIX_TOKEN_CACHE.exists(): return None with open(MATRIX_TOKEN_CACHE) as f: data = json.load(f) # Shape check if "access_token" in data and "expires_at" in data: return data except Exception as exc: logger.debug("token cache read failed: %s", exc) return None def _write_token_cache(access_token: str, expires_at_iso: str) -> None: """Write token cache with 0600 permissions. Never raises.""" try: MATRIX_TOKEN_CACHE.parent.mkdir(parents=True, exist_ok=True) payload = {"access_token": access_token, "expires_at": expires_at_iso} # Write then chmod for atomicity-ish behavior tmp = MATRIX_TOKEN_CACHE.with_suffix(".tmp") with open(tmp, "w") as f: json.dump(payload, f) os.chmod(tmp, 0o600) os.replace(tmp, MATRIX_TOKEN_CACHE) except Exception as exc: logger.warning("token cache write failed: %s", exc) def _fetch_creds() -> Optional[Tuple[str, str]]: """Fetch (user, password) from Infisical via creds. Return None on failure.""" try: user = subprocess.run( ["creds", "get", COE_MIRROR_BOT_USER_KEY, "matrix"], capture_output=True, text=True, timeout=5 ) pw = subprocess.run( ["creds", "get", COE_MIRROR_BOT_PASSWORD_KEY, "matrix"], capture_output=True, text=True, timeout=5 ) if user.returncode == 0 and pw.returncode == 0: u, p = user.stdout.strip(), pw.stdout.strip() if u and p: return u, p logger.warning("creds returned non-zero for %s/%s", COE_MIRROR_BOT_USER_KEY, COE_MIRROR_BOT_PASSWORD_KEY) except FileNotFoundError: logger.warning("creds binary not found on PATH") except subprocess.TimeoutExpired: logger.warning("creds timed out fetching Matrix credentials") except Exception as exc: logger.warning("creds fetch failed: %s", exc) return None def _matrix_login(session: requests.Session) -> Optional[str]: """Login via password, cache token, return access_token. None on failure.""" creds = _fetch_creds() if not creds: return None user, password = creds url = f"{MATRIX_HOMESERVER}/_matrix/client/r0/login" body = {"type": "m.login.password", "user": user, "password": password} try: resp = session.post(url, json=body, timeout=MATRIX_TIMEOUT) if resp.status_code != 200: logger.warning("matrix login failed: HTTP %d", resp.status_code) return None data = resp.json() token = data.get("access_token") expires_in_ms = data.get("expires_in_ms", 0) if not token: logger.warning("matrix login returned no access_token") return None expires_at = datetime.now(timezone.utc) + timedelta(milliseconds=expires_in_ms) _write_token_cache(token, expires_at.isoformat()) return token except requests.RequestException as exc: logger.warning("matrix login network error: %s", exc) except Exception as exc: logger.warning("matrix login unexpected error: %s", exc) return None def _get_matrix_token(session: requests.Session, force_refresh: bool = False) -> Optional[str]: """Return a valid token. Read cache → check expiry → login if needed.""" if not force_refresh: cached = _read_token_cache() if cached: try: expires_at = datetime.fromisoformat(cached["expires_at"]) remaining = (expires_at - datetime.now(timezone.utc)).total_seconds() if remaining > MATRIX_REFRESH_THRESHOLD_SECONDS: return cached["access_token"] except Exception as exc: logger.debug("cache expiry parse failed: %s", exc) return _matrix_login(session) def _send_matrix(message: str) -> bool: """Send a Matrix message to MATRIX_ROOM. Return True on success. Non-fatal: any failure logs WARN and returns False. Never raises. Implements one 401 retry with token refresh. """ with requests.Session() as session: session.headers.update({"User-Agent": USER_AGENT}) encoded_room = quote(MATRIX_ROOM, safe="") url = f"{MATRIX_HOMESERVER}/_matrix/client/r0/rooms/{encoded_room}/send/m.room.message" body = {"msgtype": "m.text", "body": message} for attempt in (1, 2): token = _get_matrix_token(session, force_refresh=(attempt == 2)) if not token: logger.warning("matrix send aborted: no token (attempt %d)", attempt) return False try: resp = session.post( url, json=body, headers={"Authorization": f"Bearer {token}"}, timeout=MATRIX_TIMEOUT, ) if resp.status_code == 200: logger.info("matrix sent: %s", message[:80]) return True if resp.status_code == 401 and attempt == 1: logger.info("matrix 401 on send; refreshing token and retrying") continue logger.warning("matrix send failed: HTTP %d body=%s", resp.status_code, resp.text[:200]) return False except requests.RequestException as exc: logger.warning("matrix send network error: %s", exc) return False except Exception as exc: logger.warning("matrix send unexpected error: %s", exc) return False def notify(summary: List[Tuple[str, str, str, str]], failed_owners: Optional[List[str]] = None) -> None: """Build a Matrix message and POST it. Non-fatal. `summary` is a list of (owner, repo, short_sha, final_location). `failed_owners` is a list of owners whose repo-list fetch failed this run. """ failed_owners = failed_owners or [] if not summary and not failed_owners: return parts = [] if summary: by_owner: dict = {} for owner, repo, short_sha, _ in summary: by_owner.setdefault(owner, []).append(f"{repo}@{short_sha}") chunks = [f"{owner}: {', '.join(items)}" for owner, items in by_owner.items()] parts.append(f"coe-mirror: {len(summary)} new — " + " | ".join(chunks)) if failed_owners: parts.append(f"{len(failed_owners)} owner(s) failed: {', '.join(failed_owners)}") text = " ; ".join(parts) # Always log the intended message so behavior is visible even when Matrix is down logger.info("notify: %s", text) _send_matrix(text) def main() -> int: """Main entry point.""" setup_logging() lock_fd = acquire_lock() if lock_fd is None: logger.debug("Lock already held; another instance running, exiting gracefully") return 0 try: logger.info("coe-mirror starting") # Probe pCloud health pcloud_healthy = probe_pcloud() logger.info("pCloud health check: %s", "healthy" if pcloud_healthy else "unavailable") # Flush pending at start initial_flush = flush_pending(pcloud_healthy) # Create session and iterate owners session = requests.Session() session.headers.update({"User-Agent": USER_AGENT}) new_files = list(initial_flush) failed_owners: List[str] = [] for owner in OWNERS: if not _safe_repo_name(owner): logger.warning("refusing unsafe owner name: %r", owner) failed_owners.append(owner) continue repos = fetch_repos(session, owner) if repos is None: logger.error("Could not fetch repo list for %s; continuing to next owner", owner) failed_owners.append(owner) continue logger.info("Fetched %d repos for %s", len(repos), owner) # Ensure per-owner pcloud subdir exists (best-effort; probe already covered parent) owner_pcloud = PCLOUD_TARGET / owner if pcloud_healthy: try: owner_pcloud.mkdir(parents=True, exist_ok=True) except OSError as exc: logger.warning("could not create owner subdir %s: %s", owner_pcloud, exc) # Process each repo for this owner for repo in repos: repo_name = repo.get("name") default_branch = repo.get("default_branch", "main") if not repo_name: logger.warning("Repo object missing 'name' (owner=%s)", owner) continue # Reject unsafe repo names to prevent path traversal if not _safe_repo_name(repo_name): logger.warning("refusing unsafe repo name: %r (owner=%s)", repo_name, owner) continue sha = fetch_branch_sha(session, owner, repo_name, default_branch) if sha is None: logger.error("Skipping %s/%s (could not fetch branch SHA)", owner, repo_name) continue short_sha = sha[:SHORT_SHA_LEN] # pcloud final name: {repo}_{sha8}.zip inside owner subdir pcloud_final = f"{repo_name}_{short_sha}.zip" pcloud_path = owner_pcloud / pcloud_final # pending name carries owner prefix to avoid cross-owner collisions pending_name = f"{owner}__{pcloud_final}" pending_path = PENDING_DIR / pending_name already_have = (pcloud_path.exists() and pcloud_path.stat().st_size > 0) \ or (pending_path.exists() and pending_path.stat().st_size > 0) if already_have: logger.info("Skipping %s (already present)", pending_name) continue downloaded = download_repo_zip(session, owner, repo_name, sha) if downloaded: # Try to place it if pcloud_healthy: dst = owner_pcloud / pcloud_final if place_into_pcloud(downloaded, dst): new_files.append((owner, repo_name, short_sha, str(dst))) else: logger.info("File %s left in pending (placement failed)", downloaded.name) new_files.append((owner, repo_name, short_sha, str(downloaded))) else: logger.info("pCloud unhealthy; %s queued in pending", downloaded.name) new_files.append((owner, repo_name, short_sha, str(downloaded))) # Flush pending at end final_flush = flush_pending(pcloud_healthy) new_files.extend(final_flush) # Notify notify(new_files, failed_owners=failed_owners) logger.info("coe-mirror complete") return 0 finally: release_lock(lock_fd) if __name__ == "__main__": sys.exit(main())