n0mad1k/coe-mirror
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Genericize: drop pCloud terminology, use TARGET_DIR / target_dir

Browse Source 
pCloud was never a hard dependency - the tool writes to any local directory.
Renaming makes the tool clearly generic before publishing.

- PCLOUD_TARGET → TARGET_DIR
- probe_pcloud → probe_target
- place_into_pcloud → place_into_target
- pcloud_target config key → target_dir
- COE_MIRROR_PCLOUD_TARGET env var → COE_MIRROR_TARGET_DIR
- Hardcoded default: ~/pCloudDrive/Hacking/ChurchOfMalware → ~/coe-mirror
- Log messages and docstrings updated
- Tests updated to match

DevTrack #1286
This commit is contained in:
n0mad1k
2026-07-07 11:12:46 -04:00
parent 2093b737c3
commit 376a6d3a25
3 changed files with 111 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coe_mirror.py
+42 -42
View File
@@ -1,7 +1,7 @@
"""Mirror repos from a Forgejo instance to a local target dir (e.g. pCloud).
"""Mirror repos from a Forgejo/Gitea instance to a local target directory.
Fetches all repos owned by each user in OWNERS and stores per-commit zip snapshots
under `PCLOUD_TARGET/<owner>/`. Runs unattended via systemd timer.
under `TARGET_DIR/<owner>/`. Runs unattended via systemd timer.
Local staging when the target is unavailable.
Configuration precedence (for each key): env var > config file > hardcoded default.
@@ -67,11 +67,11 @@ OWNERS: List[str] = (
or (_CFG.get("owners") if isinstance(_CFG.get("owners"), list) else None)
or ["Nightmare_Eclipse", "shai_hulud", "bikini_mirror", "ek0mssavi0r"]
)
PCLOUD_TARGET: Path = Path(
TARGET_DIR: Path = Path(
_resolve(
"COE_MIRROR_PCLOUD_TARGET",
_CFG.get("pcloud_target"),
str(Path.home() / "pCloudDrive" / "Hacking" / "ChurchOfMalware"),
"COE_MIRROR_TARGET_DIR",
_CFG.get("target_dir"),
str(Path.home() / "coe-mirror"),
)
).expanduser()
PENDING_DIR: Path = Path(
@@ -165,11 +165,11 @@ def release_lock(lock_fd: int) -> None:
pass
def probe_pcloud() -> bool:
"""Probe pCloud health. Write tiny file, read back, delete. Return True if healthy."""
def probe_target() -> bool:
"""Probe target directory health. Write tiny file, read back, delete. Return True if healthy."""
try:
probe_file = PCLOUD_TARGET / ".write_probe"
PCLOUD_TARGET.mkdir(parents=True, exist_ok=True)
probe_file = TARGET_DIR / ".write_probe"
TARGET_DIR.mkdir(parents=True, exist_ok=True)
probe_file.write_text("probe")
# fsync on file descriptor, not stat
@@ -179,15 +179,15 @@ def probe_pcloud() -> bool:
probe_file.unlink()
if content != "probe":
logger.warning("pCloud write verification failed (content mismatch); pcloud unavailable; queuing locally")
logger.warning("target directory write verification failed (content mismatch); target unavailable; queuing locally")
return False
return True
except Exception as exc:
logger.warning("pCloud probe failed: %s; pcloud unavailable; queuing locally", exc)
logger.warning("target directory probe failed: %s; target unavailable; queuing locally", exc)
return False
def place_into_pcloud(src: Path, dst: Path) -> bool:
def place_into_target(src: Path, dst: Path) -> bool:
"""Copy src → dst with size verification; unlink src only on success.
Returns True on success, False on any failure. Never raises.
@@ -277,7 +277,7 @@ def download_repo_zip(session: requests.Session, owner: str, repo_name: str, ful
Pending filename is `{owner}__{repo}_{sha8}.zip` so owners with identical repo names
don't collide in the shared PENDING_DIR. flush_pending() strips the owner prefix
when routing to the per-owner pcloud subdir.
when routing to the per-owner target subdir.
"""
# Defense in depth: validate repo_name even though main() should have done so
if not _safe_repo_name(repo_name) or not _safe_repo_name(owner):
@@ -339,16 +339,16 @@ def download_repo_zip(session: requests.Session, owner: str, repo_name: str, ful
return None
def flush_pending(pcloud_healthy: bool) -> list:
"""Move all *.zip files from PENDING_DIR to PCLOUD_TARGET/<owner>/ if healthy.
def flush_pending(target_healthy: bool) -> list:
"""Move all *.zip files from PENDING_DIR to TARGET_DIR/<owner>/ if healthy.
Pending filenames are `{owner}__{repo}_{sha8}.zip`. The owner prefix is stripped
when placing into pcloud (final name is `{repo}_{sha8}.zip` inside owner subdir).
when placing into target (final name is `{repo}_{sha8}.zip` inside owner subdir).
Return list of (owner, repo, short_sha, final_location) tuples for files moved.
"""
moved = []
if not pcloud_healthy or not PENDING_DIR.exists():
if not target_healthy or not PENDING_DIR.exists():
return moved
for pending_file in PENDING_DIR.glob("*.zip"):
@@ -360,16 +360,16 @@ def flush_pending(pcloud_healthy: bool) -> list:
if not _safe_repo_name(owner):
logger.warning("refusing unsafe owner prefix in pending file: %r", owner)
continue
owner_dir = PCLOUD_TARGET / owner
owner_dir = TARGET_DIR / owner
owner_dir.mkdir(parents=True, exist_ok=True)
dst = owner_dir / rest
if place_into_pcloud(pending_file, dst):
if place_into_target(pending_file, dst):
# Extract repo name and short sha from remainder (`{repo}_{sha8}.zip`)
name_parts = Path(rest).stem.rsplit("_", 1)
if len(name_parts) == 2:
repo, short_sha = name_parts
moved.append((owner, repo, short_sha, str(dst)))
logger.info("Flushed %s to pCloud", pending_file.name)
logger.info("Flushed %s to target directory", pending_file.name)
return moved
@@ -567,12 +567,12 @@ def main() -> int:
try:
logger.info("coe-mirror starting")
# Probe pCloud health
pcloud_healthy = probe_pcloud()
logger.info("pCloud health check: %s", "healthy" if pcloud_healthy else "unavailable")
# Probe target directory health
target_healthy = probe_target()
logger.info("target directory health check: %s", "healthy" if target_healthy else "unavailable")
# Flush pending at start
initial_flush = flush_pending(pcloud_healthy)
initial_flush = flush_pending(target_healthy)
# Create session and iterate owners
session = requests.Session()
@@ -595,13 +595,13 @@ def main() -> int:
logger.info("Fetched %d repos for %s", len(repos), owner)
# Ensure per-owner pcloud subdir exists (best-effort; probe already covered parent)
owner_pcloud = PCLOUD_TARGET / owner
if pcloud_healthy:
# Ensure per-owner target subdir exists (best-effort; probe already covered parent)
owner_target = TARGET_DIR / owner
if target_healthy:
try:
owner_pcloud.mkdir(parents=True, exist_ok=True)
owner_target.mkdir(parents=True, exist_ok=True)
except OSError as exc:
logger.warning("could not create owner subdir %s: %s", owner_pcloud, exc)
logger.warning("could not create owner subdir %s: %s", owner_target, exc)
# Process each repo for this owner
for repo in repos:
@@ -623,13 +623,13 @@ def main() -> int:
continue
short_sha = sha[:SHORT_SHA_LEN]
# pcloud final name: {repo}_{sha8}.zip inside owner subdir
pcloud_final = f"{repo_name}_{short_sha}.zip"
pcloud_path = owner_pcloud / pcloud_final
# final name in target: {repo}_{sha8}.zip inside owner subdir
target_final = f"{repo_name}_{short_sha}.zip"
target_path = owner_target / target_final
# pending name carries owner prefix to avoid cross-owner collisions
pending_name = f"{owner}__{pcloud_final}"
pending_name = f"{owner}__{target_final}"
pending_path = PENDING_DIR / pending_name
already_have = (pcloud_path.exists() and pcloud_path.stat().st_size > 0) \
already_have = (target_path.exists() and target_path.stat().st_size > 0) \
or (pending_path.exists() and pending_path.stat().st_size > 0)
if already_have:
logger.info("Skipping %s (already present)", pending_name)
@@ -638,19 +638,19 @@ def main() -> int:
downloaded = download_repo_zip(session, owner, repo_name, sha)
if downloaded:
# Try to place it
if pcloud_healthy:
dst = owner_pcloud / pcloud_final
if place_into_pcloud(downloaded, dst):
if target_healthy:
dst = owner_target / target_final
if place_into_target(downloaded, dst):
new_files.append((owner, repo_name, short_sha, str(dst)))
else:
logger.info("File %s left in pending (placement failed)", downloaded.name)
new_files.append((owner, repo_name, short_sha, str(downloaded)))
else:
logger.info("pCloud unhealthy; %s queued in pending", downloaded.name)
logger.info("target directory unhealthy; %s queued in pending", downloaded.name)
new_files.append((owner, repo_name, short_sha, str(downloaded)))
# Flush pending at end
final_flush = flush_pending(pcloud_healthy)
final_flush = flush_pending(target_healthy)
new_files.extend(final_flush)
# Notify
@@ -671,9 +671,9 @@ EXAMPLE_CONFIG = """\
api_base = "https://git.churchofmalware.org"
# Where mirrored zips are stored. One subdirectory per owner is created here.
pcloud_target = "~/pCloudDrive/Hacking/ChurchOfMalware"
target_dir = "~/coe-mirror"
# Optional: local staging when pcloud_target is unavailable.
# Optional: local staging when target_dir is unavailable.
# pending_dir = "~/.local/share/coe-mirror/pending"
# Forgejo owners (users or orgs) to mirror. Edit this list to add/remove.