n0mad1k/coe-mirror
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mirror all four Church of Malware owners, per-owner pcloud subdirs

Browse Source 
- OWNERS list replaces single OWNER: Nightmare_Eclipse, shai_hulud, bikini_mirror, ek0mssavi0r
- PCLOUD_TARGET moves to ChurchOfMalware/ parent, per-owner subdirs at runtime
- Owner threaded through fetch_repos, fetch_branch_sha, download_repo_zip
- Pending filenames get owner__ prefix to prevent cross-owner collisions
- flush_pending strips owner prefix when routing to owner subdir
- Per-owner fetch failure logs and continues; failed_owners reported in notify
- notify groups new files by owner; also picks up flushes done at run start
- Tests: parametrize on OWNERS via monkeypatch, add multi-owner collision and
  per-owner failure isolation tests

DevTrack #1284
This commit is contained in:
n0mad1k
2026-07-06 12:35:56 -04:00
parent 4d0fbff039
commit 9ffe38cbbc
3 changed files with 252 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_coe_mirror.py
+130 -41
View File
@@ -33,6 +33,7 @@ def mock_paths(tmp_path, monkeypatch):
log_dir = tmp_path / "logs"
pcloud_target.mkdir()
(pcloud_target / "TestOwner").mkdir()
pending_dir.mkdir()
log_dir.mkdir()
@@ -40,6 +41,7 @@ def mock_paths(tmp_path, monkeypatch):
monkeypatch.setattr(coe_mirror, "PENDING_DIR", pending_dir)
monkeypatch.setattr(coe_mirror, "LOG_DIR", log_dir)
monkeypatch.setattr(coe_mirror, "LOG_FILE", log_dir / "test.log")
monkeypatch.setattr(coe_mirror, "OWNERS", ["TestOwner"])
return {
"pcloud_target": pcloud_target,
@@ -94,12 +96,12 @@ class TestNewRepoDownloadsAndPlacesInPcloud:
with requests_mock.Mocker() as m:
# Mock repo list
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[{"name": "test-repo", "default_branch": "main"}],
)
# Mock branch info
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{coe_mirror.OWNER}/test-repo/branches/main",
f"{coe_mirror.API_BASE}/api/v1/repos/TestOwner/test-repo/branches/main",
json={"commit": {"id": "aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj"}},
)
@@ -110,7 +112,7 @@ class TestNewRepoDownloadsAndPlacesInPcloud:
zip_buffer.seek(0)
m.get(
f"{coe_mirror.API_BASE}/{coe_mirror.OWNER}/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
f"{coe_mirror.API_BASE}/TestOwner/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
content=zip_buffer.getvalue(),
)
@@ -118,7 +120,7 @@ class TestNewRepoDownloadsAndPlacesInPcloud:
assert result == 0
# Verify file landed in pCloud with expected name
expected_file = mock_paths["pcloud_target"] / "test-repo_aaaabbbb.zip"
expected_file = mock_paths["pcloud_target"] / "TestOwner" / "test-repo_aaaabbbb.zip"
assert expected_file.exists()
assert expected_file.stat().st_size > 0
@@ -136,17 +138,17 @@ class TestNewCommitCreatesSnapshot:
monkeypatch.setattr(coe_mirror, "probe_pcloud", lambda: True)
# Pre-create old file
old_file = mock_paths["pcloud_target"] / "test-repo_aaaabbbb.zip"
old_file = mock_paths["pcloud_target"] / "TestOwner" / "test-repo_aaaabbbb.zip"
create_valid_zip(old_file, "old")
with requests_mock.Mocker() as m:
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[{"name": "test-repo", "default_branch": "main"}],
)
# Return new SHA
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{coe_mirror.OWNER}/test-repo/branches/main",
f"{coe_mirror.API_BASE}/api/v1/repos/TestOwner/test-repo/branches/main",
json={"commit": {"id": "bbbbccccddddeeeeffffgggghhhhiiiijjjjkkkk"}},
)
@@ -157,7 +159,7 @@ class TestNewCommitCreatesSnapshot:
zip_buffer.seek(0)
m.get(
f"{coe_mirror.API_BASE}/{coe_mirror.OWNER}/test-repo/archive/bbbbccccddddeeeeffffgggghhhhiiiijjjjkkkk.zip",
f"{coe_mirror.API_BASE}/TestOwner/test-repo/archive/bbbbccccddddeeeeffffgggghhhhiiiijjjjkkkk.zip",
content=zip_buffer.getvalue(),
)
@@ -167,7 +169,7 @@ class TestNewCommitCreatesSnapshot:
# Verify old file still exists
assert old_file.exists()
# Verify new file landed
new_file = mock_paths["pcloud_target"] / "test-repo_bbbbcccc.zip"
new_file = mock_paths["pcloud_target"] / "TestOwner" / "test-repo_bbbbcccc.zip"
assert new_file.exists()
@@ -182,7 +184,7 @@ class TestPcloudFailureKeepsPending:
# Make pCloud target a file (not a dir) to cause write failure
pcloud_as_file = mock_paths["pcloud_target"]
pcloud_as_file.rmdir()
shutil.rmtree(pcloud_as_file)
pcloud_as_file.write_text("block")
# Mock place_into_pcloud to fail
@@ -193,11 +195,11 @@ class TestPcloudFailureKeepsPending:
with requests_mock.Mocker() as m:
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[{"name": "test-repo", "default_branch": "main"}],
)
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{coe_mirror.OWNER}/test-repo/branches/main",
f"{coe_mirror.API_BASE}/api/v1/repos/TestOwner/test-repo/branches/main",
json={"commit": {"id": "aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj"}},
)
@@ -207,7 +209,7 @@ class TestPcloudFailureKeepsPending:
zip_buffer.seek(0)
m.get(
f"{coe_mirror.API_BASE}/{coe_mirror.OWNER}/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
f"{coe_mirror.API_BASE}/TestOwner/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
content=zip_buffer.getvalue(),
)
@@ -215,7 +217,7 @@ class TestPcloudFailureKeepsPending:
assert result == 0
# Verify file is in pending
pending_file = mock_paths["pending_dir"] / "test-repo_aaaabbbb.zip"
pending_file = mock_paths["pending_dir"] / "TestOwner__test-repo_aaaabbbb.zip"
assert pending_file.exists()
@@ -232,13 +234,13 @@ class TestPendingFlushesWhenPcloudRecoveres:
monkeypatch.setattr(coe_mirror, "probe_pcloud", lambda: True)
# Create a pending file
pending_file = mock_paths["pending_dir"] / "test-repo_aaaabbbb.zip"
pending_file = mock_paths["pending_dir"] / "TestOwner__test-repo_aaaabbbb.zip"
create_valid_zip(pending_file, "pending")
with requests_mock.Mocker() as m:
# Empty repo list (no new downloads)
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[],
)
@@ -246,7 +248,7 @@ class TestPendingFlushesWhenPcloudRecoveres:
assert result == 0
# Verify file moved to pCloud
pcloud_file = mock_paths["pcloud_target"] / "test-repo_aaaabbbb.zip"
pcloud_file = mock_paths["pcloud_target"] / "TestOwner" / "test-repo_aaaabbbb.zip"
assert pcloud_file.exists()
assert not pending_file.exists()
@@ -262,16 +264,16 @@ class TestMalformedZipRejected:
with requests_mock.Mocker() as m:
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[{"name": "test-repo", "default_branch": "main"}],
)
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{coe_mirror.OWNER}/test-repo/branches/main",
f"{coe_mirror.API_BASE}/api/v1/repos/TestOwner/test-repo/branches/main",
json={"commit": {"id": "aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj"}},
)
# Return binary garbage
m.get(
f"{coe_mirror.API_BASE}/{coe_mirror.OWNER}/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
f"{coe_mirror.API_BASE}/TestOwner/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
content=b"GARBAGE_DATA_NOT_A_ZIP",
)
@@ -279,9 +281,9 @@ class TestMalformedZipRejected:
assert result == 0
# Verify nothing in pCloud or pending
pcloud_file = mock_paths["pcloud_target"] / "test-repo_aaaabbbb.zip"
pcloud_file = mock_paths["pcloud_target"] / "TestOwner" / "test-repo_aaaabbbb.zip"
assert not pcloud_file.exists()
pending_file = mock_paths["pending_dir"] / "test-repo_aaaabbbb.zip"
pending_file = mock_paths["pending_dir"] / "TestOwner__test-repo_aaaabbbb.zip"
assert not pending_file.exists()
@@ -294,9 +296,9 @@ class TestNotifyStub:
original_notify = coe_mirror.notify
call_log = []
def spy_notify(summary):
def spy_notify(summary, failed_owners=None):
call_log.append(summary)
return original_notify(summary)
return original_notify(summary, failed_owners=failed_owners)
monkeypatch.setattr(coe_mirror, "notify", spy_notify)
@@ -307,7 +309,7 @@ class TestNotifyStub:
# Test with files
call_log.clear()
coe_mirror.notify([("repo", "aaaabbbb", "/path")])
coe_mirror.notify([("TestOwner", "repo", "aaaabbbb", "/path")])
assert len(call_log) == 1
@@ -334,7 +336,7 @@ class TestApiRetriesOn429:
with requests_mock.Mocker() as m:
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
[
{"status_code": 429, "json": None},
{"status_code": 429, "json": None},
@@ -342,7 +344,7 @@ class TestApiRetriesOn429:
],
)
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{coe_mirror.OWNER}/test-repo/branches/main",
f"{coe_mirror.API_BASE}/api/v1/repos/TestOwner/test-repo/branches/main",
json={"commit": {"id": "aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj"}},
)
@@ -352,7 +354,7 @@ class TestApiRetriesOn429:
zip_buffer.seek(0)
m.get(
f"{coe_mirror.API_BASE}/{coe_mirror.OWNER}/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
f"{coe_mirror.API_BASE}/TestOwner/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
content=zip_buffer.getvalue(),
)
@@ -379,11 +381,11 @@ class TestSizeMismatchKeepsPending:
with requests_mock.Mocker() as m:
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[{"name": "test-repo", "default_branch": "main"}],
)
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{coe_mirror.OWNER}/test-repo/branches/main",
f"{coe_mirror.API_BASE}/api/v1/repos/TestOwner/test-repo/branches/main",
json={"commit": {"id": "aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj"}},
)
@@ -393,7 +395,7 @@ class TestSizeMismatchKeepsPending:
zip_buffer.seek(0)
m.get(
f"{coe_mirror.API_BASE}/{coe_mirror.OWNER}/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
f"{coe_mirror.API_BASE}/TestOwner/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
content=zip_buffer.getvalue(),
)
@@ -401,11 +403,11 @@ class TestSizeMismatchKeepsPending:
assert result == 0
# Verify bad dst was removed
bad_dst = mock_paths["pcloud_target"] / "test-repo_aaaabbbb.zip"
bad_dst = mock_paths["pcloud_target"] / "TestOwner" / "test-repo_aaaabbbb.zip"
assert not bad_dst.exists()
# Verify src still in pending
pending_src = mock_paths["pending_dir"] / "test-repo_aaaabbbb.zip"
pending_src = mock_paths["pending_dir"] / "TestOwner__test-repo_aaaabbbb.zip"
assert pending_src.exists()
@@ -419,7 +421,7 @@ class TestExistenceCheckSkipsZeroByte:
monkeypatch.setattr(time, "sleep", lambda x: None)
# Pre-create zero-byte file in pCloud target
zero_file = mock_paths["pcloud_target"] / "test-repo_aaaabbbb.zip"
zero_file = mock_paths["pcloud_target"] / "TestOwner" / "test-repo_aaaabbbb.zip"
zero_file.write_bytes(b"")
assert zero_file.stat().st_size == 0
@@ -433,11 +435,11 @@ class TestExistenceCheckSkipsZeroByte:
with requests_mock.Mocker() as m:
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[{"name": "test-repo", "default_branch": "main"}],
)
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{coe_mirror.OWNER}/test-repo/branches/main",
f"{coe_mirror.API_BASE}/api/v1/repos/TestOwner/test-repo/branches/main",
json={"commit": {"id": "aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj"}},
)
@@ -447,7 +449,7 @@ class TestExistenceCheckSkipsZeroByte:
zip_buffer.seek(0)
m.get(
f"{coe_mirror.API_BASE}/{coe_mirror.OWNER}/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
f"{coe_mirror.API_BASE}/TestOwner/test-repo/archive/aaaabbbbccccddddeeeeffffgggghhhhiiiijjjj.zip",
content=zip_buffer.getvalue(),
)
@@ -502,7 +504,7 @@ class TestUnsafeRepoNameRejected:
with requests_mock.Mocker() as m:
# Mock repo list returning unsafe repo name
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[{"name": "../../../etc/passwd", "default_branch": "main"}],
)
@@ -530,7 +532,7 @@ class TestSkipWhenFileAlreadyInPcloud:
monkeypatch.setattr(coe_mirror, "probe_pcloud", lambda: True)
# Pre-create a non-empty file in pCloud
existing_file = mock_paths["pcloud_target"] / "RepoX_abcdef12.zip"
existing_file = mock_paths["pcloud_target"] / "TestOwner" / "RepoX_abcdef12.zip"
create_valid_zip(existing_file, "existing")
# Spy on download_repo_zip to ensure it's not called
@@ -546,12 +548,12 @@ class TestSkipWhenFileAlreadyInPcloud:
with requests_mock.Mocker() as m:
# Mock repo list
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{coe_mirror.OWNER}/repos",
f"{coe_mirror.API_BASE}/api/v1/users/TestOwner/repos",
json=[{"name": "RepoX", "default_branch": "main"}],
)
# Return matching SHA (first 8 chars = abcdef12)
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{coe_mirror.OWNER}/RepoX/branches/main",
f"{coe_mirror.API_BASE}/api/v1/repos/TestOwner/RepoX/branches/main",
json={"commit": {"id": "abcdef1234567890"}},
)
@@ -824,3 +826,90 @@ class TestMatrixLoginWritesCacheChmod600:
with open(token_cache_file) as f:
cached = json.load(f)
assert cached["access_token"] == "new-token"
class TestMultiOwner:
"""Multi-owner behavior: collisions, per-owner failure isolation."""
def test_same_repo_name_across_owners_does_not_collide(self, mock_paths, setup_logging, monkeypatch):
"""Two owners each have a repo named 'shared' — both land in their own subdirs."""
monkeypatch.setattr(coe_mirror, "OWNERS", ["OwnerA", "OwnerB"])
(mock_paths["pcloud_target"] / "OwnerA").mkdir(exist_ok=True)
(mock_paths["pcloud_target"] / "OwnerB").mkdir(exist_ok=True)
monkeypatch.setattr(coe_mirror, "acquire_lock", lambda: 999)
monkeypatch.setattr(coe_mirror, "release_lock", lambda fd: None)
monkeypatch.setattr(time, "sleep", lambda x: None)
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr("README.md", "hi")
zip_bytes = buf.getvalue()
sha_a = "a" * 40
sha_b = "b" * 40
with requests_mock.Mocker() as m:
for owner, sha in (("OwnerA", sha_a), ("OwnerB", sha_b)):
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/{owner}/repos",
json=[{"name": "shared", "default_branch": "main"}],
)
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/{owner}/shared/branches/main",
json={"commit": {"id": sha}},
)
m.get(
f"{coe_mirror.API_BASE}/{owner}/shared/archive/{sha}.zip",
content=zip_bytes,
)
rc = coe_mirror.main()
assert rc == 0
a_file = mock_paths["pcloud_target"] / "OwnerA" / f"shared_{sha_a[:8]}.zip"
b_file = mock_paths["pcloud_target"] / "OwnerB" / f"shared_{sha_b[:8]}.zip"
assert a_file.exists() and a_file.stat().st_size > 0
assert b_file.exists() and b_file.stat().st_size > 0
def test_one_owner_failure_does_not_block_others(self, mock_paths, setup_logging, monkeypatch):
"""OwnerA repo-list fetch 500s; OwnerB still processed and reported healthy."""
monkeypatch.setattr(coe_mirror, "OWNERS", ["OwnerA", "OwnerB"])
(mock_paths["pcloud_target"] / "OwnerA").mkdir(exist_ok=True)
(mock_paths["pcloud_target"] / "OwnerB").mkdir(exist_ok=True)
monkeypatch.setattr(coe_mirror, "acquire_lock", lambda: 999)
monkeypatch.setattr(coe_mirror, "release_lock", lambda fd: None)
monkeypatch.setattr(time, "sleep", lambda x: None)
# Capture notify args
captured = {}
def spy_notify(summary, failed_owners=None):
captured["summary"] = summary
captured["failed_owners"] = failed_owners
monkeypatch.setattr(coe_mirror, "notify", spy_notify)
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr("README.md", "hi")
zip_bytes = buf.getvalue()
sha_b = "b" * 40
with requests_mock.Mocker() as m:
m.get(f"{coe_mirror.API_BASE}/api/v1/users/OwnerA/repos", status_code=500)
m.get(
f"{coe_mirror.API_BASE}/api/v1/users/OwnerB/repos",
json=[{"name": "onlyone", "default_branch": "main"}],
)
m.get(
f"{coe_mirror.API_BASE}/api/v1/repos/OwnerB/onlyone/branches/main",
json={"commit": {"id": sha_b}},
)
m.get(
f"{coe_mirror.API_BASE}/OwnerB/onlyone/archive/{sha_b}.zip",
content=zip_bytes,
)
rc = coe_mirror.main()
assert rc == 0
b_file = mock_paths["pcloud_target"] / "OwnerB" / f"onlyone_{sha_b[:8]}.zip"
assert b_file.exists()
assert captured["failed_owners"] == ["OwnerA"]
# summary contains only the OwnerB entry
assert any(owner == "OwnerB" and repo == "onlyone" for owner, repo, *_ in captured["summary"])