chore: rename project coven → hack-house ⛧

Rebrand the Rust client crate (coven/ → hh/, package+binary "hack-house"),
README, CLI strings, and branch (coven → hack-house). Gitea repo renamed
cmd-chat → hack-house to match. Crypto/server logic unchanged; selftest +
golden-vector test still green, binary is now `hack-house`.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
leetcrypt
2026-05-30 13:29:14 -07:00
parent 82a04f3e12
commit bb1d662ee1
2730 changed files with 712933 additions and 46 deletions
@@ -0,0 +1,25 @@
from enum import IntEnum, auto
from sanic.compat import UpperStrEnum
class RestartOrder(UpperStrEnum):
"""Available restart orders."""
SHUTDOWN_FIRST = auto()
STARTUP_FIRST = auto()
class ProcessState(IntEnum):
"""Process states."""
NONE = auto()
IDLE = auto()
RESTARTING = auto()
STARTING = auto()
STARTED = auto()
ACKED = auto()
JOINED = auto()
TERMINATED = auto()
FAILED = auto()
COMPLETED = auto()
@@ -0,0 +1,507 @@
from __future__ import annotations
import atexit
import grp
import os
import pwd
import signal
import sys
import time
import uuid
from dataclasses import dataclass
from pathlib import Path
from sanic.compat import OS_IS_WINDOWS
from sanic.log import logger
try:
import fcntl # type: ignore
except (ImportError, ModuleNotFoundError): # pragma: no cover
fcntl = None # type: ignore
def _get_default_runtime_dir() -> Path:
"""
Default directory for auto-generated runtime artifacts (pid/lock/log).
Priority:
1) XDG_RUNTIME_DIR/sanic (preferred, runtime-only)
2) ~/.local/state/sanic (persistent state, modern default)
3) ~/.cache/sanic (fallback)
4) cwd (last resort)
"""
xdg_runtime = os.environ.get("XDG_RUNTIME_DIR")
if xdg_runtime:
path = Path(xdg_runtime) / "sanic"
try:
path.mkdir(mode=0o700, parents=True, exist_ok=True)
return path
except OSError:
pass
state_dir = Path.home() / ".local" / "state" / "sanic"
try:
state_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
return state_dir
except OSError:
pass
cache_dir = Path.home() / ".cache" / "sanic"
try:
cache_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
return cache_dir
except OSError:
pass
return Path.cwd()
def _sanitize_name(name: str) -> str:
return (
"".join(
c if c.isalnum() or c in ("-", "_", ".") else "_" for c in name
).strip("._")
or "sanic"
)
def _process_exists(pid: int) -> bool:
try:
os.kill(pid, 0)
except OSError:
return False
return True
def _is_sanic_process(pid: int) -> bool:
"""
Best-effort Sanic process identification.
On Linux, checks /proc/<pid>/cmdline for 'sanic'. On other platforms,
falls back to True if process exists (no strong identification).
"""
proc_cmdline = Path(f"/proc/{pid}/cmdline")
if proc_cmdline.exists():
try:
data = proc_cmdline.read_bytes()
return b"sanic" in data
except OSError:
return False
return True
@dataclass(frozen=True)
class PidfileInfo:
pid: int
started: int | None = None
name: str | None = None
class DaemonError(Exception):
pass
class Daemon:
"""
Daemonization helper (Unix only).
Supports:
- Double-fork daemonization
- Optional PID file (with Sanic marker + metadata)
- Optional lock file (prevents double start / stale PID reuse issues)
- Optional logfile redirection
- Optional privilege drop (user/group)
- SIGHUP handler that preserves pidfile identity
"""
def __init__(
self,
pidfile: str | None = None,
logfile: str | None = None,
user: str | None = None,
group: str | None = None,
name: str | None = None,
lockfile: str | None = None,
):
if OS_IS_WINDOWS:
raise DaemonError(
"Daemon mode is not supported on Windows. "
"Consider using a Windows service or nssm instead."
)
self.name = _sanitize_name(name) if name else None
self._auto_pidfile = pidfile == "auto" or pidfile == ""
self.pidfile: Path | None
if self._auto_pidfile:
base = _get_default_runtime_dir()
# Prefer deterministic name if provided
if self.name:
self.pidfile = base / f"{self.name}.pid"
else:
self.pidfile = base / f"sanic-{uuid.uuid4().hex}.pid"
elif pidfile:
self.pidfile = Path(pidfile)
else:
self.pidfile = None
self.logfile = Path(logfile) if logfile else None
self.user = user
self.group = group
self._uid: int | None = None
self._gid: int | None = None
self.pid: int | None = None
self._lockfile_path: Path | None = Path(lockfile) if lockfile else None
self._lock_fd: int | None = None
@staticmethod
def get_pidfile_path(name: str) -> Path:
"""
Get the predictable pidfile path for a given name.
Args:
name: The application/daemon name
Returns:
Path to the pidfile in the default runtime directory
"""
sanitized = _sanitize_name(name)
return _get_default_runtime_dir() / f"{sanitized}.pid"
def validate(self) -> None:
self._validate_user_group()
self._validate_paths()
self._validate_runtime_state()
def daemonize(self) -> None:
"""
Double-fork daemonization.
Important: anything meant for the invoking terminal must be
printed/logged before calling this method, since stdout/stderr
are redirected after detaching.
"""
self.validate()
# Acquire lock before forking to prevent race condition
# The lock is inherited by child processes and remains held
self._acquire_lockfile()
# First fork
try:
pid = os.fork()
if pid > 0:
os._exit(0) # pragma: no cover
except OSError as e:
raise DaemonError(f"First fork failed: {e}") from e
os.chdir("/")
os.setsid()
os.umask(0o022)
# Second fork
try:
pid = os.fork()
if pid > 0:
os._exit(0) # pragma: no cover
except OSError as e:
raise DaemonError(f"Second fork failed: {e}") from e
self.pid = os.getpid()
self._redirect_streams()
self._write_pidfile()
self._setup_signal_handlers()
logger.info(
"Sanic daemon started",
extra={
"daemon_pid": self.pid,
"daemon_pidfile": str(self.pidfile) if self.pidfile else None,
"daemon_logfile": str(self.logfile) if self.logfile else None,
"daemon_name": self.name,
},
)
def drop_privileges(self) -> None:
"""
Drop privileges to configured user/group.
Call after binding privileged ports but before serving requests.
"""
if self._uid is None and self._gid is None:
return
if os.getuid() != 0:
logger.warning(
"Privilege drop requested but not running as root",
extra={"user": self.user, "group": self.group},
)
return
if self._gid is not None:
try:
os.setgid(self._gid)
if self.user:
os.initgroups(self.user, self._gid)
except PermissionError as e:
grp = self.group or self._gid
raise DaemonError(
f"Cannot change group to '{grp}'. Are you root?"
) from e
if self._uid is not None:
try:
os.setuid(self._uid)
except PermissionError as e:
usr = self.user or self._uid
raise DaemonError(
f"Cannot change user to '{usr}'. Are you root?"
) from e
logger.info(
"Dropped privileges",
extra={
"user": self.user or "unchanged",
"group": self.group or "unchanged",
},
)
def _validate_user_group(self) -> None:
if not self.user and not self.group:
return
if self.user:
try:
pw = pwd.getpwnam(self.user)
except KeyError as e:
raise DaemonError(f"User '{self.user}' does not exist") from e
self._uid = pw.pw_uid
if not self.group:
self._gid = pw.pw_gid
if self.group:
try:
gr = grp.getgrnam(self.group)
except KeyError as e:
raise DaemonError(
f"Group '{self.group}' does not exist"
) from e
self._gid = gr.gr_gid
def _validate_paths(self) -> None:
if self.pidfile:
self._validate_writable_dir(self.pidfile, "PID file")
if self.logfile:
self._validate_writable_dir(self.logfile, "Log file")
# Default lockfile beside pidfile if not explicitly set
if not self._lockfile_path and self.pidfile:
self._lockfile_path = self.pidfile.with_suffix(".lock")
if self._lockfile_path:
self._validate_writable_dir(self._lockfile_path, "Lock file")
def _validate_runtime_state(self) -> None:
# PID file check (stale vs running)
if not self.pidfile or not self.pidfile.exists():
return
info = self.read_pidfile_info(self.pidfile)
if not info:
return
if _process_exists(info.pid) and _is_sanic_process(info.pid):
raise DaemonError(f"Daemon already running with PID {info.pid}")
def _validate_writable_dir(self, path: Path, label: str) -> None:
directory = path.parent
if not directory.exists():
raise DaemonError(f"{label} directory does not exist: {directory}")
if not os.access(directory, os.W_OK):
raise DaemonError(
f"Cannot write to {label} directory: {directory}"
)
def _write_pidfile(self) -> None:
if not self.pidfile:
return
pid = os.getpid()
started = int(time.time())
name = self.name or ""
# Format:
# sanic
# pid=123
# started=...
# name=...
lines = [
"sanic",
f"pid={pid}",
f"started={started}",
]
if name:
lines.append(f"name={name}")
try:
self.pidfile.write_text("\n".join(lines) + "\n")
except OSError as e:
raise DaemonError(
f"Failed to write PID file: {self.pidfile}"
) from e
atexit.register(self._remove_pidfile)
def _remove_pidfile(self) -> None:
if not self.pidfile:
return
try:
self.pidfile.unlink(missing_ok=True)
except OSError as e:
# Best-effort cleanup: failure to remove the PID file is non-fatal.
logger.debug("Failed to remove PID file %s: %s", self.pidfile, e)
@staticmethod
def read_pidfile(pidfile: str | Path) -> int | None:
info = Daemon.read_pidfile_info(pidfile)
return info.pid if info else None
@staticmethod
def read_pidfile_info(pidfile: str | Path) -> PidfileInfo | None:
path = Path(pidfile)
if not path.exists():
return None
try:
raw = path.read_text().splitlines()
except OSError:
return None
if not raw or raw[0].strip() != "sanic":
return None
data: dict[str, str] = {}
for line in raw[1:]:
if "=" in line:
k, v = line.split("=", 1)
data[k.strip()] = v.strip()
try:
pid = int(data.get("pid", ""))
except ValueError:
return None
started = None
if "started" in data:
try:
started = int(data["started"])
except ValueError:
started = None
name = data.get("name") or None
return PidfileInfo(pid=pid, started=started, name=name)
def _acquire_lockfile(self) -> None:
if not self._lockfile_path:
return
if fcntl is None:
return
try:
fd = os.open(
str(self._lockfile_path), os.O_RDWR | os.O_CREAT, 0o644
)
except OSError as e:
raise DaemonError(
f"Failed to open lock file: {self._lockfile_path}"
) from e
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as e:
try:
os.close(fd)
except OSError:
pass
raise DaemonError(
f"Daemon already running (lock held): {self._lockfile_path}"
) from e
self._lock_fd = fd
atexit.register(self._release_lockfile)
def _release_lockfile(self) -> None:
if self._lock_fd is None:
return
try:
if fcntl is not None:
fcntl.flock(self._lock_fd, fcntl.LOCK_UN)
except OSError as e:
# Best-effort cleanup: failure to unlock is non-fatal.
logger.debug(
"Failed to unlock file descriptor %s: %s", self._lock_fd, e
)
try:
os.close(self._lock_fd)
except OSError as e:
# Best-effort cleanup: failure to close is non-fatal.
logger.debug(
"Failed to close file descriptor %s: %s", self._lock_fd, e
)
self._lock_fd = None
if self._lockfile_path:
try:
self._lockfile_path.unlink(missing_ok=True)
except OSError as e:
# Best-effort cleanup: failure to remove lock file
# is non-fatal.
logger.debug(
"Failed to remove lock file %s: %s", self._lockfile_path, e
)
def _redirect_streams(self) -> None:
sys.stdout.flush()
sys.stderr.flush()
stdin_fd = os.open(os.devnull, os.O_RDONLY)
os.dup2(stdin_fd, sys.stdin.fileno())
if self.logfile:
log_fd = os.open(
str(self.logfile),
os.O_WRONLY | os.O_CREAT | os.O_APPEND,
0o644,
)
os.dup2(log_fd, sys.stdout.fileno())
os.dup2(log_fd, sys.stderr.fileno())
if log_fd > 2:
os.close(log_fd)
else:
devnull_fd = os.open(os.devnull, os.O_RDWR)
os.dup2(devnull_fd, sys.stdout.fileno())
os.dup2(devnull_fd, sys.stderr.fileno())
if devnull_fd > 2:
os.close(devnull_fd)
if stdin_fd > 2:
os.close(stdin_fd)
def _setup_signal_handlers(self) -> None:
if not self.pidfile:
return
original = signal.getsignal(signal.SIGHUP)
def _handle_sighup(signum, frame):
# Preserve pidfile identity; ensure it exists.
self._write_pidfile()
if callable(original):
original(signum, frame)
signal.signal(signal.SIGHUP, _handle_sighup)
@@ -0,0 +1,157 @@
from __future__ import annotations
from collections.abc import Mapping
from datetime import datetime
from inspect import isawaitable
from multiprocessing.connection import Connection
from os import environ
from pathlib import Path
from typing import Any
from sanic.exceptions import Unauthorized
from sanic.helpers import Default
from sanic.log import logger
from sanic.request import Request
from sanic.response import json
class Inspector:
"""Inspector for Sanic workers.
This class is used to create an inspector for Sanic workers. It is
instantiated by the worker class and is used to create a Sanic app
that can be used to inspect and control the workers and the server.
It is not intended to be used directly by the user.
See [Inspector](/en/guide/deployment/inspector) for more information.
Args:
publisher (Connection): The connection to the worker.
app_info (Dict[str, Any]): Information about the app.
worker_state (Mapping[str, Any]): The state of the worker.
host (str): The host to bind the inspector to.
port (int): The port to bind the inspector to.
api_key (str): The API key to use for authentication.
tls_key (Union[Path, str, Default]): The path to the TLS key file.
tls_cert (Union[Path, str, Default]): The path to the TLS cert file.
"""
def __init__(
self,
publisher: Connection,
app_info: dict[str, Any],
worker_state: Mapping[str, Any],
host: str,
port: int,
api_key: str,
tls_key: Path | str | Default,
tls_cert: Path | str | Default,
):
self._publisher = publisher
self.app_info = app_info
self.worker_state = worker_state
self.host = host
self.port = port
self.api_key = api_key
self.tls_key = tls_key
self.tls_cert = tls_cert
def __call__(self, run=True, **_) -> Inspector:
from sanic import Sanic
self.app = Sanic("Inspector")
self._setup()
if run:
self.app.run(
host=self.host,
port=self.port,
single_process=True,
ssl={"key": self.tls_key, "cert": self.tls_cert}
if not isinstance(self.tls_key, Default)
and not isinstance(self.tls_cert, Default)
else None,
)
return self
def _setup(self):
self.app.get("/")(self._info)
self.app.post("/<action:str>")(self._action)
if self.api_key:
self.app.on_request(self._authentication)
environ["SANIC_IGNORE_PRODUCTION_WARNING"] = "true"
def _authentication(self, request: Request) -> None:
if request.token != self.api_key:
raise Unauthorized("Bad API key")
async def _action(self, request: Request, action: str):
logger.info("Incoming inspector action: %s", action)
output: Any = None
method = getattr(self, action, None)
if method:
kwargs = {}
if request.body:
kwargs = request.json
args = kwargs.pop("args", ())
output = method(*args, **kwargs)
if isawaitable(output):
output = await output
return await self._respond(request, output)
async def _info(self, request: Request):
return await self._respond(request, self._state_to_json())
async def _respond(self, request: Request, output: Any):
name = request.match_info.get("action", "info")
return json({"meta": {"action": name}, "result": output})
def _state_to_json(self) -> dict[str, Any]:
output = {"info": self.app_info}
output["workers"] = self._make_safe(dict(self.worker_state))
return output
@staticmethod
def _make_safe(obj: dict[str, Any]) -> dict[str, Any]:
for key, value in obj.items():
if isinstance(value, dict):
obj[key] = Inspector._make_safe(value)
elif isinstance(value, datetime):
obj[key] = value.isoformat()
return obj
def reload(self, zero_downtime: bool = False) -> None:
"""Reload the workers
Args:
zero_downtime (bool, optional): Whether to use zero downtime
reload. Defaults to `False`.
"""
message = "__ALL_PROCESSES__:"
if zero_downtime:
message += ":STARTUP_FIRST"
self._publisher.send(message)
def scale(self, replicas: str | int) -> str:
"""Scale the number of workers
Args:
replicas (Union[str, int]): The number of workers to scale to.
Returns:
str: A log message.
"""
num_workers = 1
if replicas:
num_workers = int(replicas)
log_msg = f"Scaling to {num_workers}"
logger.info(log_msg)
message = f"__SCALE__:{num_workers}"
self._publisher.send(message)
return log_msg
def shutdown(self) -> None:
"""Shutdown the workers"""
message = "__TERMINATE__"
self._publisher.send(message)
@@ -0,0 +1,167 @@
from __future__ import annotations
import os
import sys
from contextlib import suppress
from importlib import import_module
from inspect import isfunction
from pathlib import Path
from ssl import SSLContext
from typing import TYPE_CHECKING, Any, Callable, cast
from sanic.http.tls.context import process_to_context
from sanic.http.tls.creators import MkcertCreator, TrustmeCreator
if TYPE_CHECKING:
from sanic import Sanic as SanicApp
DEFAULT_APP_NAME = "app"
class AppLoader:
"""A helper to load application instances.
Generally used by the worker to load the application instance.
See [Dynamic Applications](/en/guide/deployment/app-loader) for information on when you may need to use this.
Args:
module_input (str): The module to load the application from.
as_factory (bool): Whether the application is a factory.
as_simple (bool): Whether the application is a simple server.
args (Any): Arguments to pass to the application factory.
factory (Callable[[], SanicApp]): A callable that returns a Sanic application instance.
""" # noqa: E501
def __init__(
self,
module_input: str = "",
as_factory: bool = False,
as_simple: bool = False,
args: Any = None,
factory: Callable[[], SanicApp] | None = None,
) -> None:
self.module_input = module_input
self.module_name = ""
self.app_name = ""
self.as_factory = as_factory
self.as_simple = as_simple
self.args = args
self.factory = factory
self.cwd = os.getcwd()
if module_input:
delimiter = ":" if ":" in module_input else "."
if (
delimiter in module_input
and "\\" not in module_input
and "/" not in module_input
):
module_name, app_name = module_input.rsplit(delimiter, 1)
self.module_name = module_name
self.app_name = app_name
if self.app_name.endswith("()"):
self.as_factory = True
self.app_name = self.app_name[:-2]
def load(self) -> SanicApp:
module_path = os.path.abspath(self.cwd)
if module_path not in sys.path:
sys.path.append(module_path)
if self.factory:
return self.factory()
else:
from sanic.app import Sanic
from sanic.simple import create_simple_server
maybe_path = Path(self.module_input)
if self.as_simple or (
maybe_path.is_dir()
and ("\\" in self.module_input or "/" in self.module_input)
):
app = create_simple_server(maybe_path)
else:
implied_app_name = False
if not self.module_name and not self.app_name:
self.module_name = self.module_input
self.app_name = DEFAULT_APP_NAME
implied_app_name = True
module = import_module(self.module_name)
app = getattr(module, self.app_name, None)
if not app and implied_app_name:
raise ValueError(
"Looks like you only supplied a module name. Sanic "
"tried to locate an application instance named "
f"{self.module_name}:app, but was unable to locate "
"an application instance. Please provide a path "
"to a global instance of Sanic(), or a callable that "
"will return a Sanic() application instance."
)
if self.as_factory or isfunction(app):
if not callable(app):
type_name = type(app).__name__
raise ValueError(
f"Expected a callable, but got {type_name}."
)
try:
app = app(self.args)
except TypeError:
app = app()
app_type_name = type(app).__name__
if (
not isinstance(app, Sanic)
and self.args
and hasattr(self.args, "target")
):
with suppress(ModuleNotFoundError):
maybe_module = import_module(self.module_input)
app = getattr(maybe_module, "app", None)
if not app:
message = (
"Module is not a Sanic app, "
f"it is a {app_type_name}\n"
f" Perhaps you meant {self.args.target}:app?"
)
raise ValueError(message)
return app
class CertLoader:
_creators = {
"mkcert": MkcertCreator,
"trustme": TrustmeCreator,
}
def __init__(
self,
ssl_data: SSLContext | dict[str, str | os.PathLike] | None,
):
self._ssl_data = ssl_data
self._creator_class = None
if not ssl_data or not isinstance(ssl_data, dict):
return
creator_name = cast(str, ssl_data.get("creator"))
self._creator_class = self._creators.get(creator_name)
if not creator_name:
return
if not self._creator_class:
raise RuntimeError(f"Unknown certificate creator: {creator_name}")
self._key = ssl_data["key"]
self._cert = ssl_data["cert"]
self._localhost = cast(str, ssl_data["localhost"])
def load(self, app: SanicApp):
if not self._creator_class:
return process_to_context(self._ssl_data)
creator = self._creator_class(app, self._key, self._cert)
return creator.generate_cert(self._localhost)
@@ -0,0 +1,550 @@
from __future__ import annotations
import os
import signal
from collections.abc import Iterable, MutableMapping
from contextlib import suppress
from enum import IntEnum, auto
from itertools import chain, count
from multiprocessing.connection import Connection
from multiprocessing.context import BaseContext
from random import choice
from signal import SIGINT, SIGTERM, Signals
from signal import signal as signal_func
from typing import Any, Callable
from sanic.compat import OS_IS_WINDOWS
from sanic.exceptions import ServerKilled
from sanic.log import error_logger, logger
from sanic.worker.constants import RestartOrder
from sanic.worker.process import ProcessState, Worker, WorkerProcess
from sanic.worker.restarter import Restarter
SIGKILL: int
if not OS_IS_WINDOWS:
from signal import SIGKILL
else:
SIGKILL = SIGINT
class MonitorCycle(IntEnum):
BREAK = auto()
CONTINUE = auto()
class WorkerManager:
"""Manage all of the processes.
This class is used to manage all of the processes. It is instantiated
by Sanic when in multiprocess mode (which is OOTB default) and is used
to start, stop, and restart the worker processes.
You can access it to interact with it **ONLY** when on the main process.
Therefore, you should really only access it from within the
`main_process_ready` event listener.
```python
from sanic import Sanic
app = Sanic("MyApp")
@app.main_process_ready
async def ready(app: Sanic, _):
app.manager.manage("MyProcess", my_process, {"foo": "bar"})
```
See [Worker Manager](/en/guide/deployment/manager) for more information.
"""
THRESHOLD = WorkerProcess.THRESHOLD
MAIN_NAME = "Sanic-Main"
def __init__(
self,
number: int,
serve: Callable[..., Any],
server_settings: dict[str, Any],
context: BaseContext,
monitor_pubsub: tuple[Connection[Any, Any], Connection[Any, Any]],
worker_state: MutableMapping[str, Any],
):
self.num_server = number
self.context = context
self.transient: dict[str, Worker] = {}
self.durable: dict[str, Worker] = {}
self.restarter = Restarter()
self.monitor_publisher, self.monitor_subscriber = monitor_pubsub
self.worker_state = worker_state
self.worker_state[self.MAIN_NAME] = {"pid": self.pid}
self._shutting_down = False
self._serve = serve
self._server_settings = server_settings
self._server_count = count()
if number == 0:
raise RuntimeError("Cannot serve with no workers")
for _ in range(number):
self.create_server()
signal_func(SIGINT, self.shutdown_signal)
signal_func(SIGTERM, self.shutdown_signal)
def manage(
self,
name: str,
func: Callable[..., Any],
kwargs: dict[str, Any],
transient: bool = False,
restartable: bool | None = None,
tracked: bool = True,
auto_start: bool = True,
workers: int = 1,
ident: str = "",
) -> Worker:
"""Instruct Sanic to manage a custom process.
Args:
name (str): A name for the worker process
func (Callable[..., Any]): The function to call in the background process
kwargs (Dict[str, Any]): Arguments to pass to the function
transient (bool, optional): Whether to mark the process as transient. If `True`
then the Worker Manager will restart the process along
with any global restart (ex: auto-reload), defaults to `False`
restartable (Optional[bool], optional): Whether to mark the process as restartable. If
`True` then the Worker Manager will be able to restart the process
if prompted. If `transient=True`, this property will be implied
to be `True`, defaults to `None`
tracked (bool, optional): Whether to track the process after completion,
defaults to `True`
auto_start (bool, optional): Whether to start the process immediately, defaults to `True`
workers (int, optional): The number of worker processes to run. Defaults to `1`.
ident (str, optional): The identifier for the worker. If not provided, the name
passed will be used. Defaults to `""`.
Returns:
Worker: The Worker instance
""" # noqa: E501
if name in self.transient or name in self.durable:
raise ValueError(f"Worker {name} already exists")
restartable = restartable if restartable is not None else transient
if transient and not restartable:
raise ValueError(
"Cannot create a transient worker that is not restartable"
)
container = self.transient if transient else self.durable
worker = Worker(
ident or name,
name,
func,
kwargs,
self.context,
self.worker_state,
workers,
restartable,
tracked,
auto_start,
)
container[worker.name] = worker
return worker
def create_server(self) -> Worker:
"""Create a new server process.
Returns:
Worker: The Worker instance
"""
server_number = next(self._server_count)
return self.manage(
f"{WorkerProcess.SERVER_LABEL}-{server_number}",
self._serve,
self._server_settings,
transient=True,
restartable=True,
ident=f"{WorkerProcess.SERVER_IDENTIFIER}{server_number:2}",
)
def shutdown_server(self, name: str | None = None) -> None:
"""Shutdown a server process.
Args:
name (Optional[str], optional): The name of the server process to shutdown.
If `None` then a random server will be chosen. Defaults to `None`.
""" # noqa: E501
if not name:
servers = [
worker
for worker in self.transient.values()
if worker.name.startswith(WorkerProcess.SERVER_LABEL)
]
if not servers:
error_logger.error(
"Server shutdown failed because a server was not found."
)
return
worker = choice(servers) # nosec B311
else:
worker = self.transient[name]
for process in worker.processes:
process.terminate()
del self.transient[worker.name]
def run(self):
"""Run the worker manager."""
self.start()
self.monitor()
self.join()
self.terminate()
self.cleanup()
def start(self):
"""Start the worker processes."""
for worker in self.workers:
for process in worker.processes:
if not worker.auto_start:
process.set_state(ProcessState.NONE, True)
continue
process.start()
def join(self):
"""Join the worker processes."""
logger.debug("Joining processes", extra={"verbosity": 1})
joined = set()
for process in self.processes:
logger.debug(
f"Found {process.pid} - {process.state.name}",
extra={"verbosity": 1},
)
if process.state < ProcessState.JOINED:
logger.debug(f"Joining {process.pid}", extra={"verbosity": 1})
joined.add(process.pid)
process.join()
if joined:
self.join()
def terminate(self):
"""Terminate the worker processes."""
if not self._shutting_down:
for process in self.processes:
process.terminate()
def cleanup(self):
"""Cleanup the worker processes."""
for process in self.processes:
process.exit()
def restart(
self,
process_names: list[str] | None = None,
restart_order=RestartOrder.SHUTDOWN_FIRST,
**kwargs,
):
"""Restart the worker processes.
Args:
process_names (Optional[List[str]], optional): The names of the processes to restart.
If `None` then all processes will be restarted. Defaults to `None`.
restart_order (RestartOrder, optional): The order in which to restart the processes.
Defaults to `RestartOrder.SHUTDOWN_FIRST`.
""" # noqa: E501
self.restarter.restart(
transient_processes=list(self.transient_processes),
durable_processes=list(self.durable_processes),
process_names=process_names,
restart_order=restart_order,
**kwargs,
)
def scale(self, num_worker: int):
if num_worker <= 0:
raise ValueError("Cannot scale to 0 workers.")
change = num_worker - self.num_server
if change == 0:
logger.info(
f"No change needed. There are already {num_worker} workers."
)
return
logger.info(f"Scaling from {self.num_server} to {num_worker} workers")
if change > 0:
for _ in range(change):
worker = self.create_server()
for process in worker.processes:
process.start()
else:
for _ in range(abs(change)):
self.shutdown_server()
self.num_server = num_worker
def monitor(self):
"""Monitor the worker processes.
First, wait for all of the workers to acknowledge that they are ready.
Then, wait for messages from the workers. If a message is received
then it is processed and the state of the worker is updated.
Also used to restart, shutdown, and scale the workers.
Raises:
ServerKilled: Raised when a worker fails to come online.
"""
self.wait_for_ack()
while True:
try:
cycle = self._poll_monitor()
if cycle is MonitorCycle.BREAK:
break
elif cycle is MonitorCycle.CONTINUE:
continue
self._sync_states()
self._cleanup_non_tracked_workers()
except InterruptedError:
if not OS_IS_WINDOWS:
raise
break
def wait_for_ack(self): # no cov
"""Wait for all of the workers to acknowledge that they are ready."""
misses = 0
message = (
"It seems that one or more of your workers failed to come "
"online in the allowed time. Sanic is shutting down to avoid a "
f"deadlock. The current threshold is {self.THRESHOLD / 10}s. "
"If this problem persists, please check out the documentation "
"https://sanic.dev/en/guide/deployment/manager.html#worker-ack."
)
while not self._all_workers_ack():
if self.monitor_subscriber.poll(0.1):
monitor_msg = self.monitor_subscriber.recv()
if monitor_msg != "__TERMINATE_EARLY__":
self.monitor_publisher.send(monitor_msg)
continue
misses = self.THRESHOLD
message = (
"One of your worker processes terminated before startup "
"was completed. Please solve any errors experienced "
"during startup. If you do not see an exception traceback "
"in your error logs, try running Sanic in a single "
"process using --single-process or single_process=True. "
"Once you are confident that the server is able to start "
"without errors you can switch back to multiprocess mode."
)
misses += 1
if misses > self.THRESHOLD:
error_logger.error(
"Not all workers acknowledged a successful startup. "
"Shutting down.\n\n" + message
)
self.kill()
@property
def workers(self) -> list[Worker]:
"""Get all of the workers."""
return list(self.transient.values()) + list(self.durable.values())
@property
def all_workers(self) -> Iterable[tuple[str, Worker]]:
return chain(self.transient.items(), self.durable.items())
@property
def processes(self):
"""Get all of the processes."""
for worker in self.workers:
for process in worker.processes:
if not process.pid:
continue
yield process
@property
def transient_processes(self):
"""Get all of the transient processes."""
for worker in self.transient.values():
yield from worker.processes
@property
def durable_processes(self):
for worker in self.durable.values():
yield from worker.processes
def kill(self):
"""Kill all of the processes."""
for process in self.processes:
logger.info("Killing %s [%s]", process.name, process.pid)
with suppress(ProcessLookupError):
if os.name == "nt":
# Windows has no os.killpg and SIGKILL doesn't seem to work
os.kill(process.pid, signal.CTRL_C_EVENT)
else:
try:
os.killpg(os.getpgid(process.pid), SIGKILL)
except OSError:
os.kill(process.pid, SIGKILL)
raise ServerKilled
def shutdown_signal(self, signal, frame):
"""Handle the shutdown signal."""
if self._shutting_down:
logger.info("Shutdown interrupted. Killing.")
with suppress(ServerKilled):
self.kill()
return
logger.info("Received signal %s. Shutting down.", Signals(signal).name)
self.monitor_publisher.send(None)
self.shutdown()
def shutdown(self):
"""Shutdown the worker manager."""
for process in self.processes:
if process.is_alive():
process.terminate()
self._shutting_down = True
def remove_worker(self, worker: Worker) -> None:
if worker.tracked:
error_logger.error(
f"Worker {worker.name} is tracked and cannot be removed."
)
return
if worker.has_alive_processes():
error_logger.error(
f"Worker {worker.name} has alive processes and cannot be "
"removed."
)
return
self.transient.pop(worker.name, None)
self.durable.pop(worker.name, None)
for process in worker.processes:
self.worker_state.pop(process.name, None)
logger.info("Removed worker %s", worker.name)
del worker
@property
def pid(self):
"""Get the process ID of the main process."""
return os.getpid()
def _all_workers_ack(self):
acked = [
worker_state.get("state") == ProcessState.ACKED.name
for worker_state in self.worker_state.values()
if worker_state.get("server")
]
return all(acked) and len(acked) == self.num_server
def _sync_states(self):
for process in self.processes:
try:
state = self.worker_state[process.name].get("state")
except KeyError:
process.set_state(ProcessState.TERMINATED, True)
continue
# Skip state sync if process is restarting to avoid race condition
if process.state == ProcessState.RESTARTING:
continue
if not process.is_alive():
state = "FAILED" if process.exitcode else "COMPLETED"
if state and process.state.name != state:
process.set_state(ProcessState[state], True)
def _cleanup_non_tracked_workers(self) -> None:
to_remove = [
worker
for worker in self.workers
if not worker.tracked and not worker.has_alive_processes()
]
for worker in to_remove:
self.remove_worker(worker)
def _poll_monitor(self) -> MonitorCycle | None:
if self.monitor_subscriber.poll(0.1):
message = self.monitor_subscriber.recv()
logger.debug(f"Monitor message: {message}", extra={"verbosity": 2})
if not message:
return MonitorCycle.BREAK
elif message == "__TERMINATE__":
self._handle_terminate()
return MonitorCycle.BREAK
elif isinstance(message, tuple) and (
len(message) == 7 or len(message) == 8
):
self._handle_manage(*message) # type: ignore
return MonitorCycle.CONTINUE
elif not isinstance(message, str):
error_logger.error(
"Monitor received an invalid message: %s", message
)
return MonitorCycle.CONTINUE
return self._handle_message(message)
return None
def _handle_terminate(self) -> None:
self.shutdown()
def _handle_message(self, message: str) -> MonitorCycle | None:
logger.debug(
"Incoming monitor message: %s",
message,
extra={"verbosity": 1},
)
split_message = message.split(":", 2)
if message.startswith("__SCALE__"):
self.scale(int(split_message[-1]))
return MonitorCycle.CONTINUE
processes = split_message[0]
reloaded_files = split_message[1] if len(split_message) > 1 else None
process_names: list[str] | None = [
name.strip() for name in processes.split(",")
]
if process_names and "__ALL_PROCESSES__" in process_names:
process_names = None
order = (
RestartOrder.STARTUP_FIRST
if "STARTUP_FIRST" in split_message
else RestartOrder.SHUTDOWN_FIRST
)
self.restart(
process_names=process_names,
reloaded_files=reloaded_files,
restart_order=order,
)
return None
def _handle_manage(
self,
name: str,
func: Callable[..., Any],
kwargs: dict[str, Any],
transient: bool,
restartable: bool | None,
tracked: bool,
auto_start: bool,
workers: int,
) -> None:
try:
worker = self.manage(
name,
func,
kwargs,
transient=transient,
restartable=restartable,
tracked=tracked,
auto_start=auto_start,
workers=workers,
)
except Exception:
error_logger.exception("Failed to manage worker %s", name)
else:
if not auto_start:
return
for process in worker.processes:
process.start()
@@ -0,0 +1,169 @@
from __future__ import annotations
from multiprocessing.connection import Connection
from os import environ, getpid
from typing import Any, Callable
from sanic.log import Colors, logger
from sanic.worker.process import ProcessState
from sanic.worker.state import WorkerState
class WorkerMultiplexer:
"""Multiplexer for Sanic workers.
This is instantiated inside of worker porocesses only. It is used to
communicate with the monitor process.
Args:
monitor_publisher (Connection): The connection to the monitor.
worker_state (Dict[str, Any]): The state of the worker.
"""
def __init__(
self,
monitor_publisher: Connection,
worker_state: dict[str, Any],
):
self._monitor_publisher = monitor_publisher
self._state = WorkerState(worker_state, self.name)
def ack(self):
"""Acknowledge the worker is ready."""
logger.debug(
f"{Colors.BLUE}Process ack: {Colors.BOLD}{Colors.SANIC}"
f"%s {Colors.BLUE}[%s]{Colors.END}",
self.name,
self.pid,
)
self._state._state[self.name] = {
**self._state._state[self.name],
"state": ProcessState.ACKED.name,
}
def manage(
self,
ident: str,
func: Callable[..., Any],
kwargs: dict[str, Any],
transient: bool = False,
restartable: bool | None = None,
tracked: bool = False,
auto_start: bool = True,
workers: int = 1,
) -> None:
"""Manages the initiation and monitoring of a worker process.
Args:
ident (str): A unique identifier for the worker process.
func (Callable[..., Any]): The function to be executed in the worker process.
kwargs (Dict[str, Any]): A dictionary of arguments to be passed to `func`.
transient (bool, optional): Flag to mark the process as transient. If `True`,
the Worker Manager will restart the process with any global restart
(e.g., auto-reload). Defaults to `False`.
restartable (Optional[bool], optional): Flag to mark the process as restartable. If `True`,
the Worker Manager can restart the process if prompted. Defaults to `None`.
tracked (bool, optional): Flag to indicate whether the process should be tracked
after its completion. Defaults to `False`.
auto_start (bool, optional): Flag to indicate whether the process should be started
workers (int, optional): The number of worker processes to run. Defaults to 1.
This method packages the provided arguments into a bundle and sends them back to the
main process to be managed by the Worker Manager.
""" # noqa: E501
bundle = (
ident,
func,
kwargs,
transient,
restartable,
tracked,
auto_start,
workers,
)
self._monitor_publisher.send(bundle)
def set_serving(self, serving: bool) -> None:
"""Set the worker to serving.
Args:
serving (bool): Whether the worker is serving.
"""
self._state._state[self.name] = {
**self._state._state[self.name],
"serving": serving,
}
def exit(self):
"""Run cleanup at worker exit."""
try:
del self._state._state[self.name]
except ConnectionRefusedError:
logger.debug("Monitor process has already exited.")
def restart(
self,
name: str = "",
all_workers: bool = False,
zero_downtime: bool = False,
):
"""Restart the worker.
Args:
name (str): The name of the process to restart.
all_workers (bool): Whether to restart all workers.
zero_downtime (bool): Whether to restart with zero downtime.
"""
if name and all_workers:
raise ValueError(
"Ambiguous restart with both a named process and"
" all_workers=True"
)
if not name:
name = "__ALL_PROCESSES__:" if all_workers else self.name
if not name.endswith(":"):
name += ":"
if zero_downtime:
name += ":STARTUP_FIRST"
self._monitor_publisher.send(name)
reload = restart # no cov
"""Alias for restart."""
def scale(self, num_workers: int):
"""Scale the number of workers.
Args:
num_workers (int): The number of workers to scale to.
"""
message = f"__SCALE__:{num_workers}"
self._monitor_publisher.send(message)
def terminate(self, early: bool = False):
"""Terminate the worker.
Args:
early (bool): Whether to terminate early.
"""
message = "__TERMINATE_EARLY__" if early else "__TERMINATE__"
self._monitor_publisher.send(message)
@property
def pid(self) -> int:
"""The process ID of the worker."""
return getpid()
@property
def name(self) -> str:
"""The name of the worker."""
return environ.get("SANIC_WORKER_NAME", "")
@property
def state(self):
"""The state of the worker."""
return self._state
@property
def workers(self) -> dict[str, Any]:
"""The state of all workers."""
return self.state.full()
@@ -0,0 +1,292 @@
from __future__ import annotations
import os
from collections.abc import MutableMapping
from datetime import datetime, timezone
from inspect import signature
from multiprocessing.context import BaseContext
from signal import SIGINT
from threading import Thread
from time import sleep
from typing import Any
from sanic.log import Colors, logger
from sanic.worker.constants import ProcessState, RestartOrder
def get_now():
now = datetime.now(tz=timezone.utc)
return now
class WorkerProcess:
"""A worker process."""
THRESHOLD = 300 # == 30 seconds
SERVER_LABEL = "Server"
SERVER_IDENTIFIER = "Srv"
def __init__(
self,
factory,
name,
ident,
target,
kwargs,
worker_state,
restartable: bool = False,
):
self.state = ProcessState.IDLE
self.factory = factory
self.name = name
self.ident = ident
self.target = target
self.kwargs = kwargs
self.worker_state = worker_state
self.restartable = restartable
if self.name not in self.worker_state:
self.worker_state[self.name] = {
"server": self.SERVER_LABEL in self.name
}
self.spawn()
def set_state(self, state: ProcessState, force=False):
if not force and state < self.state:
raise Exception("...")
self.state = state
self.worker_state[self.name] = {
**self.worker_state[self.name],
"state": self.state.name,
}
def start(self):
os.environ["SANIC_WORKER_NAME"] = self.name
os.environ["SANIC_WORKER_IDENTIFIER"] = self.ident
logger.debug(
f"{Colors.BLUE}Starting a process: {Colors.BOLD}"
f"{Colors.SANIC}%s{Colors.END}",
self.name,
)
self.set_state(ProcessState.STARTING)
self._current_process.start()
self.set_state(ProcessState.STARTED)
if not self.worker_state[self.name].get("starts"):
self.worker_state[self.name] = {
**self.worker_state[self.name],
"pid": self.pid,
"start_at": get_now(),
"starts": 1,
}
del os.environ["SANIC_WORKER_NAME"]
del os.environ["SANIC_WORKER_IDENTIFIER"]
def join(self):
self.set_state(ProcessState.JOINED)
self._current_process.join()
def exit(self):
limit = 100
while self.is_alive() and limit > 0:
sleep(0.1)
limit -= 1
if not self.is_alive():
try:
del self.worker_state[self.name]
except (
BrokenPipeError,
ConnectionRefusedError,
ConnectionResetError,
EOFError,
):
logger.debug("Monitor process has already exited.")
except KeyError:
logger.debug("Could not find worker state to delete.")
def terminate(self):
if self.state is not ProcessState.TERMINATED:
logger.debug(
f"{Colors.BLUE}Terminating a process: "
f"{Colors.BOLD}{Colors.SANIC}"
f"%s {Colors.BLUE}[%s]{Colors.END}",
self.name,
self.pid,
)
try:
self.set_state(ProcessState.TERMINATED, force=True)
except (BrokenPipeError, ConnectionResetError, EOFError):
pass
try:
os.kill(self.pid, SIGINT)
except (KeyError, AttributeError, ProcessLookupError):
...
def restart(self, restart_order=RestartOrder.SHUTDOWN_FIRST, **kwargs):
logger.debug(
f"{Colors.BLUE}Restarting a process: {Colors.BOLD}{Colors.SANIC}"
f"%s {Colors.BLUE}[%s]{Colors.END}",
self.name,
self.pid,
)
self.set_state(ProcessState.RESTARTING, force=True)
if restart_order is RestartOrder.SHUTDOWN_FIRST:
self._terminate_now()
else:
self._old_process = self._current_process
if self._add_config():
self.kwargs.update(
{"config": {k.upper(): v for k, v in kwargs.items()}}
)
try:
self.spawn()
self.start()
except AttributeError:
raise RuntimeError("Restart failed")
if restart_order is RestartOrder.STARTUP_FIRST:
self._terminate_soon()
self.worker_state[self.name] = {
**self.worker_state[self.name],
"pid": self.pid,
"starts": self.worker_state[self.name]["starts"] + 1,
"restart_at": get_now(),
}
def is_alive(self):
try:
return self._current_process.is_alive()
except AssertionError:
return False
def spawn(self):
if self.state not in (ProcessState.IDLE, ProcessState.RESTARTING):
raise Exception("Cannot spawn a worker process until it is idle.")
self._current_process = self.factory(
name=self.name,
target=self.target,
kwargs=self.kwargs,
daemon=True,
)
@property
def pid(self):
return self._current_process.pid
@property
def exitcode(self):
return self._current_process.exitcode
def _terminate_now(self):
if not self._current_process.is_alive():
return
logger.debug(
f"{Colors.BLUE}Begin restart termination: "
f"{Colors.BOLD}{Colors.SANIC}"
f"%s {Colors.BLUE}[%s]{Colors.END}",
self.name,
self._current_process.pid,
)
self._current_process.terminate()
def _terminate_soon(self):
logger.debug(
f"{Colors.BLUE}Begin restart termination: "
f"{Colors.BOLD}{Colors.SANIC}"
f"%s {Colors.BLUE}[%s]{Colors.END}",
self.name,
self._current_process.pid,
)
termination_thread = Thread(target=self._wait_to_terminate)
termination_thread.start()
def _wait_to_terminate(self):
logger.debug(
f"{Colors.BLUE}Waiting for process to be acked: "
f"{Colors.BOLD}{Colors.SANIC}"
f"%s {Colors.BLUE}[%s]{Colors.END}",
self.name,
self._old_process.pid,
)
misses = 0
while self.state is not ProcessState.ACKED:
sleep(0.1)
misses += 1
if misses > self.THRESHOLD:
raise TimeoutError(
f"Worker {self.name} failed to come ack within "
f"{self.THRESHOLD / 10} seconds"
)
else:
logger.debug(
f"{Colors.BLUE}Process acked. Terminating: "
f"{Colors.BOLD}{Colors.SANIC}"
f"%s {Colors.BLUE}[%s]{Colors.END}",
self.name,
self._old_process.pid,
)
self._old_process.terminate()
delattr(self, "_old_process")
def _add_config(self) -> bool:
sig = signature(self.target)
if "config" in sig.parameters or any(
param.kind == param.VAR_KEYWORD
for param in sig.parameters.values()
):
return True
return False
class Worker:
WORKER_PREFIX = "Sanic"
def __init__(
self,
ident: str,
name: str,
serve,
server_settings,
context: BaseContext,
worker_state: MutableMapping[str, Any],
num: int = 1,
restartable: bool = False,
tracked: bool = True,
auto_start: bool = True,
):
self.ident = ident
self.name = name
self.num = num
self.context = context
self.serve = serve
self.server_settings = server_settings
self.worker_state = worker_state
self.processes: set[WorkerProcess] = set()
self.restartable = restartable
self.tracked = tracked
self.auto_start = auto_start
for _ in range(num):
self.create_process()
def create_process(self) -> WorkerProcess:
process = WorkerProcess(
# Need to ignore this typing error - The problem is the
# BaseContext itself has no Process. But, all of its
# implementations do. We can safely ignore as it is a typing
# issue in the standard lib.
factory=self.context.Process, # type: ignore
name="-".join(
[self.WORKER_PREFIX, self.name, str(len(self.processes))]
),
ident=self.ident,
target=self.serve,
kwargs={**self.server_settings},
worker_state=self.worker_state,
restartable=self.restartable,
)
self.processes.add(process)
return process
def has_alive_processes(self) -> bool:
return any(process.is_alive() for process in self.processes)
@@ -0,0 +1,122 @@
from __future__ import annotations
import os
import sys
from asyncio import new_event_loop
from itertools import chain
from multiprocessing.connection import Connection
from pathlib import Path
from signal import SIGINT, SIGTERM
from signal import signal as signal_func
from time import sleep
from sanic.server.events import trigger_events
from sanic.worker.loader import AppLoader
class Reloader:
INTERVAL = 1.0 # seconds
def __init__(
self,
publisher: Connection,
interval: float,
reload_dirs: set[Path],
app_loader: AppLoader,
):
self._publisher = publisher
self.interval = interval or self.INTERVAL
self.reload_dirs = reload_dirs
self.run = True
self.app_loader = app_loader
def __call__(self) -> None:
app = self.app_loader.load()
signal_func(SIGINT, self.stop)
signal_func(SIGTERM, self.stop)
mtimes: dict[str, float] = {}
reloader_start = app.listeners.get("reload_process_start")
reloader_stop = app.listeners.get("reload_process_stop")
before_trigger = app.listeners.get("before_reload_trigger")
after_trigger = app.listeners.get("after_reload_trigger")
loop = new_event_loop()
if reloader_start:
trigger_events(reloader_start, loop, app)
while self.run:
changed = set()
for filename in self.files():
try:
if self.check_file(filename, mtimes):
path = (
filename
if isinstance(filename, str)
else filename.resolve()
)
changed.add(str(path))
except OSError:
continue
if changed:
if before_trigger:
trigger_events(before_trigger, loop, app)
self.reload(",".join(changed) if changed else "unknown")
if after_trigger:
trigger_events(after_trigger, loop, app, changed=changed)
sleep(self.interval)
else:
if reloader_stop:
trigger_events(reloader_stop, loop, app)
def stop(self, *_):
self.run = False
def reload(self, reloaded_files):
message = f"__ALL_PROCESSES__:{reloaded_files}"
self._publisher.send(message)
def files(self):
return chain(
self.python_files(),
*(d.glob("**/*") for d in self.reload_dirs),
)
def python_files(self): # no cov
"""This iterates over all relevant Python files.
It goes through all
loaded files from modules, all files in folders of already loaded
modules as well as all files reachable through a package.
"""
# The list call is necessary on Python 3 in case the module
# dictionary modifies during iteration.
for module in list(sys.modules.values()):
if module is None:
continue
filename = getattr(module, "__file__", None)
if filename:
old = None
while not os.path.isfile(filename):
old = filename
filename = os.path.dirname(filename)
if filename == old:
break
else:
if filename[-4:] in (".pyc", ".pyo"):
filename = filename[:-1]
yield filename
@staticmethod
def check_file(filename, mtimes) -> bool:
need_reload = False
mtime = os.stat(filename).st_mtime
old_time = mtimes.get(filename)
if old_time is None:
mtimes[filename] = mtime
elif mtime > old_time:
mtimes[filename] = mtime
need_reload = True
return need_reload
@@ -0,0 +1,90 @@
from __future__ import annotations
from sanic.log import error_logger
from sanic.worker.constants import RestartOrder
from sanic.worker.process import ProcessState, WorkerProcess
class Restarter:
def restart(
self,
transient_processes: list[WorkerProcess],
durable_processes: list[WorkerProcess],
process_names: list[str] | None = None,
restart_order=RestartOrder.SHUTDOWN_FIRST,
**kwargs,
) -> None:
"""Restart the worker processes.
Args:
process_names (Optional[List[str]], optional): The names of the processes to restart.
If `None`, then all processes will be restarted. Defaults to `None`.
restart_order (RestartOrder, optional): The order in which to restart the processes.
Defaults to `RestartOrder.SHUTDOWN_FIRST`.
""" # noqa: E501
restarted = self._restart_transient(
transient_processes,
process_names or [],
restart_order,
**kwargs,
)
restarted |= self._restart_durable(
durable_processes,
process_names or [],
restart_order,
**kwargs,
)
if process_names and not restarted:
error_logger.error(
f"Failed to restart processes: {', '.join(process_names)}"
)
def _restart_transient(
self,
processes: list[WorkerProcess],
process_names: list[str],
restart_order: RestartOrder,
**kwargs,
) -> set[str]:
restarted: set[str] = set()
for process in processes:
if not process.restartable or (
process_names and process.name not in process_names
):
continue
self._restart_process(process, restart_order, **kwargs)
restarted.add(process.name)
return restarted
def _restart_durable(
self,
processes: list[WorkerProcess],
process_names: list[str],
restart_order: RestartOrder,
**kwargs,
) -> set[str]:
restarted: set[str] = set()
if not process_names:
return restarted
for process in processes:
if not process.restartable or process.name not in process_names:
continue
if process.state not in (
ProcessState.COMPLETED,
ProcessState.FAILED,
ProcessState.NONE,
):
error_logger.error(
f"Cannot restart process {process.name} because "
"it is not in a final state. Current state is: "
f"{process.state.name}."
)
continue
self._restart_process(process, restart_order, **kwargs)
restarted.add(process.name)
return restarted
def _restart_process(self, process, restart_order, **kwargs):
process.restart(restart_order=restart_order, **kwargs)
@@ -0,0 +1,147 @@
from __future__ import annotations
import asyncio
import os
import socket
import warnings
from functools import partial
from multiprocessing.connection import Connection
from ssl import SSLContext
from typing import Any
from sanic.application.constants import ServerStage
from sanic.application.state import ApplicationServerInfo
from sanic.http.constants import HTTP
from sanic.log import error_logger
from sanic.logging.setup import setup_logging
from sanic.models.server_types import Signal
from sanic.server.protocols.http_protocol import HttpProtocol
from sanic.server.runners import _serve_http_1, _serve_http_3
from sanic.worker.loader import AppLoader, CertLoader
from sanic.worker.multiplexer import WorkerMultiplexer
from sanic.worker.process import Worker, WorkerProcess
def worker_serve(
host,
port,
app_name: str,
monitor_publisher: Connection | None,
app_loader: AppLoader,
worker_state: dict[str, Any] | None = None,
server_info: dict[str, list[ApplicationServerInfo]] | None = None,
ssl: SSLContext | dict[str, str | os.PathLike[str]] | None = None,
sock: socket.socket | None = None,
unix: str | None = None,
reuse_port: bool = False,
loop=None,
protocol: type[asyncio.Protocol] = HttpProtocol,
backlog: int = 100,
register_sys_signals: bool = True,
run_multiple: bool = False,
run_async: bool = False,
connections=None,
signal=Signal(),
state=None,
asyncio_server_kwargs=None,
version=HTTP.VERSION_1,
config: bytes | str | dict[str, Any] | Any | None = None,
passthru: dict[str, Any] | None = None,
):
try:
from sanic import Sanic
if app_loader:
app = app_loader.load()
else:
app = Sanic.get_app(app_name)
app.refresh(passthru)
app.setup_loop()
setup_logging(
app.state.is_debug, app.config.NO_COLOR, app.config.LOG_EXTRA
)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# Hydrate server info if needed
if server_info:
for app_name, server_info_objects in server_info.items():
a = Sanic.get_app(app_name)
if not a.state.server_info:
a.state.server_info = []
for info in server_info_objects:
if not info.settings.get("app"):
info.settings["app"] = a
a.state.server_info.append(info)
if isinstance(ssl, dict) or app.certloader_class is not CertLoader:
cert_loader = app.certloader_class(ssl or {})
ssl = cert_loader.load(app)
for info in app.state.server_info:
info.settings["ssl"] = ssl
# When in a worker process, do some init
worker_name = os.environ.get("SANIC_WORKER_NAME")
if worker_name and worker_name.startswith(
"-".join([Worker.WORKER_PREFIX, WorkerProcess.SERVER_LABEL])
):
# Hydrate apps with any passed server info
if monitor_publisher is None:
raise RuntimeError(
"No restart publisher found in worker process"
)
if worker_state is None:
raise RuntimeError("No worker state found in worker process")
# Run secondary servers
apps = list(Sanic._app_registry.values())
app.before_server_start(partial(app._start_servers, apps=apps))
for a in apps:
a.multiplexer = WorkerMultiplexer(
monitor_publisher, worker_state
)
if app.debug:
loop.set_debug(app.debug)
app.asgi = False
if app.state.server_info:
primary_server_info = app.state.server_info[0]
primary_server_info.stage = ServerStage.SERVING
if config:
app.update_config(config)
if version is HTTP.VERSION_3:
return _serve_http_3(host, port, app, loop, ssl)
return _serve_http_1(
host,
port,
app,
ssl,
sock,
unix,
reuse_port,
loop,
protocol,
backlog,
register_sys_signals,
run_multiple,
run_async,
connections,
signal,
state,
asyncio_server_kwargs,
)
except Exception as e:
warnings.simplefilter("ignore", category=RuntimeWarning)
if monitor_publisher:
error_logger.exception(e)
multiplexer = WorkerMultiplexer(monitor_publisher, {})
multiplexer.terminate(True)
else:
raise e
@@ -0,0 +1,83 @@
from __future__ import annotations
from collections.abc import ItemsView, Iterator, KeysView, Mapping, ValuesView
from collections.abc import Mapping as MappingType
from typing import Any
class WorkerState(Mapping):
RESTRICTED = (
"health",
"pid",
"requests",
"restart_at",
"server",
"start_at",
"starts",
"state",
)
def __init__(self, state: dict[str, Any], current: str) -> None:
self._name = current
self._state = state
def __getitem__(self, key: str) -> Any:
return self._state[self._name][key]
def __setitem__(self, key: str, value: Any) -> None:
if key in self.RESTRICTED:
self._write_error([key])
self._state[self._name] = {
**self._state[self._name],
key: value,
}
def __delitem__(self, key: str) -> None:
if key in self.RESTRICTED:
self._write_error([key])
self._state[self._name] = {
k: v for k, v in self._state[self._name].items() if k != key
}
def __iter__(self) -> Iterator[Any]:
return iter(self._state[self._name])
def __len__(self) -> int:
return len(self._state[self._name])
def __repr__(self) -> str:
return repr(self._state[self._name])
def __eq__(self, other: object) -> bool:
return self._state[self._name] == other
def keys(self) -> KeysView[str]:
return self._state[self._name].keys()
def values(self) -> ValuesView[Any]:
return self._state[self._name].values()
def items(self) -> ItemsView[str, Any]:
return self._state[self._name].items()
def update(self, mapping: MappingType[str, Any]) -> None:
if any(k in self.RESTRICTED for k in mapping.keys()):
self._write_error(
[k for k in mapping.keys() if k in self.RESTRICTED]
)
self._state[self._name] = {
**self._state[self._name],
**mapping,
}
def pop(self) -> None:
raise NotImplementedError
def full(self) -> dict[str, Any]:
return dict(self._state)
def _write_error(self, keys: list[str]) -> None:
raise LookupError(
f"Cannot set restricted key{'s' if len(keys) > 1 else ''} on "
f"WorkerState: {', '.join(keys)}"
)