chore: rename project coven → hack-house ⛧
Rebrand the Rust client crate (coven/ → hh/, package+binary "hack-house"), README, CLI strings, and branch (coven → hack-house). Gitea repo renamed cmd-chat → hack-house to match. Crypto/server logic unchanged; selftest + golden-vector test still green, binary is now `hack-house`. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
from enum import IntEnum, auto
|
||||
|
||||
from sanic.compat import UpperStrEnum
|
||||
|
||||
|
||||
class RestartOrder(UpperStrEnum):
|
||||
"""Available restart orders."""
|
||||
|
||||
SHUTDOWN_FIRST = auto()
|
||||
STARTUP_FIRST = auto()
|
||||
|
||||
|
||||
class ProcessState(IntEnum):
|
||||
"""Process states."""
|
||||
|
||||
NONE = auto()
|
||||
IDLE = auto()
|
||||
RESTARTING = auto()
|
||||
STARTING = auto()
|
||||
STARTED = auto()
|
||||
ACKED = auto()
|
||||
JOINED = auto()
|
||||
TERMINATED = auto()
|
||||
FAILED = auto()
|
||||
COMPLETED = auto()
|
||||
@@ -0,0 +1,507 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import grp
|
||||
import os
|
||||
import pwd
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from sanic.compat import OS_IS_WINDOWS
|
||||
from sanic.log import logger
|
||||
|
||||
|
||||
try:
|
||||
import fcntl # type: ignore
|
||||
except (ImportError, ModuleNotFoundError): # pragma: no cover
|
||||
fcntl = None # type: ignore
|
||||
|
||||
|
||||
def _get_default_runtime_dir() -> Path:
|
||||
"""
|
||||
Default directory for auto-generated runtime artifacts (pid/lock/log).
|
||||
|
||||
Priority:
|
||||
1) XDG_RUNTIME_DIR/sanic (preferred, runtime-only)
|
||||
2) ~/.local/state/sanic (persistent state, modern default)
|
||||
3) ~/.cache/sanic (fallback)
|
||||
4) cwd (last resort)
|
||||
"""
|
||||
xdg_runtime = os.environ.get("XDG_RUNTIME_DIR")
|
||||
if xdg_runtime:
|
||||
path = Path(xdg_runtime) / "sanic"
|
||||
try:
|
||||
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
return path
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
state_dir = Path.home() / ".local" / "state" / "sanic"
|
||||
try:
|
||||
state_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
return state_dir
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
cache_dir = Path.home() / ".cache" / "sanic"
|
||||
try:
|
||||
cache_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
return cache_dir
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def _sanitize_name(name: str) -> str:
|
||||
return (
|
||||
"".join(
|
||||
c if c.isalnum() or c in ("-", "_", ".") else "_" for c in name
|
||||
).strip("._")
|
||||
or "sanic"
|
||||
)
|
||||
|
||||
|
||||
def _process_exists(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_sanic_process(pid: int) -> bool:
|
||||
"""
|
||||
Best-effort Sanic process identification.
|
||||
|
||||
On Linux, checks /proc/<pid>/cmdline for 'sanic'. On other platforms,
|
||||
falls back to True if process exists (no strong identification).
|
||||
"""
|
||||
proc_cmdline = Path(f"/proc/{pid}/cmdline")
|
||||
if proc_cmdline.exists():
|
||||
try:
|
||||
data = proc_cmdline.read_bytes()
|
||||
return b"sanic" in data
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PidfileInfo:
|
||||
pid: int
|
||||
started: int | None = None
|
||||
name: str | None = None
|
||||
|
||||
|
||||
class DaemonError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Daemon:
|
||||
"""
|
||||
Daemonization helper (Unix only).
|
||||
|
||||
Supports:
|
||||
- Double-fork daemonization
|
||||
- Optional PID file (with Sanic marker + metadata)
|
||||
- Optional lock file (prevents double start / stale PID reuse issues)
|
||||
- Optional logfile redirection
|
||||
- Optional privilege drop (user/group)
|
||||
- SIGHUP handler that preserves pidfile identity
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pidfile: str | None = None,
|
||||
logfile: str | None = None,
|
||||
user: str | None = None,
|
||||
group: str | None = None,
|
||||
name: str | None = None,
|
||||
lockfile: str | None = None,
|
||||
):
|
||||
if OS_IS_WINDOWS:
|
||||
raise DaemonError(
|
||||
"Daemon mode is not supported on Windows. "
|
||||
"Consider using a Windows service or nssm instead."
|
||||
)
|
||||
|
||||
self.name = _sanitize_name(name) if name else None
|
||||
|
||||
self._auto_pidfile = pidfile == "auto" or pidfile == ""
|
||||
self.pidfile: Path | None
|
||||
if self._auto_pidfile:
|
||||
base = _get_default_runtime_dir()
|
||||
# Prefer deterministic name if provided
|
||||
if self.name:
|
||||
self.pidfile = base / f"{self.name}.pid"
|
||||
else:
|
||||
self.pidfile = base / f"sanic-{uuid.uuid4().hex}.pid"
|
||||
elif pidfile:
|
||||
self.pidfile = Path(pidfile)
|
||||
else:
|
||||
self.pidfile = None
|
||||
|
||||
self.logfile = Path(logfile) if logfile else None
|
||||
self.user = user
|
||||
self.group = group
|
||||
|
||||
self._uid: int | None = None
|
||||
self._gid: int | None = None
|
||||
self.pid: int | None = None
|
||||
|
||||
self._lockfile_path: Path | None = Path(lockfile) if lockfile else None
|
||||
self._lock_fd: int | None = None
|
||||
|
||||
@staticmethod
|
||||
def get_pidfile_path(name: str) -> Path:
|
||||
"""
|
||||
Get the predictable pidfile path for a given name.
|
||||
|
||||
Args:
|
||||
name: The application/daemon name
|
||||
|
||||
Returns:
|
||||
Path to the pidfile in the default runtime directory
|
||||
"""
|
||||
sanitized = _sanitize_name(name)
|
||||
return _get_default_runtime_dir() / f"{sanitized}.pid"
|
||||
|
||||
def validate(self) -> None:
|
||||
self._validate_user_group()
|
||||
self._validate_paths()
|
||||
self._validate_runtime_state()
|
||||
|
||||
def daemonize(self) -> None:
|
||||
"""
|
||||
Double-fork daemonization.
|
||||
|
||||
Important: anything meant for the invoking terminal must be
|
||||
printed/logged before calling this method, since stdout/stderr
|
||||
are redirected after detaching.
|
||||
"""
|
||||
self.validate()
|
||||
|
||||
# Acquire lock before forking to prevent race condition
|
||||
# The lock is inherited by child processes and remains held
|
||||
self._acquire_lockfile()
|
||||
|
||||
# First fork
|
||||
try:
|
||||
pid = os.fork()
|
||||
if pid > 0:
|
||||
os._exit(0) # pragma: no cover
|
||||
except OSError as e:
|
||||
raise DaemonError(f"First fork failed: {e}") from e
|
||||
|
||||
os.chdir("/")
|
||||
os.setsid()
|
||||
os.umask(0o022)
|
||||
|
||||
# Second fork
|
||||
try:
|
||||
pid = os.fork()
|
||||
if pid > 0:
|
||||
os._exit(0) # pragma: no cover
|
||||
except OSError as e:
|
||||
raise DaemonError(f"Second fork failed: {e}") from e
|
||||
|
||||
self.pid = os.getpid()
|
||||
|
||||
self._redirect_streams()
|
||||
self._write_pidfile()
|
||||
self._setup_signal_handlers()
|
||||
|
||||
logger.info(
|
||||
"Sanic daemon started",
|
||||
extra={
|
||||
"daemon_pid": self.pid,
|
||||
"daemon_pidfile": str(self.pidfile) if self.pidfile else None,
|
||||
"daemon_logfile": str(self.logfile) if self.logfile else None,
|
||||
"daemon_name": self.name,
|
||||
},
|
||||
)
|
||||
|
||||
def drop_privileges(self) -> None:
|
||||
"""
|
||||
Drop privileges to configured user/group.
|
||||
|
||||
Call after binding privileged ports but before serving requests.
|
||||
"""
|
||||
if self._uid is None and self._gid is None:
|
||||
return
|
||||
|
||||
if os.getuid() != 0:
|
||||
logger.warning(
|
||||
"Privilege drop requested but not running as root",
|
||||
extra={"user": self.user, "group": self.group},
|
||||
)
|
||||
return
|
||||
|
||||
if self._gid is not None:
|
||||
try:
|
||||
os.setgid(self._gid)
|
||||
if self.user:
|
||||
os.initgroups(self.user, self._gid)
|
||||
except PermissionError as e:
|
||||
grp = self.group or self._gid
|
||||
raise DaemonError(
|
||||
f"Cannot change group to '{grp}'. Are you root?"
|
||||
) from e
|
||||
|
||||
if self._uid is not None:
|
||||
try:
|
||||
os.setuid(self._uid)
|
||||
except PermissionError as e:
|
||||
usr = self.user or self._uid
|
||||
raise DaemonError(
|
||||
f"Cannot change user to '{usr}'. Are you root?"
|
||||
) from e
|
||||
|
||||
logger.info(
|
||||
"Dropped privileges",
|
||||
extra={
|
||||
"user": self.user or "unchanged",
|
||||
"group": self.group or "unchanged",
|
||||
},
|
||||
)
|
||||
|
||||
def _validate_user_group(self) -> None:
|
||||
if not self.user and not self.group:
|
||||
return
|
||||
|
||||
if self.user:
|
||||
try:
|
||||
pw = pwd.getpwnam(self.user)
|
||||
except KeyError as e:
|
||||
raise DaemonError(f"User '{self.user}' does not exist") from e
|
||||
self._uid = pw.pw_uid
|
||||
if not self.group:
|
||||
self._gid = pw.pw_gid
|
||||
|
||||
if self.group:
|
||||
try:
|
||||
gr = grp.getgrnam(self.group)
|
||||
except KeyError as e:
|
||||
raise DaemonError(
|
||||
f"Group '{self.group}' does not exist"
|
||||
) from e
|
||||
self._gid = gr.gr_gid
|
||||
|
||||
def _validate_paths(self) -> None:
|
||||
if self.pidfile:
|
||||
self._validate_writable_dir(self.pidfile, "PID file")
|
||||
|
||||
if self.logfile:
|
||||
self._validate_writable_dir(self.logfile, "Log file")
|
||||
|
||||
# Default lockfile beside pidfile if not explicitly set
|
||||
if not self._lockfile_path and self.pidfile:
|
||||
self._lockfile_path = self.pidfile.with_suffix(".lock")
|
||||
|
||||
if self._lockfile_path:
|
||||
self._validate_writable_dir(self._lockfile_path, "Lock file")
|
||||
|
||||
def _validate_runtime_state(self) -> None:
|
||||
# PID file check (stale vs running)
|
||||
if not self.pidfile or not self.pidfile.exists():
|
||||
return
|
||||
|
||||
info = self.read_pidfile_info(self.pidfile)
|
||||
if not info:
|
||||
return
|
||||
|
||||
if _process_exists(info.pid) and _is_sanic_process(info.pid):
|
||||
raise DaemonError(f"Daemon already running with PID {info.pid}")
|
||||
|
||||
def _validate_writable_dir(self, path: Path, label: str) -> None:
|
||||
directory = path.parent
|
||||
if not directory.exists():
|
||||
raise DaemonError(f"{label} directory does not exist: {directory}")
|
||||
if not os.access(directory, os.W_OK):
|
||||
raise DaemonError(
|
||||
f"Cannot write to {label} directory: {directory}"
|
||||
)
|
||||
|
||||
def _write_pidfile(self) -> None:
|
||||
if not self.pidfile:
|
||||
return
|
||||
|
||||
pid = os.getpid()
|
||||
started = int(time.time())
|
||||
name = self.name or ""
|
||||
|
||||
# Format:
|
||||
# sanic
|
||||
# pid=123
|
||||
# started=...
|
||||
# name=...
|
||||
lines = [
|
||||
"sanic",
|
||||
f"pid={pid}",
|
||||
f"started={started}",
|
||||
]
|
||||
if name:
|
||||
lines.append(f"name={name}")
|
||||
|
||||
try:
|
||||
self.pidfile.write_text("\n".join(lines) + "\n")
|
||||
except OSError as e:
|
||||
raise DaemonError(
|
||||
f"Failed to write PID file: {self.pidfile}"
|
||||
) from e
|
||||
|
||||
atexit.register(self._remove_pidfile)
|
||||
|
||||
def _remove_pidfile(self) -> None:
|
||||
if not self.pidfile:
|
||||
return
|
||||
try:
|
||||
self.pidfile.unlink(missing_ok=True)
|
||||
except OSError as e:
|
||||
# Best-effort cleanup: failure to remove the PID file is non-fatal.
|
||||
logger.debug("Failed to remove PID file %s: %s", self.pidfile, e)
|
||||
|
||||
@staticmethod
|
||||
def read_pidfile(pidfile: str | Path) -> int | None:
|
||||
info = Daemon.read_pidfile_info(pidfile)
|
||||
return info.pid if info else None
|
||||
|
||||
@staticmethod
|
||||
def read_pidfile_info(pidfile: str | Path) -> PidfileInfo | None:
|
||||
path = Path(pidfile)
|
||||
if not path.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
raw = path.read_text().splitlines()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
if not raw or raw[0].strip() != "sanic":
|
||||
return None
|
||||
|
||||
data: dict[str, str] = {}
|
||||
for line in raw[1:]:
|
||||
if "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
data[k.strip()] = v.strip()
|
||||
|
||||
try:
|
||||
pid = int(data.get("pid", ""))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
started = None
|
||||
if "started" in data:
|
||||
try:
|
||||
started = int(data["started"])
|
||||
except ValueError:
|
||||
started = None
|
||||
|
||||
name = data.get("name") or None
|
||||
return PidfileInfo(pid=pid, started=started, name=name)
|
||||
|
||||
def _acquire_lockfile(self) -> None:
|
||||
if not self._lockfile_path:
|
||||
return
|
||||
if fcntl is None:
|
||||
return
|
||||
|
||||
try:
|
||||
fd = os.open(
|
||||
str(self._lockfile_path), os.O_RDWR | os.O_CREAT, 0o644
|
||||
)
|
||||
except OSError as e:
|
||||
raise DaemonError(
|
||||
f"Failed to open lock file: {self._lockfile_path}"
|
||||
) from e
|
||||
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as e:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
raise DaemonError(
|
||||
f"Daemon already running (lock held): {self._lockfile_path}"
|
||||
) from e
|
||||
|
||||
self._lock_fd = fd
|
||||
atexit.register(self._release_lockfile)
|
||||
|
||||
def _release_lockfile(self) -> None:
|
||||
if self._lock_fd is None:
|
||||
return
|
||||
try:
|
||||
if fcntl is not None:
|
||||
fcntl.flock(self._lock_fd, fcntl.LOCK_UN)
|
||||
except OSError as e:
|
||||
# Best-effort cleanup: failure to unlock is non-fatal.
|
||||
logger.debug(
|
||||
"Failed to unlock file descriptor %s: %s", self._lock_fd, e
|
||||
)
|
||||
try:
|
||||
os.close(self._lock_fd)
|
||||
except OSError as e:
|
||||
# Best-effort cleanup: failure to close is non-fatal.
|
||||
logger.debug(
|
||||
"Failed to close file descriptor %s: %s", self._lock_fd, e
|
||||
)
|
||||
self._lock_fd = None
|
||||
if self._lockfile_path:
|
||||
try:
|
||||
self._lockfile_path.unlink(missing_ok=True)
|
||||
except OSError as e:
|
||||
# Best-effort cleanup: failure to remove lock file
|
||||
# is non-fatal.
|
||||
logger.debug(
|
||||
"Failed to remove lock file %s: %s", self._lockfile_path, e
|
||||
)
|
||||
|
||||
def _redirect_streams(self) -> None:
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
|
||||
stdin_fd = os.open(os.devnull, os.O_RDONLY)
|
||||
os.dup2(stdin_fd, sys.stdin.fileno())
|
||||
|
||||
if self.logfile:
|
||||
log_fd = os.open(
|
||||
str(self.logfile),
|
||||
os.O_WRONLY | os.O_CREAT | os.O_APPEND,
|
||||
0o644,
|
||||
)
|
||||
os.dup2(log_fd, sys.stdout.fileno())
|
||||
os.dup2(log_fd, sys.stderr.fileno())
|
||||
if log_fd > 2:
|
||||
os.close(log_fd)
|
||||
else:
|
||||
devnull_fd = os.open(os.devnull, os.O_RDWR)
|
||||
os.dup2(devnull_fd, sys.stdout.fileno())
|
||||
os.dup2(devnull_fd, sys.stderr.fileno())
|
||||
if devnull_fd > 2:
|
||||
os.close(devnull_fd)
|
||||
|
||||
if stdin_fd > 2:
|
||||
os.close(stdin_fd)
|
||||
|
||||
def _setup_signal_handlers(self) -> None:
|
||||
if not self.pidfile:
|
||||
return
|
||||
|
||||
original = signal.getsignal(signal.SIGHUP)
|
||||
|
||||
def _handle_sighup(signum, frame):
|
||||
# Preserve pidfile identity; ensure it exists.
|
||||
self._write_pidfile()
|
||||
if callable(original):
|
||||
original(signum, frame)
|
||||
|
||||
signal.signal(signal.SIGHUP, _handle_sighup)
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from inspect import isawaitable
|
||||
from multiprocessing.connection import Connection
|
||||
from os import environ
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sanic.exceptions import Unauthorized
|
||||
from sanic.helpers import Default
|
||||
from sanic.log import logger
|
||||
from sanic.request import Request
|
||||
from sanic.response import json
|
||||
|
||||
|
||||
class Inspector:
|
||||
"""Inspector for Sanic workers.
|
||||
|
||||
This class is used to create an inspector for Sanic workers. It is
|
||||
instantiated by the worker class and is used to create a Sanic app
|
||||
that can be used to inspect and control the workers and the server.
|
||||
|
||||
It is not intended to be used directly by the user.
|
||||
|
||||
See [Inspector](/en/guide/deployment/inspector) for more information.
|
||||
|
||||
Args:
|
||||
publisher (Connection): The connection to the worker.
|
||||
app_info (Dict[str, Any]): Information about the app.
|
||||
worker_state (Mapping[str, Any]): The state of the worker.
|
||||
host (str): The host to bind the inspector to.
|
||||
port (int): The port to bind the inspector to.
|
||||
api_key (str): The API key to use for authentication.
|
||||
tls_key (Union[Path, str, Default]): The path to the TLS key file.
|
||||
tls_cert (Union[Path, str, Default]): The path to the TLS cert file.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
publisher: Connection,
|
||||
app_info: dict[str, Any],
|
||||
worker_state: Mapping[str, Any],
|
||||
host: str,
|
||||
port: int,
|
||||
api_key: str,
|
||||
tls_key: Path | str | Default,
|
||||
tls_cert: Path | str | Default,
|
||||
):
|
||||
self._publisher = publisher
|
||||
self.app_info = app_info
|
||||
self.worker_state = worker_state
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.api_key = api_key
|
||||
self.tls_key = tls_key
|
||||
self.tls_cert = tls_cert
|
||||
|
||||
def __call__(self, run=True, **_) -> Inspector:
|
||||
from sanic import Sanic
|
||||
|
||||
self.app = Sanic("Inspector")
|
||||
self._setup()
|
||||
if run:
|
||||
self.app.run(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
single_process=True,
|
||||
ssl={"key": self.tls_key, "cert": self.tls_cert}
|
||||
if not isinstance(self.tls_key, Default)
|
||||
and not isinstance(self.tls_cert, Default)
|
||||
else None,
|
||||
)
|
||||
return self
|
||||
|
||||
def _setup(self):
|
||||
self.app.get("/")(self._info)
|
||||
self.app.post("/<action:str>")(self._action)
|
||||
if self.api_key:
|
||||
self.app.on_request(self._authentication)
|
||||
environ["SANIC_IGNORE_PRODUCTION_WARNING"] = "true"
|
||||
|
||||
def _authentication(self, request: Request) -> None:
|
||||
if request.token != self.api_key:
|
||||
raise Unauthorized("Bad API key")
|
||||
|
||||
async def _action(self, request: Request, action: str):
|
||||
logger.info("Incoming inspector action: %s", action)
|
||||
output: Any = None
|
||||
method = getattr(self, action, None)
|
||||
if method:
|
||||
kwargs = {}
|
||||
if request.body:
|
||||
kwargs = request.json
|
||||
args = kwargs.pop("args", ())
|
||||
output = method(*args, **kwargs)
|
||||
if isawaitable(output):
|
||||
output = await output
|
||||
|
||||
return await self._respond(request, output)
|
||||
|
||||
async def _info(self, request: Request):
|
||||
return await self._respond(request, self._state_to_json())
|
||||
|
||||
async def _respond(self, request: Request, output: Any):
|
||||
name = request.match_info.get("action", "info")
|
||||
return json({"meta": {"action": name}, "result": output})
|
||||
|
||||
def _state_to_json(self) -> dict[str, Any]:
|
||||
output = {"info": self.app_info}
|
||||
output["workers"] = self._make_safe(dict(self.worker_state))
|
||||
return output
|
||||
|
||||
@staticmethod
|
||||
def _make_safe(obj: dict[str, Any]) -> dict[str, Any]:
|
||||
for key, value in obj.items():
|
||||
if isinstance(value, dict):
|
||||
obj[key] = Inspector._make_safe(value)
|
||||
elif isinstance(value, datetime):
|
||||
obj[key] = value.isoformat()
|
||||
return obj
|
||||
|
||||
def reload(self, zero_downtime: bool = False) -> None:
|
||||
"""Reload the workers
|
||||
|
||||
Args:
|
||||
zero_downtime (bool, optional): Whether to use zero downtime
|
||||
reload. Defaults to `False`.
|
||||
"""
|
||||
message = "__ALL_PROCESSES__:"
|
||||
if zero_downtime:
|
||||
message += ":STARTUP_FIRST"
|
||||
self._publisher.send(message)
|
||||
|
||||
def scale(self, replicas: str | int) -> str:
|
||||
"""Scale the number of workers
|
||||
|
||||
Args:
|
||||
replicas (Union[str, int]): The number of workers to scale to.
|
||||
|
||||
Returns:
|
||||
str: A log message.
|
||||
"""
|
||||
num_workers = 1
|
||||
if replicas:
|
||||
num_workers = int(replicas)
|
||||
log_msg = f"Scaling to {num_workers}"
|
||||
logger.info(log_msg)
|
||||
message = f"__SCALE__:{num_workers}"
|
||||
self._publisher.send(message)
|
||||
return log_msg
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the workers"""
|
||||
message = "__TERMINATE__"
|
||||
self._publisher.send(message)
|
||||
@@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from contextlib import suppress
|
||||
from importlib import import_module
|
||||
from inspect import isfunction
|
||||
from pathlib import Path
|
||||
from ssl import SSLContext
|
||||
from typing import TYPE_CHECKING, Any, Callable, cast
|
||||
|
||||
from sanic.http.tls.context import process_to_context
|
||||
from sanic.http.tls.creators import MkcertCreator, TrustmeCreator
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sanic import Sanic as SanicApp
|
||||
|
||||
DEFAULT_APP_NAME = "app"
|
||||
|
||||
|
||||
class AppLoader:
|
||||
"""A helper to load application instances.
|
||||
|
||||
Generally used by the worker to load the application instance.
|
||||
|
||||
See [Dynamic Applications](/en/guide/deployment/app-loader) for information on when you may need to use this.
|
||||
|
||||
Args:
|
||||
module_input (str): The module to load the application from.
|
||||
as_factory (bool): Whether the application is a factory.
|
||||
as_simple (bool): Whether the application is a simple server.
|
||||
args (Any): Arguments to pass to the application factory.
|
||||
factory (Callable[[], SanicApp]): A callable that returns a Sanic application instance.
|
||||
""" # noqa: E501
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
module_input: str = "",
|
||||
as_factory: bool = False,
|
||||
as_simple: bool = False,
|
||||
args: Any = None,
|
||||
factory: Callable[[], SanicApp] | None = None,
|
||||
) -> None:
|
||||
self.module_input = module_input
|
||||
self.module_name = ""
|
||||
self.app_name = ""
|
||||
self.as_factory = as_factory
|
||||
self.as_simple = as_simple
|
||||
self.args = args
|
||||
self.factory = factory
|
||||
self.cwd = os.getcwd()
|
||||
|
||||
if module_input:
|
||||
delimiter = ":" if ":" in module_input else "."
|
||||
if (
|
||||
delimiter in module_input
|
||||
and "\\" not in module_input
|
||||
and "/" not in module_input
|
||||
):
|
||||
module_name, app_name = module_input.rsplit(delimiter, 1)
|
||||
self.module_name = module_name
|
||||
self.app_name = app_name
|
||||
if self.app_name.endswith("()"):
|
||||
self.as_factory = True
|
||||
self.app_name = self.app_name[:-2]
|
||||
|
||||
def load(self) -> SanicApp:
|
||||
module_path = os.path.abspath(self.cwd)
|
||||
if module_path not in sys.path:
|
||||
sys.path.append(module_path)
|
||||
|
||||
if self.factory:
|
||||
return self.factory()
|
||||
else:
|
||||
from sanic.app import Sanic
|
||||
from sanic.simple import create_simple_server
|
||||
|
||||
maybe_path = Path(self.module_input)
|
||||
if self.as_simple or (
|
||||
maybe_path.is_dir()
|
||||
and ("\\" in self.module_input or "/" in self.module_input)
|
||||
):
|
||||
app = create_simple_server(maybe_path)
|
||||
else:
|
||||
implied_app_name = False
|
||||
if not self.module_name and not self.app_name:
|
||||
self.module_name = self.module_input
|
||||
self.app_name = DEFAULT_APP_NAME
|
||||
implied_app_name = True
|
||||
module = import_module(self.module_name)
|
||||
app = getattr(module, self.app_name, None)
|
||||
if not app and implied_app_name:
|
||||
raise ValueError(
|
||||
"Looks like you only supplied a module name. Sanic "
|
||||
"tried to locate an application instance named "
|
||||
f"{self.module_name}:app, but was unable to locate "
|
||||
"an application instance. Please provide a path "
|
||||
"to a global instance of Sanic(), or a callable that "
|
||||
"will return a Sanic() application instance."
|
||||
)
|
||||
if self.as_factory or isfunction(app):
|
||||
if not callable(app):
|
||||
type_name = type(app).__name__
|
||||
raise ValueError(
|
||||
f"Expected a callable, but got {type_name}."
|
||||
)
|
||||
try:
|
||||
app = app(self.args)
|
||||
except TypeError:
|
||||
app = app()
|
||||
|
||||
app_type_name = type(app).__name__
|
||||
|
||||
if (
|
||||
not isinstance(app, Sanic)
|
||||
and self.args
|
||||
and hasattr(self.args, "target")
|
||||
):
|
||||
with suppress(ModuleNotFoundError):
|
||||
maybe_module = import_module(self.module_input)
|
||||
app = getattr(maybe_module, "app", None)
|
||||
if not app:
|
||||
message = (
|
||||
"Module is not a Sanic app, "
|
||||
f"it is a {app_type_name}\n"
|
||||
f" Perhaps you meant {self.args.target}:app?"
|
||||
)
|
||||
raise ValueError(message)
|
||||
return app
|
||||
|
||||
|
||||
class CertLoader:
|
||||
_creators = {
|
||||
"mkcert": MkcertCreator,
|
||||
"trustme": TrustmeCreator,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ssl_data: SSLContext | dict[str, str | os.PathLike] | None,
|
||||
):
|
||||
self._ssl_data = ssl_data
|
||||
self._creator_class = None
|
||||
if not ssl_data or not isinstance(ssl_data, dict):
|
||||
return
|
||||
|
||||
creator_name = cast(str, ssl_data.get("creator"))
|
||||
|
||||
self._creator_class = self._creators.get(creator_name)
|
||||
if not creator_name:
|
||||
return
|
||||
|
||||
if not self._creator_class:
|
||||
raise RuntimeError(f"Unknown certificate creator: {creator_name}")
|
||||
|
||||
self._key = ssl_data["key"]
|
||||
self._cert = ssl_data["cert"]
|
||||
self._localhost = cast(str, ssl_data["localhost"])
|
||||
|
||||
def load(self, app: SanicApp):
|
||||
if not self._creator_class:
|
||||
return process_to_context(self._ssl_data)
|
||||
|
||||
creator = self._creator_class(app, self._key, self._cert)
|
||||
return creator.generate_cert(self._localhost)
|
||||
@@ -0,0 +1,550 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
|
||||
from collections.abc import Iterable, MutableMapping
|
||||
from contextlib import suppress
|
||||
from enum import IntEnum, auto
|
||||
from itertools import chain, count
|
||||
from multiprocessing.connection import Connection
|
||||
from multiprocessing.context import BaseContext
|
||||
from random import choice
|
||||
from signal import SIGINT, SIGTERM, Signals
|
||||
from signal import signal as signal_func
|
||||
from typing import Any, Callable
|
||||
|
||||
from sanic.compat import OS_IS_WINDOWS
|
||||
from sanic.exceptions import ServerKilled
|
||||
from sanic.log import error_logger, logger
|
||||
from sanic.worker.constants import RestartOrder
|
||||
from sanic.worker.process import ProcessState, Worker, WorkerProcess
|
||||
from sanic.worker.restarter import Restarter
|
||||
|
||||
|
||||
SIGKILL: int
|
||||
if not OS_IS_WINDOWS:
|
||||
from signal import SIGKILL
|
||||
else:
|
||||
SIGKILL = SIGINT
|
||||
|
||||
|
||||
class MonitorCycle(IntEnum):
|
||||
BREAK = auto()
|
||||
CONTINUE = auto()
|
||||
|
||||
|
||||
class WorkerManager:
|
||||
"""Manage all of the processes.
|
||||
|
||||
This class is used to manage all of the processes. It is instantiated
|
||||
by Sanic when in multiprocess mode (which is OOTB default) and is used
|
||||
to start, stop, and restart the worker processes.
|
||||
|
||||
You can access it to interact with it **ONLY** when on the main process.
|
||||
|
||||
Therefore, you should really only access it from within the
|
||||
`main_process_ready` event listener.
|
||||
|
||||
```python
|
||||
from sanic import Sanic
|
||||
|
||||
app = Sanic("MyApp")
|
||||
|
||||
@app.main_process_ready
|
||||
async def ready(app: Sanic, _):
|
||||
app.manager.manage("MyProcess", my_process, {"foo": "bar"})
|
||||
```
|
||||
|
||||
See [Worker Manager](/en/guide/deployment/manager) for more information.
|
||||
"""
|
||||
|
||||
THRESHOLD = WorkerProcess.THRESHOLD
|
||||
MAIN_NAME = "Sanic-Main"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
number: int,
|
||||
serve: Callable[..., Any],
|
||||
server_settings: dict[str, Any],
|
||||
context: BaseContext,
|
||||
monitor_pubsub: tuple[Connection[Any, Any], Connection[Any, Any]],
|
||||
worker_state: MutableMapping[str, Any],
|
||||
):
|
||||
self.num_server = number
|
||||
self.context = context
|
||||
self.transient: dict[str, Worker] = {}
|
||||
self.durable: dict[str, Worker] = {}
|
||||
self.restarter = Restarter()
|
||||
self.monitor_publisher, self.monitor_subscriber = monitor_pubsub
|
||||
self.worker_state = worker_state
|
||||
self.worker_state[self.MAIN_NAME] = {"pid": self.pid}
|
||||
self._shutting_down = False
|
||||
self._serve = serve
|
||||
self._server_settings = server_settings
|
||||
self._server_count = count()
|
||||
|
||||
if number == 0:
|
||||
raise RuntimeError("Cannot serve with no workers")
|
||||
|
||||
for _ in range(number):
|
||||
self.create_server()
|
||||
|
||||
signal_func(SIGINT, self.shutdown_signal)
|
||||
signal_func(SIGTERM, self.shutdown_signal)
|
||||
|
||||
def manage(
|
||||
self,
|
||||
name: str,
|
||||
func: Callable[..., Any],
|
||||
kwargs: dict[str, Any],
|
||||
transient: bool = False,
|
||||
restartable: bool | None = None,
|
||||
tracked: bool = True,
|
||||
auto_start: bool = True,
|
||||
workers: int = 1,
|
||||
ident: str = "",
|
||||
) -> Worker:
|
||||
"""Instruct Sanic to manage a custom process.
|
||||
|
||||
Args:
|
||||
name (str): A name for the worker process
|
||||
func (Callable[..., Any]): The function to call in the background process
|
||||
kwargs (Dict[str, Any]): Arguments to pass to the function
|
||||
transient (bool, optional): Whether to mark the process as transient. If `True`
|
||||
then the Worker Manager will restart the process along
|
||||
with any global restart (ex: auto-reload), defaults to `False`
|
||||
restartable (Optional[bool], optional): Whether to mark the process as restartable. If
|
||||
`True` then the Worker Manager will be able to restart the process
|
||||
if prompted. If `transient=True`, this property will be implied
|
||||
to be `True`, defaults to `None`
|
||||
tracked (bool, optional): Whether to track the process after completion,
|
||||
defaults to `True`
|
||||
auto_start (bool, optional): Whether to start the process immediately, defaults to `True`
|
||||
workers (int, optional): The number of worker processes to run. Defaults to `1`.
|
||||
ident (str, optional): The identifier for the worker. If not provided, the name
|
||||
passed will be used. Defaults to `""`.
|
||||
|
||||
Returns:
|
||||
Worker: The Worker instance
|
||||
""" # noqa: E501
|
||||
if name in self.transient or name in self.durable:
|
||||
raise ValueError(f"Worker {name} already exists")
|
||||
restartable = restartable if restartable is not None else transient
|
||||
if transient and not restartable:
|
||||
raise ValueError(
|
||||
"Cannot create a transient worker that is not restartable"
|
||||
)
|
||||
container = self.transient if transient else self.durable
|
||||
worker = Worker(
|
||||
ident or name,
|
||||
name,
|
||||
func,
|
||||
kwargs,
|
||||
self.context,
|
||||
self.worker_state,
|
||||
workers,
|
||||
restartable,
|
||||
tracked,
|
||||
auto_start,
|
||||
)
|
||||
container[worker.name] = worker
|
||||
return worker
|
||||
|
||||
def create_server(self) -> Worker:
|
||||
"""Create a new server process.
|
||||
|
||||
Returns:
|
||||
Worker: The Worker instance
|
||||
"""
|
||||
server_number = next(self._server_count)
|
||||
return self.manage(
|
||||
f"{WorkerProcess.SERVER_LABEL}-{server_number}",
|
||||
self._serve,
|
||||
self._server_settings,
|
||||
transient=True,
|
||||
restartable=True,
|
||||
ident=f"{WorkerProcess.SERVER_IDENTIFIER}{server_number:2}",
|
||||
)
|
||||
|
||||
def shutdown_server(self, name: str | None = None) -> None:
|
||||
"""Shutdown a server process.
|
||||
|
||||
Args:
|
||||
name (Optional[str], optional): The name of the server process to shutdown.
|
||||
If `None` then a random server will be chosen. Defaults to `None`.
|
||||
""" # noqa: E501
|
||||
if not name:
|
||||
servers = [
|
||||
worker
|
||||
for worker in self.transient.values()
|
||||
if worker.name.startswith(WorkerProcess.SERVER_LABEL)
|
||||
]
|
||||
if not servers:
|
||||
error_logger.error(
|
||||
"Server shutdown failed because a server was not found."
|
||||
)
|
||||
return
|
||||
worker = choice(servers) # nosec B311
|
||||
else:
|
||||
worker = self.transient[name]
|
||||
|
||||
for process in worker.processes:
|
||||
process.terminate()
|
||||
|
||||
del self.transient[worker.name]
|
||||
|
||||
def run(self):
|
||||
"""Run the worker manager."""
|
||||
self.start()
|
||||
self.monitor()
|
||||
self.join()
|
||||
self.terminate()
|
||||
self.cleanup()
|
||||
|
||||
def start(self):
|
||||
"""Start the worker processes."""
|
||||
for worker in self.workers:
|
||||
for process in worker.processes:
|
||||
if not worker.auto_start:
|
||||
process.set_state(ProcessState.NONE, True)
|
||||
continue
|
||||
process.start()
|
||||
|
||||
def join(self):
|
||||
"""Join the worker processes."""
|
||||
logger.debug("Joining processes", extra={"verbosity": 1})
|
||||
joined = set()
|
||||
for process in self.processes:
|
||||
logger.debug(
|
||||
f"Found {process.pid} - {process.state.name}",
|
||||
extra={"verbosity": 1},
|
||||
)
|
||||
if process.state < ProcessState.JOINED:
|
||||
logger.debug(f"Joining {process.pid}", extra={"verbosity": 1})
|
||||
joined.add(process.pid)
|
||||
process.join()
|
||||
if joined:
|
||||
self.join()
|
||||
|
||||
def terminate(self):
|
||||
"""Terminate the worker processes."""
|
||||
if not self._shutting_down:
|
||||
for process in self.processes:
|
||||
process.terminate()
|
||||
|
||||
def cleanup(self):
|
||||
"""Cleanup the worker processes."""
|
||||
for process in self.processes:
|
||||
process.exit()
|
||||
|
||||
def restart(
|
||||
self,
|
||||
process_names: list[str] | None = None,
|
||||
restart_order=RestartOrder.SHUTDOWN_FIRST,
|
||||
**kwargs,
|
||||
):
|
||||
"""Restart the worker processes.
|
||||
|
||||
Args:
|
||||
process_names (Optional[List[str]], optional): The names of the processes to restart.
|
||||
If `None` then all processes will be restarted. Defaults to `None`.
|
||||
restart_order (RestartOrder, optional): The order in which to restart the processes.
|
||||
Defaults to `RestartOrder.SHUTDOWN_FIRST`.
|
||||
""" # noqa: E501
|
||||
self.restarter.restart(
|
||||
transient_processes=list(self.transient_processes),
|
||||
durable_processes=list(self.durable_processes),
|
||||
process_names=process_names,
|
||||
restart_order=restart_order,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def scale(self, num_worker: int):
|
||||
if num_worker <= 0:
|
||||
raise ValueError("Cannot scale to 0 workers.")
|
||||
|
||||
change = num_worker - self.num_server
|
||||
if change == 0:
|
||||
logger.info(
|
||||
f"No change needed. There are already {num_worker} workers."
|
||||
)
|
||||
return
|
||||
|
||||
logger.info(f"Scaling from {self.num_server} to {num_worker} workers")
|
||||
if change > 0:
|
||||
for _ in range(change):
|
||||
worker = self.create_server()
|
||||
for process in worker.processes:
|
||||
process.start()
|
||||
else:
|
||||
for _ in range(abs(change)):
|
||||
self.shutdown_server()
|
||||
self.num_server = num_worker
|
||||
|
||||
def monitor(self):
|
||||
"""Monitor the worker processes.
|
||||
|
||||
First, wait for all of the workers to acknowledge that they are ready.
|
||||
Then, wait for messages from the workers. If a message is received
|
||||
then it is processed and the state of the worker is updated.
|
||||
|
||||
Also used to restart, shutdown, and scale the workers.
|
||||
|
||||
Raises:
|
||||
ServerKilled: Raised when a worker fails to come online.
|
||||
"""
|
||||
self.wait_for_ack()
|
||||
while True:
|
||||
try:
|
||||
cycle = self._poll_monitor()
|
||||
if cycle is MonitorCycle.BREAK:
|
||||
break
|
||||
elif cycle is MonitorCycle.CONTINUE:
|
||||
continue
|
||||
self._sync_states()
|
||||
self._cleanup_non_tracked_workers()
|
||||
except InterruptedError:
|
||||
if not OS_IS_WINDOWS:
|
||||
raise
|
||||
break
|
||||
|
||||
def wait_for_ack(self): # no cov
|
||||
"""Wait for all of the workers to acknowledge that they are ready."""
|
||||
misses = 0
|
||||
message = (
|
||||
"It seems that one or more of your workers failed to come "
|
||||
"online in the allowed time. Sanic is shutting down to avoid a "
|
||||
f"deadlock. The current threshold is {self.THRESHOLD / 10}s. "
|
||||
"If this problem persists, please check out the documentation "
|
||||
"https://sanic.dev/en/guide/deployment/manager.html#worker-ack."
|
||||
)
|
||||
while not self._all_workers_ack():
|
||||
if self.monitor_subscriber.poll(0.1):
|
||||
monitor_msg = self.monitor_subscriber.recv()
|
||||
if monitor_msg != "__TERMINATE_EARLY__":
|
||||
self.monitor_publisher.send(monitor_msg)
|
||||
continue
|
||||
misses = self.THRESHOLD
|
||||
message = (
|
||||
"One of your worker processes terminated before startup "
|
||||
"was completed. Please solve any errors experienced "
|
||||
"during startup. If you do not see an exception traceback "
|
||||
"in your error logs, try running Sanic in a single "
|
||||
"process using --single-process or single_process=True. "
|
||||
"Once you are confident that the server is able to start "
|
||||
"without errors you can switch back to multiprocess mode."
|
||||
)
|
||||
misses += 1
|
||||
if misses > self.THRESHOLD:
|
||||
error_logger.error(
|
||||
"Not all workers acknowledged a successful startup. "
|
||||
"Shutting down.\n\n" + message
|
||||
)
|
||||
self.kill()
|
||||
|
||||
@property
|
||||
def workers(self) -> list[Worker]:
|
||||
"""Get all of the workers."""
|
||||
return list(self.transient.values()) + list(self.durable.values())
|
||||
|
||||
@property
|
||||
def all_workers(self) -> Iterable[tuple[str, Worker]]:
|
||||
return chain(self.transient.items(), self.durable.items())
|
||||
|
||||
@property
|
||||
def processes(self):
|
||||
"""Get all of the processes."""
|
||||
for worker in self.workers:
|
||||
for process in worker.processes:
|
||||
if not process.pid:
|
||||
continue
|
||||
yield process
|
||||
|
||||
@property
|
||||
def transient_processes(self):
|
||||
"""Get all of the transient processes."""
|
||||
for worker in self.transient.values():
|
||||
yield from worker.processes
|
||||
|
||||
@property
|
||||
def durable_processes(self):
|
||||
for worker in self.durable.values():
|
||||
yield from worker.processes
|
||||
|
||||
def kill(self):
|
||||
"""Kill all of the processes."""
|
||||
for process in self.processes:
|
||||
logger.info("Killing %s [%s]", process.name, process.pid)
|
||||
with suppress(ProcessLookupError):
|
||||
if os.name == "nt":
|
||||
# Windows has no os.killpg and SIGKILL doesn't seem to work
|
||||
os.kill(process.pid, signal.CTRL_C_EVENT)
|
||||
else:
|
||||
try:
|
||||
os.killpg(os.getpgid(process.pid), SIGKILL)
|
||||
except OSError:
|
||||
os.kill(process.pid, SIGKILL)
|
||||
raise ServerKilled
|
||||
|
||||
def shutdown_signal(self, signal, frame):
|
||||
"""Handle the shutdown signal."""
|
||||
if self._shutting_down:
|
||||
logger.info("Shutdown interrupted. Killing.")
|
||||
with suppress(ServerKilled):
|
||||
self.kill()
|
||||
return
|
||||
|
||||
logger.info("Received signal %s. Shutting down.", Signals(signal).name)
|
||||
self.monitor_publisher.send(None)
|
||||
self.shutdown()
|
||||
|
||||
def shutdown(self):
|
||||
"""Shutdown the worker manager."""
|
||||
for process in self.processes:
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
self._shutting_down = True
|
||||
|
||||
def remove_worker(self, worker: Worker) -> None:
|
||||
if worker.tracked:
|
||||
error_logger.error(
|
||||
f"Worker {worker.name} is tracked and cannot be removed."
|
||||
)
|
||||
return
|
||||
if worker.has_alive_processes():
|
||||
error_logger.error(
|
||||
f"Worker {worker.name} has alive processes and cannot be "
|
||||
"removed."
|
||||
)
|
||||
return
|
||||
self.transient.pop(worker.name, None)
|
||||
self.durable.pop(worker.name, None)
|
||||
for process in worker.processes:
|
||||
self.worker_state.pop(process.name, None)
|
||||
logger.info("Removed worker %s", worker.name)
|
||||
del worker
|
||||
|
||||
@property
|
||||
def pid(self):
|
||||
"""Get the process ID of the main process."""
|
||||
return os.getpid()
|
||||
|
||||
def _all_workers_ack(self):
|
||||
acked = [
|
||||
worker_state.get("state") == ProcessState.ACKED.name
|
||||
for worker_state in self.worker_state.values()
|
||||
if worker_state.get("server")
|
||||
]
|
||||
return all(acked) and len(acked) == self.num_server
|
||||
|
||||
def _sync_states(self):
|
||||
for process in self.processes:
|
||||
try:
|
||||
state = self.worker_state[process.name].get("state")
|
||||
except KeyError:
|
||||
process.set_state(ProcessState.TERMINATED, True)
|
||||
continue
|
||||
# Skip state sync if process is restarting to avoid race condition
|
||||
if process.state == ProcessState.RESTARTING:
|
||||
continue
|
||||
if not process.is_alive():
|
||||
state = "FAILED" if process.exitcode else "COMPLETED"
|
||||
if state and process.state.name != state:
|
||||
process.set_state(ProcessState[state], True)
|
||||
|
||||
def _cleanup_non_tracked_workers(self) -> None:
|
||||
to_remove = [
|
||||
worker
|
||||
for worker in self.workers
|
||||
if not worker.tracked and not worker.has_alive_processes()
|
||||
]
|
||||
|
||||
for worker in to_remove:
|
||||
self.remove_worker(worker)
|
||||
|
||||
def _poll_monitor(self) -> MonitorCycle | None:
|
||||
if self.monitor_subscriber.poll(0.1):
|
||||
message = self.monitor_subscriber.recv()
|
||||
logger.debug(f"Monitor message: {message}", extra={"verbosity": 2})
|
||||
if not message:
|
||||
return MonitorCycle.BREAK
|
||||
elif message == "__TERMINATE__":
|
||||
self._handle_terminate()
|
||||
return MonitorCycle.BREAK
|
||||
elif isinstance(message, tuple) and (
|
||||
len(message) == 7 or len(message) == 8
|
||||
):
|
||||
self._handle_manage(*message) # type: ignore
|
||||
return MonitorCycle.CONTINUE
|
||||
elif not isinstance(message, str):
|
||||
error_logger.error(
|
||||
"Monitor received an invalid message: %s", message
|
||||
)
|
||||
return MonitorCycle.CONTINUE
|
||||
return self._handle_message(message)
|
||||
return None
|
||||
|
||||
def _handle_terminate(self) -> None:
|
||||
self.shutdown()
|
||||
|
||||
def _handle_message(self, message: str) -> MonitorCycle | None:
|
||||
logger.debug(
|
||||
"Incoming monitor message: %s",
|
||||
message,
|
||||
extra={"verbosity": 1},
|
||||
)
|
||||
split_message = message.split(":", 2)
|
||||
if message.startswith("__SCALE__"):
|
||||
self.scale(int(split_message[-1]))
|
||||
return MonitorCycle.CONTINUE
|
||||
|
||||
processes = split_message[0]
|
||||
reloaded_files = split_message[1] if len(split_message) > 1 else None
|
||||
process_names: list[str] | None = [
|
||||
name.strip() for name in processes.split(",")
|
||||
]
|
||||
if process_names and "__ALL_PROCESSES__" in process_names:
|
||||
process_names = None
|
||||
order = (
|
||||
RestartOrder.STARTUP_FIRST
|
||||
if "STARTUP_FIRST" in split_message
|
||||
else RestartOrder.SHUTDOWN_FIRST
|
||||
)
|
||||
self.restart(
|
||||
process_names=process_names,
|
||||
reloaded_files=reloaded_files,
|
||||
restart_order=order,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _handle_manage(
|
||||
self,
|
||||
name: str,
|
||||
func: Callable[..., Any],
|
||||
kwargs: dict[str, Any],
|
||||
transient: bool,
|
||||
restartable: bool | None,
|
||||
tracked: bool,
|
||||
auto_start: bool,
|
||||
workers: int,
|
||||
) -> None:
|
||||
try:
|
||||
worker = self.manage(
|
||||
name,
|
||||
func,
|
||||
kwargs,
|
||||
transient=transient,
|
||||
restartable=restartable,
|
||||
tracked=tracked,
|
||||
auto_start=auto_start,
|
||||
workers=workers,
|
||||
)
|
||||
except Exception:
|
||||
error_logger.exception("Failed to manage worker %s", name)
|
||||
else:
|
||||
if not auto_start:
|
||||
return
|
||||
for process in worker.processes:
|
||||
process.start()
|
||||
@@ -0,0 +1,169 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from multiprocessing.connection import Connection
|
||||
from os import environ, getpid
|
||||
from typing import Any, Callable
|
||||
|
||||
from sanic.log import Colors, logger
|
||||
from sanic.worker.process import ProcessState
|
||||
from sanic.worker.state import WorkerState
|
||||
|
||||
|
||||
class WorkerMultiplexer:
|
||||
"""Multiplexer for Sanic workers.
|
||||
|
||||
This is instantiated inside of worker porocesses only. It is used to
|
||||
communicate with the monitor process.
|
||||
|
||||
Args:
|
||||
monitor_publisher (Connection): The connection to the monitor.
|
||||
worker_state (Dict[str, Any]): The state of the worker.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
monitor_publisher: Connection,
|
||||
worker_state: dict[str, Any],
|
||||
):
|
||||
self._monitor_publisher = monitor_publisher
|
||||
self._state = WorkerState(worker_state, self.name)
|
||||
|
||||
def ack(self):
|
||||
"""Acknowledge the worker is ready."""
|
||||
logger.debug(
|
||||
f"{Colors.BLUE}Process ack: {Colors.BOLD}{Colors.SANIC}"
|
||||
f"%s {Colors.BLUE}[%s]{Colors.END}",
|
||||
self.name,
|
||||
self.pid,
|
||||
)
|
||||
self._state._state[self.name] = {
|
||||
**self._state._state[self.name],
|
||||
"state": ProcessState.ACKED.name,
|
||||
}
|
||||
|
||||
def manage(
|
||||
self,
|
||||
ident: str,
|
||||
func: Callable[..., Any],
|
||||
kwargs: dict[str, Any],
|
||||
transient: bool = False,
|
||||
restartable: bool | None = None,
|
||||
tracked: bool = False,
|
||||
auto_start: bool = True,
|
||||
workers: int = 1,
|
||||
) -> None:
|
||||
"""Manages the initiation and monitoring of a worker process.
|
||||
|
||||
Args:
|
||||
ident (str): A unique identifier for the worker process.
|
||||
func (Callable[..., Any]): The function to be executed in the worker process.
|
||||
kwargs (Dict[str, Any]): A dictionary of arguments to be passed to `func`.
|
||||
transient (bool, optional): Flag to mark the process as transient. If `True`,
|
||||
the Worker Manager will restart the process with any global restart
|
||||
(e.g., auto-reload). Defaults to `False`.
|
||||
restartable (Optional[bool], optional): Flag to mark the process as restartable. If `True`,
|
||||
the Worker Manager can restart the process if prompted. Defaults to `None`.
|
||||
tracked (bool, optional): Flag to indicate whether the process should be tracked
|
||||
after its completion. Defaults to `False`.
|
||||
auto_start (bool, optional): Flag to indicate whether the process should be started
|
||||
workers (int, optional): The number of worker processes to run. Defaults to 1.
|
||||
|
||||
This method packages the provided arguments into a bundle and sends them back to the
|
||||
main process to be managed by the Worker Manager.
|
||||
""" # noqa: E501
|
||||
bundle = (
|
||||
ident,
|
||||
func,
|
||||
kwargs,
|
||||
transient,
|
||||
restartable,
|
||||
tracked,
|
||||
auto_start,
|
||||
workers,
|
||||
)
|
||||
self._monitor_publisher.send(bundle)
|
||||
|
||||
def set_serving(self, serving: bool) -> None:
|
||||
"""Set the worker to serving.
|
||||
|
||||
Args:
|
||||
serving (bool): Whether the worker is serving.
|
||||
"""
|
||||
self._state._state[self.name] = {
|
||||
**self._state._state[self.name],
|
||||
"serving": serving,
|
||||
}
|
||||
|
||||
def exit(self):
|
||||
"""Run cleanup at worker exit."""
|
||||
try:
|
||||
del self._state._state[self.name]
|
||||
except ConnectionRefusedError:
|
||||
logger.debug("Monitor process has already exited.")
|
||||
|
||||
def restart(
|
||||
self,
|
||||
name: str = "",
|
||||
all_workers: bool = False,
|
||||
zero_downtime: bool = False,
|
||||
):
|
||||
"""Restart the worker.
|
||||
|
||||
Args:
|
||||
name (str): The name of the process to restart.
|
||||
all_workers (bool): Whether to restart all workers.
|
||||
zero_downtime (bool): Whether to restart with zero downtime.
|
||||
"""
|
||||
if name and all_workers:
|
||||
raise ValueError(
|
||||
"Ambiguous restart with both a named process and"
|
||||
" all_workers=True"
|
||||
)
|
||||
if not name:
|
||||
name = "__ALL_PROCESSES__:" if all_workers else self.name
|
||||
if not name.endswith(":"):
|
||||
name += ":"
|
||||
if zero_downtime:
|
||||
name += ":STARTUP_FIRST"
|
||||
self._monitor_publisher.send(name)
|
||||
|
||||
reload = restart # no cov
|
||||
"""Alias for restart."""
|
||||
|
||||
def scale(self, num_workers: int):
|
||||
"""Scale the number of workers.
|
||||
|
||||
Args:
|
||||
num_workers (int): The number of workers to scale to.
|
||||
"""
|
||||
message = f"__SCALE__:{num_workers}"
|
||||
self._monitor_publisher.send(message)
|
||||
|
||||
def terminate(self, early: bool = False):
|
||||
"""Terminate the worker.
|
||||
|
||||
Args:
|
||||
early (bool): Whether to terminate early.
|
||||
"""
|
||||
message = "__TERMINATE_EARLY__" if early else "__TERMINATE__"
|
||||
self._monitor_publisher.send(message)
|
||||
|
||||
@property
|
||||
def pid(self) -> int:
|
||||
"""The process ID of the worker."""
|
||||
return getpid()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""The name of the worker."""
|
||||
return environ.get("SANIC_WORKER_NAME", "")
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
"""The state of the worker."""
|
||||
return self._state
|
||||
|
||||
@property
|
||||
def workers(self) -> dict[str, Any]:
|
||||
"""The state of all workers."""
|
||||
return self.state.full()
|
||||
@@ -0,0 +1,292 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from collections.abc import MutableMapping
|
||||
from datetime import datetime, timezone
|
||||
from inspect import signature
|
||||
from multiprocessing.context import BaseContext
|
||||
from signal import SIGINT
|
||||
from threading import Thread
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
from sanic.log import Colors, logger
|
||||
from sanic.worker.constants import ProcessState, RestartOrder
|
||||
|
||||
|
||||
def get_now():
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
return now
|
||||
|
||||
|
||||
class WorkerProcess:
|
||||
"""A worker process."""
|
||||
|
||||
THRESHOLD = 300 # == 30 seconds
|
||||
SERVER_LABEL = "Server"
|
||||
SERVER_IDENTIFIER = "Srv"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
factory,
|
||||
name,
|
||||
ident,
|
||||
target,
|
||||
kwargs,
|
||||
worker_state,
|
||||
restartable: bool = False,
|
||||
):
|
||||
self.state = ProcessState.IDLE
|
||||
self.factory = factory
|
||||
self.name = name
|
||||
self.ident = ident
|
||||
self.target = target
|
||||
self.kwargs = kwargs
|
||||
self.worker_state = worker_state
|
||||
self.restartable = restartable
|
||||
if self.name not in self.worker_state:
|
||||
self.worker_state[self.name] = {
|
||||
"server": self.SERVER_LABEL in self.name
|
||||
}
|
||||
self.spawn()
|
||||
|
||||
def set_state(self, state: ProcessState, force=False):
|
||||
if not force and state < self.state:
|
||||
raise Exception("...")
|
||||
self.state = state
|
||||
self.worker_state[self.name] = {
|
||||
**self.worker_state[self.name],
|
||||
"state": self.state.name,
|
||||
}
|
||||
|
||||
def start(self):
|
||||
os.environ["SANIC_WORKER_NAME"] = self.name
|
||||
os.environ["SANIC_WORKER_IDENTIFIER"] = self.ident
|
||||
logger.debug(
|
||||
f"{Colors.BLUE}Starting a process: {Colors.BOLD}"
|
||||
f"{Colors.SANIC}%s{Colors.END}",
|
||||
self.name,
|
||||
)
|
||||
self.set_state(ProcessState.STARTING)
|
||||
self._current_process.start()
|
||||
self.set_state(ProcessState.STARTED)
|
||||
if not self.worker_state[self.name].get("starts"):
|
||||
self.worker_state[self.name] = {
|
||||
**self.worker_state[self.name],
|
||||
"pid": self.pid,
|
||||
"start_at": get_now(),
|
||||
"starts": 1,
|
||||
}
|
||||
del os.environ["SANIC_WORKER_NAME"]
|
||||
del os.environ["SANIC_WORKER_IDENTIFIER"]
|
||||
|
||||
def join(self):
|
||||
self.set_state(ProcessState.JOINED)
|
||||
self._current_process.join()
|
||||
|
||||
def exit(self):
|
||||
limit = 100
|
||||
while self.is_alive() and limit > 0:
|
||||
sleep(0.1)
|
||||
limit -= 1
|
||||
|
||||
if not self.is_alive():
|
||||
try:
|
||||
del self.worker_state[self.name]
|
||||
except (
|
||||
BrokenPipeError,
|
||||
ConnectionRefusedError,
|
||||
ConnectionResetError,
|
||||
EOFError,
|
||||
):
|
||||
logger.debug("Monitor process has already exited.")
|
||||
except KeyError:
|
||||
logger.debug("Could not find worker state to delete.")
|
||||
|
||||
def terminate(self):
|
||||
if self.state is not ProcessState.TERMINATED:
|
||||
logger.debug(
|
||||
f"{Colors.BLUE}Terminating a process: "
|
||||
f"{Colors.BOLD}{Colors.SANIC}"
|
||||
f"%s {Colors.BLUE}[%s]{Colors.END}",
|
||||
self.name,
|
||||
self.pid,
|
||||
)
|
||||
try:
|
||||
self.set_state(ProcessState.TERMINATED, force=True)
|
||||
except (BrokenPipeError, ConnectionResetError, EOFError):
|
||||
pass
|
||||
try:
|
||||
os.kill(self.pid, SIGINT)
|
||||
except (KeyError, AttributeError, ProcessLookupError):
|
||||
...
|
||||
|
||||
def restart(self, restart_order=RestartOrder.SHUTDOWN_FIRST, **kwargs):
|
||||
logger.debug(
|
||||
f"{Colors.BLUE}Restarting a process: {Colors.BOLD}{Colors.SANIC}"
|
||||
f"%s {Colors.BLUE}[%s]{Colors.END}",
|
||||
self.name,
|
||||
self.pid,
|
||||
)
|
||||
self.set_state(ProcessState.RESTARTING, force=True)
|
||||
if restart_order is RestartOrder.SHUTDOWN_FIRST:
|
||||
self._terminate_now()
|
||||
else:
|
||||
self._old_process = self._current_process
|
||||
if self._add_config():
|
||||
self.kwargs.update(
|
||||
{"config": {k.upper(): v for k, v in kwargs.items()}}
|
||||
)
|
||||
try:
|
||||
self.spawn()
|
||||
self.start()
|
||||
except AttributeError:
|
||||
raise RuntimeError("Restart failed")
|
||||
|
||||
if restart_order is RestartOrder.STARTUP_FIRST:
|
||||
self._terminate_soon()
|
||||
|
||||
self.worker_state[self.name] = {
|
||||
**self.worker_state[self.name],
|
||||
"pid": self.pid,
|
||||
"starts": self.worker_state[self.name]["starts"] + 1,
|
||||
"restart_at": get_now(),
|
||||
}
|
||||
|
||||
def is_alive(self):
|
||||
try:
|
||||
return self._current_process.is_alive()
|
||||
except AssertionError:
|
||||
return False
|
||||
|
||||
def spawn(self):
|
||||
if self.state not in (ProcessState.IDLE, ProcessState.RESTARTING):
|
||||
raise Exception("Cannot spawn a worker process until it is idle.")
|
||||
self._current_process = self.factory(
|
||||
name=self.name,
|
||||
target=self.target,
|
||||
kwargs=self.kwargs,
|
||||
daemon=True,
|
||||
)
|
||||
|
||||
@property
|
||||
def pid(self):
|
||||
return self._current_process.pid
|
||||
|
||||
@property
|
||||
def exitcode(self):
|
||||
return self._current_process.exitcode
|
||||
|
||||
def _terminate_now(self):
|
||||
if not self._current_process.is_alive():
|
||||
return
|
||||
logger.debug(
|
||||
f"{Colors.BLUE}Begin restart termination: "
|
||||
f"{Colors.BOLD}{Colors.SANIC}"
|
||||
f"%s {Colors.BLUE}[%s]{Colors.END}",
|
||||
self.name,
|
||||
self._current_process.pid,
|
||||
)
|
||||
self._current_process.terminate()
|
||||
|
||||
def _terminate_soon(self):
|
||||
logger.debug(
|
||||
f"{Colors.BLUE}Begin restart termination: "
|
||||
f"{Colors.BOLD}{Colors.SANIC}"
|
||||
f"%s {Colors.BLUE}[%s]{Colors.END}",
|
||||
self.name,
|
||||
self._current_process.pid,
|
||||
)
|
||||
termination_thread = Thread(target=self._wait_to_terminate)
|
||||
termination_thread.start()
|
||||
|
||||
def _wait_to_terminate(self):
|
||||
logger.debug(
|
||||
f"{Colors.BLUE}Waiting for process to be acked: "
|
||||
f"{Colors.BOLD}{Colors.SANIC}"
|
||||
f"%s {Colors.BLUE}[%s]{Colors.END}",
|
||||
self.name,
|
||||
self._old_process.pid,
|
||||
)
|
||||
misses = 0
|
||||
while self.state is not ProcessState.ACKED:
|
||||
sleep(0.1)
|
||||
misses += 1
|
||||
if misses > self.THRESHOLD:
|
||||
raise TimeoutError(
|
||||
f"Worker {self.name} failed to come ack within "
|
||||
f"{self.THRESHOLD / 10} seconds"
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
f"{Colors.BLUE}Process acked. Terminating: "
|
||||
f"{Colors.BOLD}{Colors.SANIC}"
|
||||
f"%s {Colors.BLUE}[%s]{Colors.END}",
|
||||
self.name,
|
||||
self._old_process.pid,
|
||||
)
|
||||
self._old_process.terminate()
|
||||
delattr(self, "_old_process")
|
||||
|
||||
def _add_config(self) -> bool:
|
||||
sig = signature(self.target)
|
||||
if "config" in sig.parameters or any(
|
||||
param.kind == param.VAR_KEYWORD
|
||||
for param in sig.parameters.values()
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class Worker:
|
||||
WORKER_PREFIX = "Sanic"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ident: str,
|
||||
name: str,
|
||||
serve,
|
||||
server_settings,
|
||||
context: BaseContext,
|
||||
worker_state: MutableMapping[str, Any],
|
||||
num: int = 1,
|
||||
restartable: bool = False,
|
||||
tracked: bool = True,
|
||||
auto_start: bool = True,
|
||||
):
|
||||
self.ident = ident
|
||||
self.name = name
|
||||
self.num = num
|
||||
self.context = context
|
||||
self.serve = serve
|
||||
self.server_settings = server_settings
|
||||
self.worker_state = worker_state
|
||||
self.processes: set[WorkerProcess] = set()
|
||||
self.restartable = restartable
|
||||
self.tracked = tracked
|
||||
self.auto_start = auto_start
|
||||
for _ in range(num):
|
||||
self.create_process()
|
||||
|
||||
def create_process(self) -> WorkerProcess:
|
||||
process = WorkerProcess(
|
||||
# Need to ignore this typing error - The problem is the
|
||||
# BaseContext itself has no Process. But, all of its
|
||||
# implementations do. We can safely ignore as it is a typing
|
||||
# issue in the standard lib.
|
||||
factory=self.context.Process, # type: ignore
|
||||
name="-".join(
|
||||
[self.WORKER_PREFIX, self.name, str(len(self.processes))]
|
||||
),
|
||||
ident=self.ident,
|
||||
target=self.serve,
|
||||
kwargs={**self.server_settings},
|
||||
worker_state=self.worker_state,
|
||||
restartable=self.restartable,
|
||||
)
|
||||
self.processes.add(process)
|
||||
return process
|
||||
|
||||
def has_alive_processes(self) -> bool:
|
||||
return any(process.is_alive() for process in self.processes)
|
||||
@@ -0,0 +1,122 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from asyncio import new_event_loop
|
||||
from itertools import chain
|
||||
from multiprocessing.connection import Connection
|
||||
from pathlib import Path
|
||||
from signal import SIGINT, SIGTERM
|
||||
from signal import signal as signal_func
|
||||
from time import sleep
|
||||
|
||||
from sanic.server.events import trigger_events
|
||||
from sanic.worker.loader import AppLoader
|
||||
|
||||
|
||||
class Reloader:
|
||||
INTERVAL = 1.0 # seconds
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
publisher: Connection,
|
||||
interval: float,
|
||||
reload_dirs: set[Path],
|
||||
app_loader: AppLoader,
|
||||
):
|
||||
self._publisher = publisher
|
||||
self.interval = interval or self.INTERVAL
|
||||
self.reload_dirs = reload_dirs
|
||||
self.run = True
|
||||
self.app_loader = app_loader
|
||||
|
||||
def __call__(self) -> None:
|
||||
app = self.app_loader.load()
|
||||
signal_func(SIGINT, self.stop)
|
||||
signal_func(SIGTERM, self.stop)
|
||||
mtimes: dict[str, float] = {}
|
||||
|
||||
reloader_start = app.listeners.get("reload_process_start")
|
||||
reloader_stop = app.listeners.get("reload_process_stop")
|
||||
before_trigger = app.listeners.get("before_reload_trigger")
|
||||
after_trigger = app.listeners.get("after_reload_trigger")
|
||||
loop = new_event_loop()
|
||||
if reloader_start:
|
||||
trigger_events(reloader_start, loop, app)
|
||||
|
||||
while self.run:
|
||||
changed = set()
|
||||
for filename in self.files():
|
||||
try:
|
||||
if self.check_file(filename, mtimes):
|
||||
path = (
|
||||
filename
|
||||
if isinstance(filename, str)
|
||||
else filename.resolve()
|
||||
)
|
||||
changed.add(str(path))
|
||||
except OSError:
|
||||
continue
|
||||
if changed:
|
||||
if before_trigger:
|
||||
trigger_events(before_trigger, loop, app)
|
||||
self.reload(",".join(changed) if changed else "unknown")
|
||||
if after_trigger:
|
||||
trigger_events(after_trigger, loop, app, changed=changed)
|
||||
sleep(self.interval)
|
||||
else:
|
||||
if reloader_stop:
|
||||
trigger_events(reloader_stop, loop, app)
|
||||
|
||||
def stop(self, *_):
|
||||
self.run = False
|
||||
|
||||
def reload(self, reloaded_files):
|
||||
message = f"__ALL_PROCESSES__:{reloaded_files}"
|
||||
self._publisher.send(message)
|
||||
|
||||
def files(self):
|
||||
return chain(
|
||||
self.python_files(),
|
||||
*(d.glob("**/*") for d in self.reload_dirs),
|
||||
)
|
||||
|
||||
def python_files(self): # no cov
|
||||
"""This iterates over all relevant Python files.
|
||||
|
||||
It goes through all
|
||||
loaded files from modules, all files in folders of already loaded
|
||||
modules as well as all files reachable through a package.
|
||||
"""
|
||||
# The list call is necessary on Python 3 in case the module
|
||||
# dictionary modifies during iteration.
|
||||
for module in list(sys.modules.values()):
|
||||
if module is None:
|
||||
continue
|
||||
filename = getattr(module, "__file__", None)
|
||||
if filename:
|
||||
old = None
|
||||
while not os.path.isfile(filename):
|
||||
old = filename
|
||||
filename = os.path.dirname(filename)
|
||||
if filename == old:
|
||||
break
|
||||
else:
|
||||
if filename[-4:] in (".pyc", ".pyo"):
|
||||
filename = filename[:-1]
|
||||
yield filename
|
||||
|
||||
@staticmethod
|
||||
def check_file(filename, mtimes) -> bool:
|
||||
need_reload = False
|
||||
|
||||
mtime = os.stat(filename).st_mtime
|
||||
old_time = mtimes.get(filename)
|
||||
if old_time is None:
|
||||
mtimes[filename] = mtime
|
||||
elif mtime > old_time:
|
||||
mtimes[filename] = mtime
|
||||
need_reload = True
|
||||
|
||||
return need_reload
|
||||
@@ -0,0 +1,90 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sanic.log import error_logger
|
||||
from sanic.worker.constants import RestartOrder
|
||||
from sanic.worker.process import ProcessState, WorkerProcess
|
||||
|
||||
|
||||
class Restarter:
|
||||
def restart(
|
||||
self,
|
||||
transient_processes: list[WorkerProcess],
|
||||
durable_processes: list[WorkerProcess],
|
||||
process_names: list[str] | None = None,
|
||||
restart_order=RestartOrder.SHUTDOWN_FIRST,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""Restart the worker processes.
|
||||
|
||||
Args:
|
||||
process_names (Optional[List[str]], optional): The names of the processes to restart.
|
||||
If `None`, then all processes will be restarted. Defaults to `None`.
|
||||
restart_order (RestartOrder, optional): The order in which to restart the processes.
|
||||
Defaults to `RestartOrder.SHUTDOWN_FIRST`.
|
||||
""" # noqa: E501
|
||||
restarted = self._restart_transient(
|
||||
transient_processes,
|
||||
process_names or [],
|
||||
restart_order,
|
||||
**kwargs,
|
||||
)
|
||||
restarted |= self._restart_durable(
|
||||
durable_processes,
|
||||
process_names or [],
|
||||
restart_order,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if process_names and not restarted:
|
||||
error_logger.error(
|
||||
f"Failed to restart processes: {', '.join(process_names)}"
|
||||
)
|
||||
|
||||
def _restart_transient(
|
||||
self,
|
||||
processes: list[WorkerProcess],
|
||||
process_names: list[str],
|
||||
restart_order: RestartOrder,
|
||||
**kwargs,
|
||||
) -> set[str]:
|
||||
restarted: set[str] = set()
|
||||
for process in processes:
|
||||
if not process.restartable or (
|
||||
process_names and process.name not in process_names
|
||||
):
|
||||
continue
|
||||
self._restart_process(process, restart_order, **kwargs)
|
||||
restarted.add(process.name)
|
||||
return restarted
|
||||
|
||||
def _restart_durable(
|
||||
self,
|
||||
processes: list[WorkerProcess],
|
||||
process_names: list[str],
|
||||
restart_order: RestartOrder,
|
||||
**kwargs,
|
||||
) -> set[str]:
|
||||
restarted: set[str] = set()
|
||||
if not process_names:
|
||||
return restarted
|
||||
for process in processes:
|
||||
if not process.restartable or process.name not in process_names:
|
||||
continue
|
||||
if process.state not in (
|
||||
ProcessState.COMPLETED,
|
||||
ProcessState.FAILED,
|
||||
ProcessState.NONE,
|
||||
):
|
||||
error_logger.error(
|
||||
f"Cannot restart process {process.name} because "
|
||||
"it is not in a final state. Current state is: "
|
||||
f"{process.state.name}."
|
||||
)
|
||||
continue
|
||||
self._restart_process(process, restart_order, **kwargs)
|
||||
restarted.add(process.name)
|
||||
|
||||
return restarted
|
||||
|
||||
def _restart_process(self, process, restart_order, **kwargs):
|
||||
process.restart(restart_order=restart_order, **kwargs)
|
||||
@@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import socket
|
||||
import warnings
|
||||
|
||||
from functools import partial
|
||||
from multiprocessing.connection import Connection
|
||||
from ssl import SSLContext
|
||||
from typing import Any
|
||||
|
||||
from sanic.application.constants import ServerStage
|
||||
from sanic.application.state import ApplicationServerInfo
|
||||
from sanic.http.constants import HTTP
|
||||
from sanic.log import error_logger
|
||||
from sanic.logging.setup import setup_logging
|
||||
from sanic.models.server_types import Signal
|
||||
from sanic.server.protocols.http_protocol import HttpProtocol
|
||||
from sanic.server.runners import _serve_http_1, _serve_http_3
|
||||
from sanic.worker.loader import AppLoader, CertLoader
|
||||
from sanic.worker.multiplexer import WorkerMultiplexer
|
||||
from sanic.worker.process import Worker, WorkerProcess
|
||||
|
||||
|
||||
def worker_serve(
|
||||
host,
|
||||
port,
|
||||
app_name: str,
|
||||
monitor_publisher: Connection | None,
|
||||
app_loader: AppLoader,
|
||||
worker_state: dict[str, Any] | None = None,
|
||||
server_info: dict[str, list[ApplicationServerInfo]] | None = None,
|
||||
ssl: SSLContext | dict[str, str | os.PathLike[str]] | None = None,
|
||||
sock: socket.socket | None = None,
|
||||
unix: str | None = None,
|
||||
reuse_port: bool = False,
|
||||
loop=None,
|
||||
protocol: type[asyncio.Protocol] = HttpProtocol,
|
||||
backlog: int = 100,
|
||||
register_sys_signals: bool = True,
|
||||
run_multiple: bool = False,
|
||||
run_async: bool = False,
|
||||
connections=None,
|
||||
signal=Signal(),
|
||||
state=None,
|
||||
asyncio_server_kwargs=None,
|
||||
version=HTTP.VERSION_1,
|
||||
config: bytes | str | dict[str, Any] | Any | None = None,
|
||||
passthru: dict[str, Any] | None = None,
|
||||
):
|
||||
try:
|
||||
from sanic import Sanic
|
||||
|
||||
if app_loader:
|
||||
app = app_loader.load()
|
||||
else:
|
||||
app = Sanic.get_app(app_name)
|
||||
|
||||
app.refresh(passthru)
|
||||
app.setup_loop()
|
||||
setup_logging(
|
||||
app.state.is_debug, app.config.NO_COLOR, app.config.LOG_EXTRA
|
||||
)
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
# Hydrate server info if needed
|
||||
if server_info:
|
||||
for app_name, server_info_objects in server_info.items():
|
||||
a = Sanic.get_app(app_name)
|
||||
if not a.state.server_info:
|
||||
a.state.server_info = []
|
||||
for info in server_info_objects:
|
||||
if not info.settings.get("app"):
|
||||
info.settings["app"] = a
|
||||
a.state.server_info.append(info)
|
||||
|
||||
if isinstance(ssl, dict) or app.certloader_class is not CertLoader:
|
||||
cert_loader = app.certloader_class(ssl or {})
|
||||
ssl = cert_loader.load(app)
|
||||
for info in app.state.server_info:
|
||||
info.settings["ssl"] = ssl
|
||||
|
||||
# When in a worker process, do some init
|
||||
worker_name = os.environ.get("SANIC_WORKER_NAME")
|
||||
if worker_name and worker_name.startswith(
|
||||
"-".join([Worker.WORKER_PREFIX, WorkerProcess.SERVER_LABEL])
|
||||
):
|
||||
# Hydrate apps with any passed server info
|
||||
|
||||
if monitor_publisher is None:
|
||||
raise RuntimeError(
|
||||
"No restart publisher found in worker process"
|
||||
)
|
||||
if worker_state is None:
|
||||
raise RuntimeError("No worker state found in worker process")
|
||||
|
||||
# Run secondary servers
|
||||
apps = list(Sanic._app_registry.values())
|
||||
app.before_server_start(partial(app._start_servers, apps=apps))
|
||||
for a in apps:
|
||||
a.multiplexer = WorkerMultiplexer(
|
||||
monitor_publisher, worker_state
|
||||
)
|
||||
|
||||
if app.debug:
|
||||
loop.set_debug(app.debug)
|
||||
|
||||
app.asgi = False
|
||||
|
||||
if app.state.server_info:
|
||||
primary_server_info = app.state.server_info[0]
|
||||
primary_server_info.stage = ServerStage.SERVING
|
||||
if config:
|
||||
app.update_config(config)
|
||||
|
||||
if version is HTTP.VERSION_3:
|
||||
return _serve_http_3(host, port, app, loop, ssl)
|
||||
return _serve_http_1(
|
||||
host,
|
||||
port,
|
||||
app,
|
||||
ssl,
|
||||
sock,
|
||||
unix,
|
||||
reuse_port,
|
||||
loop,
|
||||
protocol,
|
||||
backlog,
|
||||
register_sys_signals,
|
||||
run_multiple,
|
||||
run_async,
|
||||
connections,
|
||||
signal,
|
||||
state,
|
||||
asyncio_server_kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
warnings.simplefilter("ignore", category=RuntimeWarning)
|
||||
if monitor_publisher:
|
||||
error_logger.exception(e)
|
||||
multiplexer = WorkerMultiplexer(monitor_publisher, {})
|
||||
multiplexer.terminate(True)
|
||||
else:
|
||||
raise e
|
||||
@@ -0,0 +1,83 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import ItemsView, Iterator, KeysView, Mapping, ValuesView
|
||||
from collections.abc import Mapping as MappingType
|
||||
from typing import Any
|
||||
|
||||
|
||||
class WorkerState(Mapping):
|
||||
RESTRICTED = (
|
||||
"health",
|
||||
"pid",
|
||||
"requests",
|
||||
"restart_at",
|
||||
"server",
|
||||
"start_at",
|
||||
"starts",
|
||||
"state",
|
||||
)
|
||||
|
||||
def __init__(self, state: dict[str, Any], current: str) -> None:
|
||||
self._name = current
|
||||
self._state = state
|
||||
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
return self._state[self._name][key]
|
||||
|
||||
def __setitem__(self, key: str, value: Any) -> None:
|
||||
if key in self.RESTRICTED:
|
||||
self._write_error([key])
|
||||
self._state[self._name] = {
|
||||
**self._state[self._name],
|
||||
key: value,
|
||||
}
|
||||
|
||||
def __delitem__(self, key: str) -> None:
|
||||
if key in self.RESTRICTED:
|
||||
self._write_error([key])
|
||||
self._state[self._name] = {
|
||||
k: v for k, v in self._state[self._name].items() if k != key
|
||||
}
|
||||
|
||||
def __iter__(self) -> Iterator[Any]:
|
||||
return iter(self._state[self._name])
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._state[self._name])
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return repr(self._state[self._name])
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
return self._state[self._name] == other
|
||||
|
||||
def keys(self) -> KeysView[str]:
|
||||
return self._state[self._name].keys()
|
||||
|
||||
def values(self) -> ValuesView[Any]:
|
||||
return self._state[self._name].values()
|
||||
|
||||
def items(self) -> ItemsView[str, Any]:
|
||||
return self._state[self._name].items()
|
||||
|
||||
def update(self, mapping: MappingType[str, Any]) -> None:
|
||||
if any(k in self.RESTRICTED for k in mapping.keys()):
|
||||
self._write_error(
|
||||
[k for k in mapping.keys() if k in self.RESTRICTED]
|
||||
)
|
||||
self._state[self._name] = {
|
||||
**self._state[self._name],
|
||||
**mapping,
|
||||
}
|
||||
|
||||
def pop(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def full(self) -> dict[str, Any]:
|
||||
return dict(self._state)
|
||||
|
||||
def _write_error(self, keys: list[str]) -> None:
|
||||
raise LookupError(
|
||||
f"Cannot set restricted key{'s' if len(keys) > 1 else ''} on "
|
||||
f"WorkerState: {', '.join(keys)}"
|
||||
)
|
||||
Reference in New Issue
Block a user