diff --git a/README.md b/README.md index a4afcc4..87ecd66 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,27 @@ When running, don't forget to mount `/var/run/docker.sock` into the container. You should also force a container hostname to avoid unpredictable Docker hashes. For example: - ``` +# With Docker docker run --rm -ti \ -v /var/run/docker.sock:/var/run/docker.sock \ --hostname worker01 \ + --env DOCKER_HOST=/var/run/docker.sock + --env AYON_API_KEY=verysecureapikey \ + --env AYON_SERVER_URL="http://172.18.0.1:5000" \ + ynput/ayon-ash +``` + +``` +# With (Rootless) Podman +podman run --rm -ti \ + --security-opt label=disable \ + -v /run/user/1000/podman/podman.sock:/run/user/1000/podman/podman.sock \ + --env CONTAINER_HOST=/run/user/1000/podman/podman.sock + --hostname worker01 \ --env AYON_API_KEY=verysecureapikey \ --env AYON_SERVER_URL="http://172.18.0.1:5000" \ + --env AYON_USE_PODMAN="true" \ ynput/ayon-ash ``` @@ -45,3 +59,17 @@ the server url) will be passed to the spawned services. ### AYON_HOSTNAME Optional setting to override the hostname. + +### AYON_NETWORK + +Optional setting to specify the network where the **backend** is running. Otherwise infered by the running containers. + +### AYON_NETWORK_MODE + +Optional setting to specify the network **mode** which the **backend** is running on. Otherwise infered by the running containers. + +### DOCKER_HOST | CONTAINER_HOST + +Optional setting to specify a different Docker/Podman socket. + + diff --git a/ash/api.py b/ash/api.py index ce9a88f..ed96826 100644 --- a/ash/api.py +++ b/ash/api.py @@ -26,8 +26,9 @@ def __init__(self): while True: try: response = self.get("users/me") - except Exception: - logging.warning("Unable to connect to the server... Retrying") + except Exception as e: + logging.warning(f"Unable to connect to the server: {e}\nRetrying...") + time.sleep(5) continue break diff --git a/ash/config.py b/ash/config.py index be87b9f..db036ff 100644 --- a/ash/config.py +++ b/ash/config.py @@ -2,12 +2,14 @@ import socket import sys from typing import Literal +from urllib.parse import urlparse -import docker import dotenv -from nxtools import critical_error, logging +from nxtools import critical_error, log_traceback, logging from pydantic import BaseModel, Field, ValidationError +from .containers import PODMAN, get_container_client + logging.user = "ash" dotenv.load_dotenv() @@ -23,21 +25,42 @@ class Config(BaseModel): def get_local_info(): - client = docker.DockerClient(base_url="unix://var/run/docker.sock") - api = docker.APIClient(base_url="unix://var/run/docker.sock") + """Infer info from ASH's container. + + We get the "network" and "network_mode" from the current running + ASH (what runs this code) container. + + These two can be provided via `AYON_NETWORK` and `AYON_NETWORK_MODE`. + """ + client, api = get_container_client() + + logging.info("Querying existing containers...") for container in client.containers.list(): - insp = api.inspect_container(container.id) + if PODMAN: + insp = container.inspect() + else: + insp = api.inspect_container(container.id) if insp["Config"]["Hostname"] != socket.gethostname(): + logging.debug( + f"Hostname for container {insp['Name']} doesn't match ash's, ignoring." + ) continue - # print(json.dumps(insp, indent=4)) break else: logging.error("Weird, no container found for this host") sys.exit(1) - networks = insp["NetworkSettings"]["Networks"] + try: + network = next(iter(insp["NetworkSettings"]["Networks"].keys()), None) + network_mode = insp["HostConfig"]["NetworkMode"] + except Exception as e: + logging.error( + "ASH is not running in a defined network... make sure it's in" + "the same network as ayon-docker containers." + ) + log_traceback(e) - return {"networks": list(networks.keys())} + return {"network": network, "network_mode": network_mode} def get_config() -> Config: @@ -46,6 +69,19 @@ def get_config() -> Config: key = key.lower() if not key.startswith("ayon_"): continue + if key == "ayon_server_url": + # We won't be able to connect if we receive an `AYON_SERVER_URL` + # such as `http://localhost:5000` or `http://ayon-docker_server_1` + # So here we try to resolve it to an actual IP. If we fail, means + # we can't reach the backend at all. + try: + server_hostname = urlparse(val).hostname + assert server_hostname is not None, "Invalid URL" + server_ip = socket.gethostbyname(server_hostname) + val = val.replace(server_hostname, server_ip) + except Exception: + critical_error(f"Unable to resolve `AYON_SERVER_URL` {val}") + data[key.replace("ayon_", "", 1)] = val try: config = Config(**data) @@ -57,11 +93,12 @@ def get_config() -> Config: critical_error("Unable to configure API") - local_info = get_local_info() - if config.network is None and config.network_mode is None: - config.network = local_info["networks"][0] + local_info = get_local_info() + config.network = local_info["network"] + config.network_mode = local_info["network_mode"] + logging.debug(f"ASH Config is: {config}") return config diff --git a/ash/containers.py b/ash/containers.py new file mode 100644 index 0000000..fea575c --- /dev/null +++ b/ash/containers.py @@ -0,0 +1,45 @@ +import os + +from nxtools import logging + +PODMAN = os.getenv("AYON_USE_PODMAN", False) + +DOCKER_HOST = os.getenv("DOCKER_HOST", None) + +if not DOCKER_HOST: + DOCKER_HOST = os.getenv("CONTAINER_HOST", None) + +if not DOCKER_HOST: + if PODMAN: + DOCKER_HOST = "unix:///run/user/1000/podman/podman.sock" + else: + DOCKER_HOST = "unix://var/run/docker.sock" + + +def get_container_client(): + """Creates a Client connection to the Socket + + Depending on teh container runtime we use, it will import and + create the class acordingly. + + Note that podman does not require the "APIClient" to inspect `Containers.` + + Returns: + tuple(client, api): The Client object, and in case of Docker the APIClient. + """ + client = None + api = None + + if PODMAN: + from podman import PodmanClient + + client = PodmanClient(base_url=DOCKER_HOST) + logging.info("Using container client: Podman") + else: + from docker import APIClient, DockerClient + + client = DockerClient(base_url=DOCKER_HOST) + api = APIClient(base_url=DOCKER_HOST) + logging.info("Using container client: Docker") + + return client, api diff --git a/ash/service_logging.py b/ash/service_logging.py index 4cca7d0..4e1d58e 100644 --- a/ash/service_logging.py +++ b/ash/service_logging.py @@ -1,6 +1,6 @@ import threading -from nxtools import logging +from nxtools import log_traceback, logging class ServiceLog: @@ -10,16 +10,22 @@ def __init__(self, service_name: str, container): threading.Thread(target=self._run, daemon=True).start() def _run(self): - logging.info(f"Starting log stream for {self.service_name}") - for line in self.container.logs(stream=True, tail=1, follow=True): - print(f"{line.decode().strip()}") + logging.info( + f"Starting log stream for {self.service_name}, last 10 lines were..." + ) + for line in self.container.logs(stream=True, tail=10, stderr=True): + print(line.decode().strip()) # service exited # print the status code and free the container - status_code = self.container.wait()["StatusCode"] - logging.warning(f"{self.service_name} exited with code {status_code}") - self.container = None + try: + status_code = self.container.wait()["StatusCode"] + logging.warning(f"{self.service_name} exited with code {status_code}") + except Exception as e: + logging.warning("Lost connection to the container:") + log_traceback(e) + self.container = None class ServiceLogger: diff --git a/ash/services.py b/ash/services.py index 3f1ffcb..a96253d 100644 --- a/ash/services.py +++ b/ash/services.py @@ -1,18 +1,19 @@ -import docker from nxtools import logging, slugify from .config import config +from .containers import get_container_client from .models import ServiceConfigModel from .service_logging import ServiceLogger class Services: - client: docker.DockerClient | None = None + client = None prefix: str = "io.ayon.service" @classmethod def connect(cls): - cls.client = docker.DockerClient(base_url="unix://var/run/docker.sock") + client, _ = get_container_client() + cls.client = client @classmethod def get_running_services(cls) -> list[str]: @@ -56,13 +57,17 @@ def spawn( if cls.client is None: return + network_mode = None + if config.network_mode and (not config.network): + network_mode = config.network_mode + container = cls.client.containers.run( image, detach=True, auto_remove=True, environment=environment, hostname=hostname, - network_mode=config.network_mode, + network_mode=network_mode, network=config.network, name=hostname, labels=labels, @@ -88,7 +93,6 @@ def ensure_running( # # Check whether it is running already # - container = None for container in cls.client.containers.list(): @@ -104,7 +108,6 @@ def ensure_running( except AssertionError: logging.error("SERVICE MISMATCH. This shouldn't happen. Stopping.") container.stop() - break else: # And start it @@ -132,5 +135,4 @@ def ensure_running( container = cls.spawn(image, hostname, environment, labels) - # Ensure container logger is running ServiceLogger.add(service_name, container) diff --git a/mypy.ini b/mypy.ini index 445cf68..553cf27 100644 --- a/mypy.ini +++ b/mypy.ini @@ -23,6 +23,11 @@ ignore_errors = true follow_imports = skip ignore_missing_imports = true +[mypy-podman.*] +ignore_errors = true +follow_imports = skip +ignore_missing_imports = true + [mypy-requests.*] ignore_errors = true follow_imports = skip diff --git a/pyproject.toml b/pyproject.toml index d389279..02311eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ pydantic = "^1.10.2" psutil = "^5.9.2" python-dotenv = "^0.21.0" docker = "^6.0.0" +podman = "^4.5.0" [tool.poetry.dev-dependencies] pytest = "^7.0"