From bb4a293caef1253a643f283d1a6f7b27009efeed Mon Sep 17 00:00:00 2001 From: Oscar Domingo Date: Mon, 30 Oct 2023 16:57:33 +0000 Subject: [PATCH 1/9] New `containers` module A module to handle the containers connection, based on the desired runtime. --- ash/containers.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 ash/containers.py diff --git a/ash/containers.py b/ash/containers.py new file mode 100644 index 0000000..cc82c6e --- /dev/null +++ b/ash/containers.py @@ -0,0 +1,43 @@ +import os + +from nxtools import logging + +PODMAN = os.getenv("AYON_USE_PODMAN", False) + +DOCKER_HOST = os.getenv("DOCKER_HOST", None) + +if not DOCKER_HOST: + DOCKER_HOST = os.getenv("CONTAINER_HOST", None) + +if not DOCKER_HOST: + if PODMAN: + DOCKER_HOST = "unix:///run/user/1000/podman/podman.sock" + else: + DOCKER_HOST = "unix://var/run/docker.sock" + + +def get_container_client(): + """Creates a Client connection to the Socket + + Depending on teh container runtime we use, it will import and + create the class acordingly. + + Note that podman does not require the "APIClient" to inspect `Containers.` + + Returns: + tuple(client, api): The Client object, and in case of Docker the APIClient. + """ + client = None + api = None + + if PODMAN: + from podman import PodmanClient + client = PodmanClient(base_url=DOCKER_HOST) + logging.info("Using container client: Podman") + else: + from docker import APIClient, DockerClient + client = DockerClient(base_url=DOCKER_HOST) + api = APIClient(base_url=DOCKER_HOST) + logging.info("Using container client: Docker") + + return client, api From e3a968b0df0034bc37fda8289181105db2bde8ce Mon Sep 17 00:00:00 2001 From: Oscar Domingo Date: Tue, 30 May 2023 16:58:00 +0200 Subject: [PATCH 2/9] Add rootless `podman` support This commit will allow the person to set `AYON_USE_PODMAN` in order to use `podman` instead of `docker`. This approach relies in a `podman` daemon being run which we then pass to `ash` by mapping it such as: `/run/user/1000/podman/podman.sock:/run/user/1000/podman/podman.sock` to get a user service running it can be created and enabled with: `systemctl --user enable --now io.podman.socket` so the above socket is available. --- README.md | 30 ++++++++++++++++++++++++- ash/api.py | 5 +++-- ash/config.py | 60 ++++++++++++++++++++++++++++++++++++++++--------- ash/services.py | 20 ++++++++++++----- pyproject.toml | 1 + 5 files changed, 96 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a4afcc4..87ecd66 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,27 @@ When running, don't forget to mount `/var/run/docker.sock` into the container. You should also force a container hostname to avoid unpredictable Docker hashes. For example: - ``` +# With Docker docker run --rm -ti \ -v /var/run/docker.sock:/var/run/docker.sock \ --hostname worker01 \ + --env DOCKER_HOST=/var/run/docker.sock + --env AYON_API_KEY=verysecureapikey \ + --env AYON_SERVER_URL="http://172.18.0.1:5000" \ + ynput/ayon-ash +``` + +``` +# With (Rootless) Podman +podman run --rm -ti \ + --security-opt label=disable \ + -v /run/user/1000/podman/podman.sock:/run/user/1000/podman/podman.sock \ + --env CONTAINER_HOST=/run/user/1000/podman/podman.sock + --hostname worker01 \ --env AYON_API_KEY=verysecureapikey \ --env AYON_SERVER_URL="http://172.18.0.1:5000" \ + --env AYON_USE_PODMAN="true" \ ynput/ayon-ash ``` @@ -45,3 +59,17 @@ the server url) will be passed to the spawned services. ### AYON_HOSTNAME Optional setting to override the hostname. + +### AYON_NETWORK + +Optional setting to specify the network where the **backend** is running. Otherwise infered by the running containers. + +### AYON_NETWORK_MODE + +Optional setting to specify the network **mode** which the **backend** is running on. Otherwise infered by the running containers. + +### DOCKER_HOST | CONTAINER_HOST + +Optional setting to specify a different Docker/Podman socket. + + diff --git a/ash/api.py b/ash/api.py index ce9a88f..ed96826 100644 --- a/ash/api.py +++ b/ash/api.py @@ -26,8 +26,9 @@ def __init__(self): while True: try: response = self.get("users/me") - except Exception: - logging.warning("Unable to connect to the server... Retrying") + except Exception as e: + logging.warning(f"Unable to connect to the server: {e}\nRetrying...") + time.sleep(5) continue break diff --git a/ash/config.py b/ash/config.py index be87b9f..15fcc0c 100644 --- a/ash/config.py +++ b/ash/config.py @@ -2,10 +2,12 @@ import socket import sys from typing import Literal +from urllib.parse import urlparse + +from .containers import PODMAN, get_container_client -import docker import dotenv -from nxtools import critical_error, logging +from nxtools import critical_error, logging, log_traceback from pydantic import BaseModel, Field, ValidationError logging.user = "ash" @@ -23,21 +25,41 @@ class Config(BaseModel): def get_local_info(): - client = docker.DockerClient(base_url="unix://var/run/docker.sock") - api = docker.APIClient(base_url="unix://var/run/docker.sock") + """Infer info from ASH's container. + + We get the "network" and "network_mode" from the current running + ASH (what runs this code) container. + + These two can be provided via `AYON_NETWORK` and `AYON_NETWORK_MODE`. + """ + client, api = get_container_client() + + logging.info("Querying existing containers...") for container in client.containers.list(): - insp = api.inspect_container(container.id) + if PODMAN: + insp = container.inspect() + else: + insp = api.inspect_container(container.id) if insp["Config"]["Hostname"] != socket.gethostname(): + logging.debug( + f"Hostname for container {insp['Name']} doesn't match ash's, ignoring." + ) continue - # print(json.dumps(insp, indent=4)) break else: logging.error("Weird, no container found for this host") sys.exit(1) - networks = insp["NetworkSettings"]["Networks"] + try: + network = next(iter(insp["NetworkSettings"]["Networks"].keys()), None) + network_mode = insp["HostConfig"]["NetworkMode"] + except Exceptions as e: + logging.error( + "ASH is not running in a defined network... make sure it's in" + "the same network as ayon-docker containers.") + log_traceback(e) - return {"networks": list(networks.keys())} + return {"network": network, "network_mode": network_mode} def get_config() -> Config: @@ -46,6 +68,21 @@ def get_config() -> Config: key = key.lower() if not key.startswith("ayon_"): continue + if key == "ayon_server_url": + # We won't be able to connect if we receive an `AYON_SERVER_URL` + # such as `http://localhost:5000` or `http://ayon-docker_server_1` + # So here we try to resolve it to an actual IP. If we fail, means + # we can't reach the backend at all. + try: + original_value = val + server_hostname = urlparse(val).hostname + server_ip = socket.gethostbyname(server_hostname) + val = val.replace(server_hostname, server_ip) + except Exception as e: + critical_error( + "Unable to resolve `AYON_SERVER_URL` {original_value}" + ) + data[key.replace("ayon_", "", 1)] = val try: config = Config(**data) @@ -57,11 +94,12 @@ def get_config() -> Config: critical_error("Unable to configure API") - local_info = get_local_info() - if config.network is None and config.network_mode is None: - config.network = local_info["networks"][0] + local_info = get_local_info() + config.network = local_info["network"] + config.network_mode = local_info["network_mode"] + logging.debug(f"ASH Config is: {config}") return config diff --git a/ash/services.py b/ash/services.py index 3f1ffcb..1858895 100644 --- a/ash/services.py +++ b/ash/services.py @@ -1,18 +1,23 @@ -import docker +import os +import socket + + from nxtools import logging, slugify from .config import config +from .containers import PODMAN, get_container_client from .models import ServiceConfigModel from .service_logging import ServiceLogger class Services: - client: docker.DockerClient | None = None + client = None prefix: str = "io.ayon.service" @classmethod def connect(cls): - cls.client = docker.DockerClient(base_url="unix://var/run/docker.sock") + client, _ = get_container_client() + cls.client = client @classmethod def get_running_services(cls) -> list[str]: @@ -22,10 +27,13 @@ def get_running_services(cls) -> list[str]: if cls.client is None: return result + logging.debug("Checking for Running services.") + for container in cls.client.containers.list(): labels = container.labels if service_name := labels.get(f"{cls.prefix}.service_name"): result.append(service_name) + logging.debug("Found {0} running services: {1}".format(len(result), result)) return result @classmethod @@ -88,7 +96,7 @@ def ensure_running( # # Check whether it is running already # - + logging.info("Checking if Service is already running.") container = None for container in cls.client.containers.list(): @@ -104,7 +112,8 @@ def ensure_running( except AssertionError: logging.error("SERVICE MISMATCH. This shouldn't happen. Stopping.") container.stop() - + else: + logging.debug(f"Service {service_name} already running at {container.id}") break else: # And start it @@ -132,5 +141,4 @@ def ensure_running( container = cls.spawn(image, hostname, environment, labels) - # Ensure container logger is running ServiceLogger.add(service_name, container) diff --git a/pyproject.toml b/pyproject.toml index d389279..02311eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ pydantic = "^1.10.2" psutil = "^5.9.2" python-dotenv = "^0.21.0" docker = "^6.0.0" +podman = "^4.5.0" [tool.poetry.dev-dependencies] pytest = "^7.0" From ff7aeee9a5c269acb7007871b48a47ec4a16e3fc Mon Sep 17 00:00:00 2001 From: Oscar Domingo Date: Mon, 30 Oct 2023 16:59:32 +0000 Subject: [PATCH 3/9] `ServiceLogger` Improve output of services logs. A better formatting, so they slightly match ASH's logs. --- ash/service_logging.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/ash/service_logging.py b/ash/service_logging.py index 4cca7d0..cb599f6 100644 --- a/ash/service_logging.py +++ b/ash/service_logging.py @@ -1,6 +1,6 @@ import threading -from nxtools import logging +from nxtools import logging, log_traceback class ServiceLog: @@ -10,16 +10,28 @@ def __init__(self, service_name: str, container): threading.Thread(target=self._run, daemon=True).start() def _run(self): - logging.info(f"Starting log stream for {self.service_name}") - for line in self.container.logs(stream=True, tail=1, follow=True): - print(f"{line.decode().strip()}") + logging.info(f"Starting log stream for {self.service_name}, last 10 lines were...") + for line in self.container.logs(stream=True, tail=10, stderr=True): + log_string = line.decode().strip() + log_elements = log_string.split(" ") + + log_date = log_elements[0] + log_time = log_elements[1] + log_severity = log_elements[2] + + log_message = log_string.split(log_severity)[-1] + print(f"{log_date} {log_time} {log_severity} {self.service_name} {log_message}") # service exited # print the status code and free the container - status_code = self.container.wait()["StatusCode"] - logging.warning(f"{self.service_name} exited with code {status_code}") - self.container = None + try: + status_code = self.container.wait()["StatusCode"] + logging.warning(f"{self.service_name} exited with code {status_code}") + except Exception as e: + logging.warning(f"Lost connection to the container:") + log_traceback(e) + self.container = None class ServiceLogger: From 198d4d9d71fdcdbcc6eb205a58099f42f1ac3fd1 Mon Sep 17 00:00:00 2001 From: Martin Wacker Date: Tue, 31 Oct 2023 13:09:31 +0100 Subject: [PATCH 4/9] fix: typo Exceptions x Exception --- ash/config.py | 18 ++++++++---------- ash/containers.py | 4 +++- ash/service_logging.py | 12 ++++++++---- ash/services.py | 10 ++++------ 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/ash/config.py b/ash/config.py index 15fcc0c..fa921c4 100644 --- a/ash/config.py +++ b/ash/config.py @@ -4,12 +4,12 @@ from typing import Literal from urllib.parse import urlparse -from .containers import PODMAN, get_container_client - import dotenv -from nxtools import critical_error, logging, log_traceback +from nxtools import critical_error, log_traceback, logging from pydantic import BaseModel, Field, ValidationError +from .containers import PODMAN, get_container_client + logging.user = "ash" dotenv.load_dotenv() @@ -53,10 +53,11 @@ def get_local_info(): try: network = next(iter(insp["NetworkSettings"]["Networks"].keys()), None) network_mode = insp["HostConfig"]["NetworkMode"] - except Exceptions as e: + except Exception as e: logging.error( "ASH is not running in a defined network... make sure it's in" - "the same network as ayon-docker containers.") + "the same network as ayon-docker containers." + ) log_traceback(e) return {"network": network, "network_mode": network_mode} @@ -74,14 +75,11 @@ def get_config() -> Config: # So here we try to resolve it to an actual IP. If we fail, means # we can't reach the backend at all. try: - original_value = val server_hostname = urlparse(val).hostname server_ip = socket.gethostbyname(server_hostname) val = val.replace(server_hostname, server_ip) - except Exception as e: - critical_error( - "Unable to resolve `AYON_SERVER_URL` {original_value}" - ) + except Exception: + critical_error("Unable to resolve `AYON_SERVER_URL` {original_value}") data[key.replace("ayon_", "", 1)] = val try: diff --git a/ash/containers.py b/ash/containers.py index cc82c6e..fea575c 100644 --- a/ash/containers.py +++ b/ash/containers.py @@ -17,7 +17,7 @@ def get_container_client(): - """Creates a Client connection to the Socket + """Creates a Client connection to the Socket Depending on teh container runtime we use, it will import and create the class acordingly. @@ -32,10 +32,12 @@ def get_container_client(): if PODMAN: from podman import PodmanClient + client = PodmanClient(base_url=DOCKER_HOST) logging.info("Using container client: Podman") else: from docker import APIClient, DockerClient + client = DockerClient(base_url=DOCKER_HOST) api = APIClient(base_url=DOCKER_HOST) logging.info("Using container client: Docker") diff --git a/ash/service_logging.py b/ash/service_logging.py index cb599f6..6e8e7d4 100644 --- a/ash/service_logging.py +++ b/ash/service_logging.py @@ -1,6 +1,6 @@ import threading -from nxtools import logging, log_traceback +from nxtools import log_traceback, logging class ServiceLog: @@ -10,7 +10,9 @@ def __init__(self, service_name: str, container): threading.Thread(target=self._run, daemon=True).start() def _run(self): - logging.info(f"Starting log stream for {self.service_name}, last 10 lines were...") + logging.info( + f"Starting log stream for {self.service_name}, last 10 lines were..." + ) for line in self.container.logs(stream=True, tail=10, stderr=True): log_string = line.decode().strip() log_elements = log_string.split(" ") @@ -20,7 +22,9 @@ def _run(self): log_severity = log_elements[2] log_message = log_string.split(log_severity)[-1] - print(f"{log_date} {log_time} {log_severity} {self.service_name} {log_message}") + print( + f"{log_date} {log_time} {log_severity} {self.service_name} {log_message}" + ) # service exited # print the status code and free the container @@ -29,7 +33,7 @@ def _run(self): status_code = self.container.wait()["StatusCode"] logging.warning(f"{self.service_name} exited with code {status_code}") except Exception as e: - logging.warning(f"Lost connection to the container:") + logging.warning("Lost connection to the container:") log_traceback(e) self.container = None diff --git a/ash/services.py b/ash/services.py index 1858895..d01dd9a 100644 --- a/ash/services.py +++ b/ash/services.py @@ -1,11 +1,7 @@ -import os -import socket - - from nxtools import logging, slugify from .config import config -from .containers import PODMAN, get_container_client +from .containers import get_container_client from .models import ServiceConfigModel from .service_logging import ServiceLogger @@ -113,7 +109,9 @@ def ensure_running( logging.error("SERVICE MISMATCH. This shouldn't happen. Stopping.") container.stop() else: - logging.debug(f"Service {service_name} already running at {container.id}") + logging.debug( + f"Service {service_name} already running at {container.id}" + ) break else: # And start it From 89ef1f45aa735210c968cd0ee861b8fa444053b1 Mon Sep 17 00:00:00 2001 From: Martin Wacker Date: Tue, 31 Oct 2023 13:17:09 +0100 Subject: [PATCH 5/9] make mypy happy --- ash/config.py | 1 + mypy.ini | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/ash/config.py b/ash/config.py index fa921c4..5113b7f 100644 --- a/ash/config.py +++ b/ash/config.py @@ -76,6 +76,7 @@ def get_config() -> Config: # we can't reach the backend at all. try: server_hostname = urlparse(val).hostname + assert server_hostname is not None, "Invalid URL" server_ip = socket.gethostbyname(server_hostname) val = val.replace(server_hostname, server_ip) except Exception: diff --git a/mypy.ini b/mypy.ini index 445cf68..553cf27 100644 --- a/mypy.ini +++ b/mypy.ini @@ -23,6 +23,11 @@ ignore_errors = true follow_imports = skip ignore_missing_imports = true +[mypy-podman.*] +ignore_errors = true +follow_imports = skip +ignore_missing_imports = true + [mypy-requests.*] ignore_errors = true follow_imports = skip From 80e126a16f22d6c1d874ba38f1077600b1b9a7a6 Mon Sep 17 00:00:00 2001 From: Oscar Domingo Date: Tue, 31 Oct 2023 14:35:16 +0000 Subject: [PATCH 6/9] Undo log output parsing --- ash/service_logging.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/ash/service_logging.py b/ash/service_logging.py index 6e8e7d4..4e1d58e 100644 --- a/ash/service_logging.py +++ b/ash/service_logging.py @@ -14,17 +14,7 @@ def _run(self): f"Starting log stream for {self.service_name}, last 10 lines were..." ) for line in self.container.logs(stream=True, tail=10, stderr=True): - log_string = line.decode().strip() - log_elements = log_string.split(" ") - - log_date = log_elements[0] - log_time = log_elements[1] - log_severity = log_elements[2] - - log_message = log_string.split(log_severity)[-1] - print( - f"{log_date} {log_time} {log_severity} {self.service_name} {log_message}" - ) + print(line.decode().strip()) # service exited # print the status code and free the container From f1b12940968897dc8fd7745b698ec12c6820aa1d Mon Sep 17 00:00:00 2001 From: Oscar Domingo Date: Tue, 31 Oct 2023 14:35:48 +0000 Subject: [PATCH 7/9] Fix accessing non existant variable --- ash/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ash/config.py b/ash/config.py index 5113b7f..db036ff 100644 --- a/ash/config.py +++ b/ash/config.py @@ -80,7 +80,7 @@ def get_config() -> Config: server_ip = socket.gethostbyname(server_hostname) val = val.replace(server_hostname, server_ip) except Exception: - critical_error("Unable to resolve `AYON_SERVER_URL` {original_value}") + critical_error(f"Unable to resolve `AYON_SERVER_URL` {val}") data[key.replace("ayon_", "", 1)] = val try: From 0739d4e22c0d1b001e03171c91a36db28855c88e Mon Sep 17 00:00:00 2001 From: Oscar Domingo Date: Tue, 31 Oct 2023 14:37:30 +0000 Subject: [PATCH 8/9] `services` Remove extra logs in main loop --- ash/services.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ash/services.py b/ash/services.py index d01dd9a..ff87af5 100644 --- a/ash/services.py +++ b/ash/services.py @@ -23,13 +23,10 @@ def get_running_services(cls) -> list[str]: if cls.client is None: return result - logging.debug("Checking for Running services.") - for container in cls.client.containers.list(): labels = container.labels if service_name := labels.get(f"{cls.prefix}.service_name"): result.append(service_name) - logging.debug("Found {0} running services: {1}".format(len(result), result)) return result @classmethod @@ -92,7 +89,6 @@ def ensure_running( # # Check whether it is running already # - logging.info("Checking if Service is already running.") container = None for container in cls.client.containers.list(): @@ -108,10 +104,6 @@ def ensure_running( except AssertionError: logging.error("SERVICE MISMATCH. This shouldn't happen. Stopping.") container.stop() - else: - logging.debug( - f"Service {service_name} already running at {container.id}" - ) break else: # And start it From 30416965e3e3ba3b26ffe5cf24e86c18e20c1a30 Mon Sep 17 00:00:00 2001 From: Martin Wacker Date: Wed, 1 Nov 2023 11:35:27 +0100 Subject: [PATCH 9/9] fix: incompatible directives network and network_mode --- ash/services.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ash/services.py b/ash/services.py index ff87af5..a96253d 100644 --- a/ash/services.py +++ b/ash/services.py @@ -57,13 +57,17 @@ def spawn( if cls.client is None: return + network_mode = None + if config.network_mode and (not config.network): + network_mode = config.network_mode + container = cls.client.containers.run( image, detach=True, auto_remove=True, environment=environment, hostname=hostname, - network_mode=config.network_mode, + network_mode=network_mode, network=config.network, name=hostname, labels=labels,