refactor: do not run the requirements installation if the virtualenv already exists

Signed-off-by: Dominik Willner <th33xitus@gmail.com>

kiauh/components/klipper/klipper_setup.py

@@ -154,8 +154,8 @@ def setup_klipper_prerequesites() -> None:
     # install klipper dependencies and create python virtualenv
     try:
         install_klipper_packages()
-        create_python_venv(KLIPPER_ENV_DIR)
+        if create_python_venv(KLIPPER_ENV_DIR):
-        install_python_requirements(KLIPPER_ENV_DIR, KLIPPER_REQ_FILE)
+            install_python_requirements(KLIPPER_ENV_DIR, KLIPPER_REQ_FILE)
     except Exception:
         Logger.print_error("Error during installation of Klipper requirements!")
         raise

kiauh/components/moonraker/moonraker_setup.py

@@ -150,9 +150,9 @@ def setup_moonraker_prerequesites() -> None:
 
     # install moonraker dependencies and create python virtualenv
     install_moonraker_packages()
-    create_python_venv(MOONRAKER_ENV_DIR)
+    if create_python_venv(MOONRAKER_ENV_DIR):
-    install_python_requirements(MOONRAKER_ENV_DIR, MOONRAKER_REQ_FILE)
+        install_python_requirements(MOONRAKER_ENV_DIR, MOONRAKER_REQ_FILE)
-    install_python_requirements(MOONRAKER_ENV_DIR, MOONRAKER_SPEEDUPS_REQ_FILE)
+        install_python_requirements(MOONRAKER_ENV_DIR, MOONRAKER_SPEEDUPS_REQ_FILE)
 
 
 def install_moonraker_packages() -> None:

kiauh/components/webui_client/client_setup.py

@@ -28,8 +28,14 @@ from components.webui_client.client_dialogs import (
     print_moonraker_not_found_dialog,
 )
 from components.webui_client.client_utils import (
+    copy_common_vars_nginx_cfg,
+    copy_upstream_nginx_cfg,
+    create_nginx_cfg,
     detect_client_cfg_conflict,
     enable_mainsail_remotemode,
+    get_next_free_port,
+    is_valid_port,
+    read_ports_from_nginx_configs,
     symlink_webui_nginx_log,
 )
 from core.instance_manager.instance_manager import InstanceManager
@@ -37,15 +43,7 @@ from core.logger import Logger
 from core.settings.kiauh_settings import KiauhSettings
 from utils.common import check_install_dependencies
 from utils.config_utils import add_config_section
-from utils.fs_utils import (
+from utils.fs_utils import unzip
-    copy_common_vars_nginx_cfg,
-    copy_upstream_nginx_cfg,
-    create_nginx_cfg,
-    get_next_free_port,
-    is_valid_port,
-    read_ports_from_nginx_configs,
-    unzip,
-)
 from utils.input_utils import get_confirm, get_number_input
 from utils.sys_utils import (
     cmd_sysctl_service,
@@ -144,7 +142,7 @@ def install_client(client: BaseWebClient) -> None:
         )
 
         if kl_instances:
-            symlink_webui_nginx_log(kl_instances)
+            symlink_webui_nginx_log(client, kl_instances)
         cmd_sysctl_service("nginx", "restart")
 
     except Exception as e:

kiauh/components/webui_client/client_utils.py

@@ -9,10 +9,14 @@
 from __future__ import annotations
 
 import json
+import re
+import shutil
 from pathlib import Path
+from subprocess import PIPE, CalledProcessError, run
 from typing import List, get_args
 
 from components.klipper.klipper import Klipper
+from components.webui_client import MODULE_PATH
 from components.webui_client.base_data import (
     BaseWebClient,
     WebClientType,
@@ -25,12 +29,14 @@ from core.constants import (
     COLOR_YELLOW,
     NGINX_CONFD,
     NGINX_SITES_AVAILABLE,
+    NGINX_SITES_ENABLED,
     RESET_FORMAT,
 )
 from core.logger import Logger
 from core.settings.kiauh_settings import KiauhSettings
 from core.types import ComponentStatus
 from utils.common import get_install_status
+from utils.fs_utils import create_symlink, remove_file
 from utils.git_utils import (
     get_latest_tag,
     get_latest_unstable_tag,
@@ -95,17 +101,19 @@ def enable_mainsail_remotemode() -> None:
     Logger.print_ok("Mainsails remote mode enabled!")
 
 
-def symlink_webui_nginx_log(klipper_instances: List[Klipper]) -> None:
+def symlink_webui_nginx_log(
+    client: BaseWebClient, klipper_instances: List[Klipper]
+) -> None:
     Logger.print_status("Link NGINX logs into log directory ...")
-    access_log = Path("/var/log/nginx/mainsail-access.log")
+    access_log = client.nginx_access_log
-    error_log = Path("/var/log/nginx/mainsail-error.log")
+    error_log = client.nginx_error_log
 
     for instance in klipper_instances:
-        desti_access = instance.log_dir.joinpath("mainsail-access.log")
+        desti_access = instance.log_dir.joinpath(access_log.name)
         if not desti_access.exists():
             desti_access.symlink_to(access_log)
 
-        desti_error = instance.log_dir.joinpath("mainsail-error.log")
+        desti_error = instance.log_dir.joinpath(error_log.name)
         if not desti_error.exists():
             desti_error.symlink_to(error_log)
 
@@ -146,9 +154,7 @@ def backup_client_data(client: BaseWebClient) -> None:
 
     bm = BackupManager()
     bm.backup_directory(f"{name}-{version}", src, dest)
-    if name == "mainsail":
+    bm.backup_file(client.config_file, dest)
-        c_json = MainsailData().client_dir.joinpath("config.json")
-        bm.backup_file(c_json, dest)
     bm.backup_file(NGINX_SITES_AVAILABLE.joinpath(name), dest)
 
 
@@ -206,3 +212,132 @@ def get_download_url(base_url: str, client: BaseWebClient) -> str:
         return f"{base_url}/download/{unstable_tag}/{client.name}.zip"
     except Exception:
         return stable_url
+
+
+#################################################
+## NGINX RELATED FUNCTIONS
+#################################################
+
+
+def copy_upstream_nginx_cfg() -> None:
+    """
+    Creates an upstream.conf in /etc/nginx/conf.d
+    :return: None
+    """
+    source = MODULE_PATH.joinpath("assets/upstreams.conf")
+    target = NGINX_CONFD.joinpath("upstreams.conf")
+    try:
+        command = ["sudo", "cp", source, target]
+        run(command, stderr=PIPE, check=True)
+    except CalledProcessError as e:
+        log = f"Unable to create upstreams.conf: {e.stderr.decode()}"
+        Logger.print_error(log)
+        raise
+
+
+def copy_common_vars_nginx_cfg() -> None:
+    """
+    Creates a common_vars.conf in /etc/nginx/conf.d
+    :return: None
+    """
+    source = MODULE_PATH.joinpath("assets/common_vars.conf")
+    target = NGINX_CONFD.joinpath("common_vars.conf")
+    try:
+        command = ["sudo", "cp", source, target]
+        run(command, stderr=PIPE, check=True)
+    except CalledProcessError as e:
+        log = f"Unable to create upstreams.conf: {e.stderr.decode()}"
+        Logger.print_error(log)
+        raise
+
+
+def generate_nginx_cfg_from_template(name: str, template_src: Path, **kwargs) -> None:
+    """
+    Creates an NGINX config from a template file and
+    replaces all placeholders passed as kwargs. A placeholder must be defined
+    in the template file as %{placeholder}%.
+    :param name: name of the config to create
+    :param template_src: the path to the template file
+    :return: None
+    """
+    tmp = Path.home().joinpath(f"{name}.tmp")
+    shutil.copy(template_src, tmp)
+    with open(tmp, "r+") as f:
+        content = f.read()
+
+        for key, value in kwargs.items():
+            content = content.replace(f"%{key}%", str(value))
+
+        f.seek(0)
+        f.write(content)
+        f.truncate()
+
+    target = NGINX_SITES_AVAILABLE.joinpath(name)
+    try:
+        command = ["sudo", "mv", tmp, target]
+        run(command, stderr=PIPE, check=True)
+    except CalledProcessError as e:
+        log = f"Unable to create '{target}': {e.stderr.decode()}"
+        Logger.print_error(log)
+        raise
+
+
+def create_nginx_cfg(
+    display_name: str,
+    cfg_name: str,
+    template_src: Path,
+    **kwargs,
+) -> None:
+    from utils.sys_utils import set_nginx_permissions
+
+    try:
+        Logger.print_status(f"Creating NGINX config for {display_name} ...")
+
+        source = NGINX_SITES_AVAILABLE.joinpath(cfg_name)
+        target = NGINX_SITES_ENABLED.joinpath(cfg_name)
+        remove_file(Path("/etc/nginx/sites-enabled/default"), True)
+        generate_nginx_cfg_from_template(cfg_name, template_src=template_src, **kwargs)
+        create_symlink(source, target, True)
+        set_nginx_permissions()
+
+        Logger.print_ok(f"NGINX config for {display_name} successfully created.")
+    except Exception:
+        Logger.print_error(f"Creating NGINX config for {display_name} failed!")
+        raise
+
+
+def read_ports_from_nginx_configs() -> List[int]:
+    """
+    Helper function to iterate over all NGINX configs and read all ports defined for listen
+    :return: A sorted list of listen ports
+    """
+    if not NGINX_SITES_ENABLED.exists():
+        return []
+
+    port_list = []
+    for config in NGINX_SITES_ENABLED.iterdir():
+        if not config.is_file():
+            continue
+
+        with open(config, "r") as cfg:
+            lines = cfg.readlines()
+
+        for line in lines:
+            line = line.replace("default_server", "")
+            line = re.sub(r"[;:\[\]]", "", line.strip())
+            if line.startswith("listen") and line.split()[-1] not in port_list:
+                port_list.append(line.split()[-1])
+
+    ports_to_ints_list = [int(port) for port in port_list]
+    return sorted(ports_to_ints_list, key=lambda x: int(x))
+
+
+def is_valid_port(port: int, ports_in_use: List[int]) -> bool:
+    return port not in ports_in_use
+
+
+def get_next_free_port(ports_in_use: List[int]) -> int:
+    valid_ports = set(range(80, 7125))
+    used_ports = set(map(int, ports_in_use))
+
+    return min(valid_ports - used_ports)

kiauh/extensions/obico/__init__.py

@@ -31,3 +31,4 @@ OBICO_ENV_DIR = Path.home().joinpath("moonraker-obico-env")
 OBICO_SERVICE_TEMPLATE = MODULE_PATH.joinpath(f"assets/{OBICO_SERVICE_NAME}")
 OBICO_ENV_FILE_TEMPLATE = MODULE_PATH.joinpath(f"assets/{OBICO_ENV_FILE_NAME}")
 OBICO_LINK_SCRIPT = OBICO_DIR.joinpath("scripts/link.sh")
+OBICO_REQ_FILE = OBICO_DIR.joinpath("requirements.txt")

kiauh/extensions/obico/moonraker_obico_extension.py

@@ -23,6 +23,7 @@ from extensions.obico import (
     OBICO_ENV_DIR,
     OBICO_MACROS_CFG_NAME,
     OBICO_REPO,
+    OBICO_REQ_FILE,
     OBICO_UPDATE_CFG_NAME,
     OBICO_UPDATE_CFG_SAMPLE_NAME,
 )
@@ -239,9 +240,8 @@ class ObicoExtension(BaseExtension):
         check_install_dependencies({*package_list})
 
         # create virtualenv
-        create_python_venv(OBICO_ENV_DIR)
+        if create_python_venv(OBICO_ENV_DIR):
-        requirements = OBICO_DIR.joinpath("requirements.txt")
+            install_python_requirements(OBICO_ENV_DIR, OBICO_REQ_FILE)
-        install_python_requirements(OBICO_ENV_DIR, requirements)
 
     def _create_obico_macros_cfg(self, moonraker) -> None:
         macros_cfg = OBICO_DIR.joinpath(f"include_cfgs/{OBICO_MACROS_CFG_NAME}")

kiauh/extensions/telegram_bot/__init__.py

@@ -26,3 +26,4 @@ TG_BOT_ENV = Path.home().joinpath("moonraker-telegram-bot-env")
 # files
 TG_BOT_SERVICE_TEMPLATE = MODULE_PATH.joinpath(f"assets/{TG_BOT_SERVICE_NAME}")
 TG_BOT_ENV_FILE_TEMPLATE = MODULE_PATH.joinpath(f"assets/{TG_BOT_ENV_FILE_NAME}")
+TG_BOT_REQ_FILE = TG_BOT_DIR.joinpath("scripts/requirements.txt")

kiauh/extensions/telegram_bot/moonraker_telegram_bot_extension.py

@@ -14,7 +14,7 @@ from components.moonraker.moonraker import Moonraker
 from core.instance_manager.instance_manager import InstanceManager
 from core.logger import DialogType, Logger
 from extensions.base_extension import BaseExtension
-from extensions.telegram_bot import TG_BOT_REPO
+from extensions.telegram_bot import TG_BOT_REPO, TG_BOT_REQ_FILE
 from extensions.telegram_bot.moonraker_telegram_bot import (
     TG_BOT_DIR,
     TG_BOT_ENV,
@@ -161,9 +161,8 @@ class TelegramBotExtension(BaseExtension):
         check_install_dependencies({*package_list})
 
         # create virtualenv
-        create_python_venv(TG_BOT_ENV)
+        if create_python_venv(TG_BOT_ENV):
-        requirements = TG_BOT_DIR.joinpath("scripts/requirements.txt")
+            install_python_requirements(TG_BOT_ENV, TG_BOT_REQ_FILE)
-        install_python_requirements(TG_BOT_ENV, requirements)
 
     def _patch_bot_update_manager(self, instances: List[Moonraker]) -> None:
         env_py = f"{TG_BOT_ENV}/bin/python"

kiauh/utils/fs_utils.py

@@ -9,21 +9,13 @@
 #  This file may be distributed under the terms of the GNU GPLv3 license  #
 # ======================================================================= #
 
-import re
 import shutil
 from pathlib import Path
 from subprocess import DEVNULL, PIPE, CalledProcessError, check_output, run
-from typing import List
 from zipfile import ZipFile
 
-from core.constants import (
-    NGINX_CONFD,
-    NGINX_SITES_AVAILABLE,
-    NGINX_SITES_ENABLED,
-)
 from core.decorators import deprecated
 from core.logger import Logger
-from utils import MODULE_PATH
 
 
 def check_file_exist(file_path: Path, sudo=False) -> bool:
@@ -80,23 +72,23 @@ def remove_file(file_path: Path, sudo=False) -> None:
 
 def run_remove_routines(file: Path) -> None:
     try:
-        if not file.exists():
+        if not file.is_symlink() and not file.exists():
             Logger.print_info(f"File '{file}' does not exist. Skipped ...")
             return
 
         if file.is_dir():
             shutil.rmtree(file)
-        elif file.is_file():
+        elif file.is_file() or file.is_symlink():
             file.unlink()
         else:
             raise OSError(f"File '{file}' is neither a file nor a directory!")
-        Logger.print_ok("Successfully removed!")
+        Logger.print_ok(f"File '{file}' was successfully removed!")
     except OSError as e:
         Logger.print_error(f"Unable to delete '{file}':\n{e}")
         try:
             Logger.print_info("Trying to remove with sudo ...")
             remove_with_sudo(file)
-            Logger.print_ok("Successfully removed!")
+            Logger.print_ok(f"File '{file}' was successfully removed!")
         except CalledProcessError as e:
             Logger.print_error(f"Error deleting '{file}' with sudo:\n{e}")
             Logger.print_error("Remove this directory manually!")
@@ -111,127 +103,3 @@ def unzip(filepath: Path, target_dir: Path) -> None:
     """
     with ZipFile(filepath, "r") as _zip:
         _zip.extractall(target_dir)
-
-
-def copy_upstream_nginx_cfg() -> None:
-    """
-    Creates an upstream.conf in /etc/nginx/conf.d
-    :return: None
-    """
-    source = MODULE_PATH.joinpath("assets/upstreams.conf")
-    target = NGINX_CONFD.joinpath("upstreams.conf")
-    try:
-        command = ["sudo", "cp", source, target]
-        run(command, stderr=PIPE, check=True)
-    except CalledProcessError as e:
-        log = f"Unable to create upstreams.conf: {e.stderr.decode()}"
-        Logger.print_error(log)
-        raise
-
-
-def copy_common_vars_nginx_cfg() -> None:
-    """
-    Creates a common_vars.conf in /etc/nginx/conf.d
-    :return: None
-    """
-    source = MODULE_PATH.joinpath("assets/common_vars.conf")
-    target = NGINX_CONFD.joinpath("common_vars.conf")
-    try:
-        command = ["sudo", "cp", source, target]
-        run(command, stderr=PIPE, check=True)
-    except CalledProcessError as e:
-        log = f"Unable to create upstreams.conf: {e.stderr.decode()}"
-        Logger.print_error(log)
-        raise
-
-
-def generate_nginx_cfg_from_template(name: str, template_src: Path, **kwargs) -> None:
-    """
-    Creates an NGINX config from a template file and
-    replaces all placeholders passed as kwargs. A placeholder must be defined
-    in the template file as %{placeholder}%.
-    :param name: name of the config to create
-    :param template_src: the path to the template file
-    :return: None
-    """
-    tmp = Path.home().joinpath(f"{name}.tmp")
-    shutil.copy(template_src, tmp)
-    with open(tmp, "r+") as f:
-        content = f.read()
-
-        for key, value in kwargs.items():
-            content = content.replace(f"%{key}%", str(value))
-
-        f.seek(0)
-        f.write(content)
-        f.truncate()
-
-    target = NGINX_SITES_AVAILABLE.joinpath(name)
-    try:
-        command = ["sudo", "mv", tmp, target]
-        run(command, stderr=PIPE, check=True)
-    except CalledProcessError as e:
-        log = f"Unable to create '{target}': {e.stderr.decode()}"
-        Logger.print_error(log)
-        raise
-
-
-def create_nginx_cfg(
-    display_name: str,
-    cfg_name: str,
-    template_src: Path,
-    **kwargs,
-) -> None:
-    from utils.sys_utils import set_nginx_permissions
-
-    try:
-        Logger.print_status(f"Creating NGINX config for {display_name} ...")
-
-        source = NGINX_SITES_AVAILABLE.joinpath(cfg_name)
-        target = NGINX_SITES_ENABLED.joinpath(cfg_name)
-        remove_file(Path("/etc/nginx/sites-enabled/default"), True)
-        generate_nginx_cfg_from_template(cfg_name, template_src=template_src, **kwargs)
-        create_symlink(source, target, True)
-        set_nginx_permissions()
-
-        Logger.print_ok(f"NGINX config for {display_name} successfully created.")
-    except Exception:
-        Logger.print_error(f"Creating NGINX config for {display_name} failed!")
-        raise
-
-
-def read_ports_from_nginx_configs() -> List[int]:
-    """
-    Helper function to iterate over all NGINX configs and read all ports defined for listen
-    :return: A sorted list of listen ports
-    """
-    if not NGINX_SITES_ENABLED.exists():
-        return []
-
-    port_list = []
-    for config in NGINX_SITES_ENABLED.iterdir():
-        if not config.is_file():
-            continue
-
-        with open(config, "r") as cfg:
-            lines = cfg.readlines()
-
-        for line in lines:
-            line = line.replace("default_server", "")
-            line = re.sub(r"[;:\[\]]", "", line.strip())
-            if line.startswith("listen") and line.split()[-1] not in port_list:
-                port_list.append(line.split()[-1])
-
-    ports_to_ints_list = [int(port) for port in port_list]
-    return sorted(ports_to_ints_list, key=lambda x: int(x))
-
-
-def is_valid_port(port: int, ports_in_use: List[int]) -> bool:
-    return port not in ports_in_use
-
-
-def get_next_free_port(ports_in_use: List[int]) -> int:
-    valid_ports = set(range(80, 7125))
-    used_ports = set(map(int, ports_in_use))
-
-    return min(valid_ports - used_ports)

kiauh/utils/git_utils.py

@@ -165,7 +165,8 @@ def git_cmd_clone(repo: str, target_dir: Path) -> None:
 
         Logger.print_ok("Clone successful!")
     except CalledProcessError as e:
-        log = f"Error cloning repository {repo}: {e.stderr.decode()}"
+        error = e.stderr.decode() if e.stderr else "Unknown error"
+        log = f"Error cloning repository {repo}: {error}"
         Logger.print_error(log)
         raise
 

kiauh/utils/sys_utils.py

@@ -87,11 +87,13 @@ def parse_packages_from_file(source_file: Path) -> List[str]:
     return packages
 
 
-def create_python_venv(target: Path) -> None:
+def create_python_venv(target: Path) -> bool:
     """
-    Create a python 3 virtualenv at the provided target destination |
+    Create a python 3 virtualenv at the provided target destination.
+    Returns True if the virtualenv was created successfully.
+    Returns False if the virtualenv already exists, recreation was declined or creation failed.
     :param target: Path where to create the virtualenv at
-    :return: None
+    :return: bool
     """
     Logger.print_status("Set up Python virtual environment ...")
     if not target.exists():
@@ -99,20 +101,25 @@ def create_python_venv(target: Path) -> None:
             cmd = ["virtualenv", "-p", "/usr/bin/python3", target.as_posix()]
             run(cmd, check=True)
             Logger.print_ok("Setup of virtualenv successful!")
+            return True
         except CalledProcessError as e:
             Logger.print_error(f"Error setting up virtualenv:\n{e}")
-            raise
+            return False
     else:
-        if get_confirm("Virtualenv already exists. Re-create?", default_choice=False):
+        if not get_confirm(
-            try:
+            "Virtualenv already exists. Re-create?", default_choice=False
-                shutil.rmtree(target)
+        ):
-                create_python_venv(target)
-            except OSError as e:
-                log = f"Error removing existing virtualenv: {e.strerror}"
-                Logger.print_error(log, False)
-                raise
-        else:
             Logger.print_info("Skipping re-creation of virtualenv ...")
+            return False
+
+        try:
+            shutil.rmtree(target)
+            create_python_venv(target)
+            return True
+        except OSError as e:
+            log = f"Error removing existing virtualenv: {e.strerror}"
+            Logger.print_error(log, False)
+            return False
 
 
 def update_python_pip(target: Path) -> None: