wip

CoDaBuddy/backup_modules/_base_backupper.py

@@ -149,13 +149,6 @@ class BaseBackupper:
         )
 
     def list_databases(self, show_all: bool = False) -> List[str]:
-        print("FU")
-        result = self.executer._run_docker_exec(
-            command="cat -n", stream_file_to_stdin="/tmp/myfile.txt"
-        )
-        print(result)
-        # "#############"
-        exit()
         result = self.executer.container_exec(command=self.get_list_database_command())
         return [
             database.strip()

CoDaBuddy/cli.py

@@ -338,7 +338,6 @@ def backup_docker(target_dir):
     for container in ContainerHelper.docker_get_container_to_be_backed_up(
         describe=True
     ):
-        print(container)
         if container.coda_labels[ValidLabels.enabled].val:
             BackupperClass = database_type_backupper_mapping[
                 container.coda_labels[ValidLabels.database_type].val

CoDaBuddy/container_helper.py

@@ -334,7 +334,6 @@ class ContainerHelper:
         containers: List[Container] = []
         for c in docker_containers:
             containers.append(Container.from_docker_container(c))
-        log.debug(f"Containers found: {containers}")
         return containers
 
     @classmethod

CoDaBuddy/executer.py

 import logging
 from typing import Callable, Union, List
+from unicodedata import decimal
 from Configs import getConfig
 from pathlib import Path
 from config import DEFAULT
@@ -15,6 +16,7 @@ from kubernetes.stream.ws_client import WSClient
 import time
 import tarfile
 import io
+import uuid
 
 # docker api
 import docker
@@ -67,7 +69,7 @@ class Executer:
         docker_client = docker.from_env()
         container: DockerContainer = docker_client.containers.get(self.container_name)
 
-        ##### METHOD 2 of "streaming"/"piping" a file into a container command. we cheat here:
+        ##### "streaming"/"piping" a file into a container command. we cheat here a little bit:
         # put_archive. Instead of streaming the file to our command we first create an tar-archive of the file and copy&save the tar-file into the container and stream the file from inside the container via a pipe bash in exec_run
         # helping links:
         # https://docker-py.readthedocs.io/en/stable/api.html#docker.api.container.ContainerApiMixin.put_archive
@@ -91,21 +93,15 @@ class Executer:
             )
             cmd = cmd + f" < {container_target_path}"
         if stream_result_to_file:
-            if compress_result_to_file:
-                f = gzip.open(stream_result_to_file, "wb")
-            else:
-                f = open(stream_result_to_file, "wb")
-            # ToDo: with https://docker-py.readthedocs.io/en/stable/api.html#docker.api.container.ContainerApiMixin.get_archive
-        print("cmd:", cmd)
+            internal_output_file_path = f"/tmp/output_codabuddy_{str(uuid.uuid4())[:8]}"
+            cmd = cmd + f" > {internal_output_file_path}"
+
+        # RUN the actual command
+        print(cmd)
         process = container.exec_run(
-            cmd=["/bin/sh", "-c", f'"{cmd}"'],
+            cmd=["/bin/sh", "-c", cmd],
             stream=True,
             demux=True,
-            # METHOD 1 (see further down for explaination)
-            # socket=True if stream_file_to_stdin else False,
-            # stdin=True if stream_file_to_stdin else False,
-            # tty=True if stream_file_to_stdin else False,
-            # METHOD 2
             socket=False,
             stdin=False,
             tty=False,
@@ -114,41 +110,38 @@ class Executer:
         stdout = b""
         stderr = b""
         for output in process.output:
-            if stream_result_to_file and output[0]:
-                f.write(output)
-            elif output[0]:
+            if output[0]:
+                log.debug(f"stdout: {stdout}")
                 stdout += output[0]
             if output[1]:
                 stderr += output[1]
-        if stream_result_to_file:
-            f.close
         if stderr:
             raise Exception(
-                f"ERROR during container ({self.container_name}) exec:\nCommand: '{command}'\nError: '{stderr.decode('utf-8')}'"
+                f"ERROR during container ({self.container_name}) exec:\nCommand: '{cmd}'\nError: '{stderr.decode('utf-8')}'"
             )
+        ##### "streaming"/"piping" the result out of the container command. we cheat here a little bit:
+        # instead of streaming the file we created a file in the container itself and output it via get_archive
+        # https://docker-py.readthedocs.io/en/stable/api.html#docker.api.container.ContainerApiMixin.get_archive
+        # this is less error prone, but a real streaming solution would be more faster and less resource intensive.
+        # but for now we go with this method
+        if stream_result_to_file:
+            if compress_result_to_file:
+                f = gzip.open(stream_result_to_file, "wb")
+            else:
+                f = open(stream_result_to_file, "wb")
+            stream, meta = container.get_archive(path=internal_output_file_path)
+            for chunk in stream:
+                f.write(chunk)
+            # tidy up the internal temp result file
+            rm_res = container.exec_run(["rm", internal_output_file_path], demux=True)
 
-        elif decode_byte_output:
-            print(stdout)
-            return stdout.decode("utf-8").rstrip("\n")
-        else:
-            print(stdout)
-            return stdout
-        exit()
-        stdout = b""
-        stderr = b""
-        for output in process.output:
-            stdout += output[0]
-            if stdout:
-                log.debug(f"{self.container_name} - {stdout}")
-            stderr += output[1]
-            if stderr:
-                log.debug(f"{self.container_name} - ERROR - {stderr}")
-        if stderr:
-            raise Exception(
-                f"ERROR during container ({self.container_name}) exec:\nCommand: '{command}'\nError: '{stderr}'"
-            )
-        else:
-            return stdout
+            if rm_res.exit_code != 0:
+                # there was an error while tiding up the tmp internal result file
+                raise Exception(
+                    f"ERROR during removing internal temp result file ({internal_output_file_path}) in container '{self.container_name}' Error:\n'{rm_res.output[1].decode('utf-8')}'"
+                )
+        # done
+        return stdout.decode("utf-8").rstrip("\n") if decode_byte_output else stdout
 
     def _run_kubernetes_exec(
         self,