wip

CoDaBackup/backup_manager.py

@@ -36,11 +36,11 @@ class Backup:
         if isinstance(path, str):
             path = Path(path)
         self.path: Path = path
-        self.backup_name: str = path.stem
+        self.name: str = path.stem
         self.retention_type: RetentionType = RetentionType(path.parent.stem)
 
         self.database_name: str = path.parent.parent.stem
-        self.creation_time: float = self._parse_filename_for_timestamp(self.backup_name)
+        self.creation_time: float = self._parse_filename_for_timestamp(self.name)
 
     @property
     def age(self):
@@ -59,6 +59,15 @@ class Backup:
         timestamp_unix = time.mktime(timestamp_datetime.timetuple())
         return timestamp_unix
 
+    """
+    def to_dict(self):
+        return {
+            "name": self.name,
+            "path": str(self.path),
+            "type": str(self.retention_type),
+        }
+    """
+
 
 class BackupManager:
     # mapping of how many backups are kept per retention type

CoDaBackup/cli.py

@@ -24,6 +24,10 @@ from backupper import (
     Neo4jOfflineBackupper,
     Neo4jOnlineBackupper,
 )
+from cli_helper import (
+    format_make_list_of_backups_human_readable,
+    format_backup_list_human_readable,
+)
 from log import log
 
 from label import ValidLabels
@@ -150,6 +154,7 @@ def backup_now(
 @click.option(
     "--all-namespaces",
     "-all-namespaces",
+    is_flag=True,
     default=False,
     help="If we should search for pods, to be backed up, in all namespaces. When used `--namespace` will be ignored.",
 )
@@ -169,6 +174,9 @@ def backup_kubernetes(namespace, all_namespaces, target_dir):
         namespace=namespace, all_namespaces=all_namespaces
     ):
         if pod.backup_labels[ValidLabels.enabled].val:
+            print(pod)
+            print(pod.backup_labels["backup.dzd-ev.de/type"])
+            print(ValidLabels.database_type == "backup.dzd-ev.de/type")
             BackupperClass = database_type_backupper_mapping[
                 pod.backup_labels[ValidLabels.database_type].val
             ]
@@ -180,7 +188,7 @@ def backup_kubernetes(namespace, all_namespaces, target_dir):
                 password=pod.backup_labels[ValidLabels.database_password].val,
                 kubernetes_all_namespaces=all_namespaces,
                 kubernetes_namespace=pod.kubernetes_namespace,
-                backups_base_path=target_dir
+                backups_base_path=Path(target_dir)
                 if target_dir
                 else pod.backup_labels[ValidLabels.backup_dir].val,
             )
@@ -235,7 +243,7 @@ def backup_docker(target_dir):
                 host=container.backup_labels[ValidLabels.database_host].val,
                 user=container.backup_labels[ValidLabels.database_username].val,
                 password=container.backup_labels[ValidLabels.database_password].val,
-                backups_base_path=target_dir
+                backups_base_path=Path(target_dir)
                 if target_dir
                 else container.backup_labels[ValidLabels.backup_dir].val,
             )
@@ -356,6 +364,7 @@ def restore_now(
 @click.option(
     "--all-namespaces",
     "-all-namespaces",
+    is_flag=True,
     default=False,
     help="If we should search for pods, to be backed up, in all namespaces. When used `--namespace` will be ignored.",
 )
@@ -394,6 +403,7 @@ def restore_kubernetes(ctx, namespace, all_namespaces, pod_name, backup_name):
 @click.option(
     "--all-namespaces",
     "-all-namespaces",
+    is_flag=True,
     default=False,
     help="If we should search for pods, with backups, in all namespaces. When used `--namespace` will be ignored.",
 )
@@ -413,7 +423,24 @@ def restore_kubernetes_list(
     yaml,
     source_dir,
 ):
-    backup_list: Dict = {}
+
+    t = [
+        {
+            "namespace": "default",
+            "database_containers": [
+                {
+                    "name": "mariadb01",
+                    "databases": [
+                        {
+                            "name": "coda_test",
+                            "backups": [{"daily": [{"name": ""}], "weekly": []}],
+                        }
+                    ],
+                }
+            ],
+        }
+    ]
+    backup_list: List = {}
     for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
         namespace=namespace, all_namespaces=all_namespaces
     ):
@@ -431,7 +458,7 @@ def restore_kubernetes_list(
                 password=pod.backup_labels[ValidLabels.database_password].val,
                 kubernetes_all_namespaces=all_namespaces,
                 kubernetes_namespace=pod.kubernetes_namespace,
-                backups_base_path=source_dir
+                backups_base_path=Path(source_dir)
                 if source_dir
                 else pod.backup_labels[ValidLabels.backup_dir].val,
             )
@@ -443,11 +470,19 @@ def restore_kubernetes_list(
                 databases = bu.list_databases()
 
             for db in databases:
-                db_backups = bu.manager.list_backups(database_name=db)
+                db_backups = format_make_list_of_backups_human_readable(
+                    bu.manager.list_backups(database_name=db)
+                )
+                print(db_backups)
+                exit()
                 if db_backups:
-                    backup_list[f'{pod.parent["metadata"]["name"]}/{db}'] = db_backups
+                    if not pod.kubernetes_namespace in backup_list.keys():
+                        backup_list[pod.kubernetes_namespace] = {}
+                    backup_list[pod.kubernetes_namespace][
+                        f'{pod.parent["metadata"]["name"]}/{db}'
+                    ] = db_backups
     # make human readable or json/yaml
-    click.echo(backup_list)
+    print(backup_list)
 
 
 def restore_docker():

CoDaBackup/cli_helper.py

+from typing import Dict, List
+from datetime import datetime, timedelta
+import time
+import humanize
+import tabulate
+
+tabulate.PRESERVE_WHITESPACE = True
+from backup_manager import Backup, RetentionType
+
+
+def format_backup_list_human_readable(backup_list: Dict):
+    pass
+
+
+# 🖥️
+def format_make_list_of_backups_human_readable(
+    backups: Dict[RetentionType, List[Backup]]
+):
+    headers = ["", "Name", "Date", "Age", "Path"]
+    s = ""
+    for retenetion_type, backup_list in backups.items():
+        s = (
+            s
+            + f"\n\n\n  ⌚️ {retenetion_type}  {'@ ' + str(backup_list[0].path.parents[1].absolute()) if backup_list else ''}\n\n"
+        )
+        backups_table_struc = []
+        for backup in backup_list:
+
+            if not backup.retention_type == retenetion_type:
+                continue
+
+            backups_table_struc.append(
+                [
+                    "    💾",
+                    backup.name,
+                    datetime.utcfromtimestamp(backup.creation_time).strftime(
+                        "%Y-%m-%d %H:%M:%S"
+                    ),
+                    humanize.naturaldelta(
+                        timedelta(seconds=time.time() - backup.creation_time)
+                    ),
+                    backup.path.parents[0].name + "/" + backup.path.name,
+                ]
+            )
+
+        s = s + tabulate.tabulate(backups_table_struc, headers=headers)
+    return s

CoDaBackup/executer.py

@@ -31,10 +31,7 @@ class Executer:
         if self.mode.lower() == "docker":
             return config.DOCKER_COMMAND + " exec"
         elif self.mode.lower() == "kubernetes":
-            if self.all_namespaces:
-                self.namespace_arg = "--all-namespaces"
-            else:
-                self.namespace_arg = f"-n {self.namespace if self.namespace else config.KUBECTL_DEFAULT_NAMESPACE}"
+            self.namespace_arg = f"-n {self.namespace if self.namespace else config.KUBECTL_DEFAULT_NAMESPACE}"
             return f"{config.KUBECTL_COMMAND} exec {self.namespace_arg}"
         elif self.mode.lower() == "native":
             return ""

CoDaBackup/label.py

@@ -16,7 +16,7 @@ class Label:
     def __init__(
         self,
         label: Union[str, Dict[str, str]],
-        type: Type = str,
+        type_: Type = str,
         possible_values: List = None,
         default: Any = None,
         info: str = None,
@@ -29,20 +29,29 @@ class Label:
             key, *val = label.split("=")
         self.key: str = f"{base_label_key + '/' if base_label_key and not key.startswith(base_label_key) else ''}{key}"
         try:
-            if type == bool and isinstance(val, str):
+            if type_ == bool and isinstance(val, str):
                 val = util.strtobool(val)
-            self.val: Union[str, int, bool] = type(val) if val else default
+            self.val: Union[str, int, bool] = type_(val) if val else default
         except:
             raise ValueError(
-                f"Label '{key}={val}' expected to be of type '{type}'. Can not convert '{val}' to {type}"
+                f"Label '{key}={val}' expected to be of type '{type_}'. Can not convert '{val}' to {type_}"
             )
-        self.type: Type = type
+        if possible_values and self.val and self.val not in possible_values:
+            raise ValueError(
+                f"Label value of '{self.key}' expected to be one of {possible_values} but is '{self.val}' (value type: '{type(self.val)}')"
+            )
+        self.type: Type = type_
         self.possible_values: List = possible_values
         self.default: Union[str, int, bool] = default
         self.info: str = info
 
-    def __eq__(self, other: "Label"):
-        return type(other) is type(self) and self.key == other.key
+    def __eq__(self, other: Union["Label", str]):
+        return (
+            type(other) is type(self)
+            and self.key == other.key
+            or type(other) == str
+            and self.key == other
+        )
 
     def __hash__(self):
         return hash(self.key)
@@ -185,7 +194,7 @@ class ValidLabels:
     def label_from_valid_label_as_template(cls, valid_label: Label, val):
         return Label(
             label={valid_label.key: val},
-            type=valid_label.type,
+            type_=valid_label.type,
             possible_values=valid_label.possible_values,
             default=valid_label.default,
             info=valid_label.info,

DEV_NOTES.md

+# Create postgres testdata
+
+
+```bash
+docker exec mysql /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@localhost -c "CREATE DATABASE coda_ps_test;" && /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@localhost -c "\
+  BEGIN; \
+  create schema coda_ps_test_scheme; \
+  CREATE TABLE IF NOT EXISTS coda_ps_test_scheme.my_table(id integer PRIMARY KEY, firstname VARCHAR(32)); \
+  INSERT INTO coda_ps_test_scheme.my_table(id,firstname) VALUES (1,'Anna'); \
+  INSERT INTO coda_ps_test_scheme.my_table(id,firstname) VALUES (2,'Thomas'); \
+  commit;"
+```
\ No newline at end of file

reqs.txt

-DZDConfigs
-click
\ No newline at end of file

setup.py

@@ -14,7 +14,7 @@ setup(
     author_email="tim.bleimehl@helmholtz-muenchen.de",
     license="MIT",
     packages=["CoDaBackup"],
-    install_requires=["DZDConfigs", "Click"],
+    install_requires=["DZDConfigs", "Click", "tabulate", "humanize"],
     python_requires=">=3.8",
     zip_safe=False,
     include_package_data=True,