rc1 of cli.py restore kubernetes list

CoDaBackup/backup_manager.py

-from typing import List, Dict, Union
+from typing import List, Dict, Union, Callable
 import logging
 from datetime import datetime
 import time
@@ -8,7 +8,7 @@ from enum import Enum
 from pathlib import Path, PurePath
 from Configs import getConfig
 import operator
-
+import json
 from config import DEFAULT
 
 config: DEFAULT = getConfig(config_classes=[DEFAULT])
@@ -59,6 +59,19 @@ class Backup:
         timestamp_unix = time.mktime(timestamp_datetime.timetuple())
         return timestamp_unix
 
+    def to_dict(self, meta_data: bool = False) -> Dict:
+        data = {
+            "name": self.name,
+            "creation_time": self.creation_time,
+            "path": self.path,
+        }
+        if meta_data:
+            data = data | {
+                "retention_type": self.retention_type,
+                "database_name": self.database_name,
+            }
+        return data
+
     """
     def to_dict(self):
         return {
@@ -123,6 +136,10 @@ class BackupManager:
         self.backup_file_suffix = backup_file_suffix
         # self.create_pathes_if_not_exists()
 
+    def list_backuped_databases(self) -> List[str]:
+        """List database names based on available backups and not on available online databases"""
+        return [dir.name for dir in self.base_path.iterdir() if dir.is_dir()]
+
     def list_backups(
         self,
         database_name: str,

CoDaBackup/cli.py

 #!/usr/bin/env python3
 
-from email.policy import default
-from pydoc import describe
+# from email.policy import default
+# from pydoc import describe
 import click
 import os
 import sys
 from typing import Dict, List, Type
 from pathlib import Path, PurePath
+import json
+from collections import OrderedDict
 
 if __name__ == "__main__":
     SCRIPT_DIR = os.path.dirname(
@@ -26,8 +28,8 @@ from backupper import (
     Neo4jOnlineBackupper,
 )
 from cli_helper import (
-    format_make_list_of_backups_human_readable,
-    format_backup_list_human_readable,
+    backup_list_to_human,
+    backup_list_to_machine_readable,
 )
 from log import log
 
@@ -380,13 +382,16 @@ def restore_kubernetes(ctx, namespace, all_namespaces, pod_name, backup_name):
 
 @restore_kubernetes.command(name="list")
 @click.option(
-    "--pod-name",
-    "-p",
+    "--workload-name",
+    "-w",
     default=None,
     help="User to access the database?",
 )
 @click.option(
-    "--databases", default=None, help="List only these databases (seperated by comma)"
+    "--databases",
+    "-d",
+    default=None,
+    help="List only these databases (seperated by comma)",
 )
 @click.option(
     "--namespace",
@@ -409,59 +414,84 @@ def restore_kubernetes(ctx, namespace, all_namespaces, pod_name, backup_name):
     default=None,
     help=f"Where to look for the backups. Will default to '{ValidLabels.backup_dir.val}'",
 )
-@click.option("--yaml", default=False, help="Output list machine readable")
+@click.option(
+    "--output-format",
+    "-o",
+    default="human",
+    help="Define the output format: 'human' an easy to read table, 'json' or 'yaml' for inter-process readability, dict if used in a python script",
+    type=click.Choice(["human", "json", "yaml", "dict"], case_sensitive=False),
+)
 def restore_kubernetes_list(
-    pod_name,
-    databases,
-    namespace,
-    all_namespaces,
-    yaml,
-    source_dir,
+    workload_name, databases, namespace, all_namespaces, source_dir, output_format
 ):
+    """Display available backup files"""
 
+    # ToDo: This func is messy :) we sure can improve this.
     backup_list: List = {}
+    data = []
     for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
         namespace=namespace, all_namespaces=all_namespaces
     ):
-        if pod_name and pod_name != pod.name:
+        if databases:
+            databases = databases.split(",")
+        if workload_name and pod.parent["metadata"]["name"] != workload_name:
             continue
+        pod_workload_name = pod.parent["metadata"]["name"]
         if pod.coda_labels[ValidLabels.enabled].val:
             BackupperClass = database_type_backupper_mapping[
                 pod.coda_labels[ValidLabels.database_type].val
             ]
-            bu = BackupperClass(
-                mode="kubernetes",
-                container_name=pod.name,
-                host=pod.coda_labels[ValidLabels.database_host].val,
-                user=pod.coda_labels[ValidLabels.database_username].val,
-                password=pod.coda_labels[ValidLabels.database_password].val,
-                kubernetes_all_namespaces=all_namespaces,
-                kubernetes_namespace=pod.kubernetes_namespace,
-                backups_base_path=Path(source_dir)
-                if source_dir
-                else pod.coda_labels[ValidLabels.backup_dir].val,
-            )
-            if databases:
-                databases = databases.split(",")
-            elif pod.coda_labels[ValidLabels.database_names].val:
-                databases = pod.coda_labels[ValidLabels.database_names].val.split(",")
-            else:
-                databases = bu.list_databases()
-
-            for db in databases:
-                db_backups = format_make_list_of_backups_human_readable(
-                    bu.manager.list_backups(database_name=db)
+            bu = BackupperClass(pod)
+            for database_name in bu.manager.list_backuped_databases():
+                if databases and not database_name in databases:
+                    continue
+                namespace = None
+                for ns in data:
+                    if (
+                        "namespace" in ns
+                        and ns["namespace"] == pod.kubernetes_namespace
+                    ):
+                        namespace = ns
+                if namespace is None:
+                    namespace = {
+                        "namespace": pod.kubernetes_namespace,
+                        "mode": "kubernetes",
+                        "database_containers": [],
+                    }
+                    data.append(namespace)
+                pod_entry = next(
+                    (
+                        p
+                        for p in namespace["database_containers"]
+                        if p["name"] == pod_workload_name
+                    ),
+                    None,
+                )
+                if not pod_entry:
+                    pod_entry = {
+                        "name": pod_workload_name,
+                        "basedir": bu.manager.base_path,
+                        "databases": [],
+                    }
+                    namespace["database_containers"].append(pod_entry)
+
+                pod_entry["databases"].append(
+                    {
+                        "name": database_name,
+                        "basedir": Path(PurePath(bu.manager.base_path, database_name)),
+                        "backups": bu.manager.list_backups(database_name),
+                    }
                 )
-                print(db_backups)
-                exit()
-                if db_backups:
-                    if not pod.kubernetes_namespace in backup_list.keys():
-                        backup_list[pod.kubernetes_namespace] = {}
-                    backup_list[pod.kubernetes_namespace][
-                        f'{pod.parent["metadata"]["name"]}/{db}'
-                    ] = db_backups
-    # make human readable or json/yaml
-    print(backup_list)
+    if output_format in ["json", "yaml"]:
+        output = backup_list_to_machine_readable(data, output_format)
+        print(output)
+        return output
+    elif output_format == "human":
+        output = backup_list_to_human(data)
+        print(output)
+        return output
+    elif output_format == "dict":
+        return data
 
 
 def restore_docker():

CoDaBackup/cli_helper.py

-from typing import Dict, List
+from ast import Call
+from typing import Dict, List, Union, Type, Callable
 from datetime import datetime, timedelta
 import time
 import humanize
+from numpy import sort
+from pathlib import Path
 import tabulate
 
 tabulate.PRESERVE_WHITESPACE = True
 from backup_manager import Backup, RetentionType
+import json
+import yaml
 
 
-def format_backup_list_human_readable(backup_list: Dict):
+def backup_list_to_human(backup_list: Dict):
     """[summary]
 
     Args:
@@ -40,6 +45,9 @@ def format_backup_list_human_readable(backup_list: Dict):
                                         }
                                     ],
                                     "weekly": [],
+                                    "monthly": [],
+                                    "yearly": [],
+                                    "manual": [],
                                 }
                             ],
                         }
@@ -50,18 +58,29 @@ def format_backup_list_human_readable(backup_list: Dict):
     ]
     ```
     """
+    data = ""
+    for namespace in backup_list:
+        if len(backup_list) > 1:
+            # we not only have the default namespace. otherwise we could hide it
+            data = data + f"\n\n🌐 NAMESPACE: {namespace['namespace']}\n\n"
+        for container in namespace["database_containers"]:
+            data = data + f" 🖥️  Container: {container['name']}\n\n"
+            for database in container["databases"]:
+                data = data + f"  🛢  Database: {database['name']}\n"
+                data = data + backup_file_list_to_human(database["backups"], 6)
+    return data
 
 
 # 🖥️
-def format_make_list_of_backups_human_readable(
-    backups: Dict[RetentionType, List[Backup]]
+def backup_file_list_to_human(
+    backups: Dict[RetentionType, List[Backup]], indent: int = 4
 ):
     headers = ["", "Name", "Date", "Age", "Path"]
     s = ""
     for retenetion_type, backup_list in backups.items():
         s = (
             s
-            + f"\n\n\n  ⌚️ {retenetion_type}  {'@ ' + str(backup_list[0].path.parents[1].absolute()) if backup_list else ''}\n\n"
+            + f"\n    ⌚️ {retenetion_type}  {'@ ' + str(backup_list[0].path.parents[1].absolute()) if backup_list else ''}\n"
         )
         backups_table_struc = []
         for backup in backup_list:
@@ -71,7 +90,7 @@ def format_make_list_of_backups_human_readable(
 
             backups_table_struc.append(
                 [
-                    "    💾",
+                    "💾",
                     backup.name,
                     datetime.utcfromtimestamp(backup.creation_time).strftime(
                         "%Y-%m-%d %H:%M:%S"
@@ -82,6 +101,61 @@ def format_make_list_of_backups_human_readable(
                     backup.path.parents[0].name + "/" + backup.path.name,
                 ]
             )
-
-        s = s + tabulate.tabulate(backups_table_struc, headers=headers)
+        table = tabulate.tabulate(backups_table_struc, headers=headers)
+        indented_table = ""
+        for line in table.split("\n"):
+            indented_table = indented_table + " " * indent + line + "\n"
+        s = s + indented_table
     return s
+
+
+def backup_list_to_machine_readable(backup_list: List, format: str = "json"):
+    """Serialize all dict entries of a backup_list"""
+    if format != "json" and format != "yaml":
+        raise ValueError(
+            f"Unknown format style. Expected 'json' or 'yaml' got '{format}'"
+        )
+    elif format == "json":
+        dump_func = json.dumps
+    elif format == "yaml":
+        dump_func = lambda input: yaml.dump(input, sort_keys=False)
+    for namespace in backup_list:
+        for database_container in namespace["database_containers"]:
+            for database in database_container["databases"]:
+                backups_ = {}
+                for retention, backups in database["backups"].items():
+                    backups_[str(retention.value)] = [bu.to_dict() for bu in backups]
+                database["backups"] = backups_
+    backup_list = cast_type_in_nested_dict_or_list(
+        backup_list, Path, lambda p: str(p.absolute())
+    )
+    return dump_func(backup_list)
+
+
+def cast_type_in_nested_dict_or_list(
+    data: Union[List, Dict], target_type: Type, cast_func: Callable
+) -> Union[List, Dict]:
+
+    if isinstance(data, list):
+        return [
+            cast_type_in_nested_dict_or_list(item, target_type, cast_func)
+            for item in data
+        ]
+    elif isinstance(data, dict):
+        new_data = {}
+        for key, val in data.items():
+            if isinstance(val, list):
+                new_data[key] = cast_type_in_nested_dict_or_list(
+                    val, target_type, cast_func
+                )
+            elif isinstance(val, dict):
+                new_data[key] = cast_type_in_nested_dict_or_list(
+                    val, target_type, cast_func
+                )
+            elif issubclass(type(val), target_type):
+                new_data[key] = cast_func(val)
+            else:
+                new_data[key] = val
+        return new_data
+    elif isinstance(data, target_type):
+        return cast_func(data)

README.md

@@ -89,6 +89,7 @@ Thats it. We now have a directory `./backups/` in front of us, with all database
 * (Planned) Email notification
 * (Planned) Support of docker and kubernetes secrets for providing database auth
 * (Planned) Docker Event listener / Kubectl hooks to react to container started/stopped
+* (Planned) Improve human readable output by printing a tree structure (https://www.baeldung.com/java-print-binary-tree-diagram)
 * (Idea) Database tools (Create non existing databases, based on labels)
 * (Idea) restore by label (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
 * (Idea) Matrix notifications