Commit 33aca929 authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

rc1 of cli.py restore kubernetes list

parent aa582472
from typing import List, Dict, Union
from typing import List, Dict, Union, Callable
import logging
from datetime import datetime
import time
......@@ -8,7 +8,7 @@ from enum import Enum
from pathlib import Path, PurePath
from Configs import getConfig
import operator
import json
from config import DEFAULT
config: DEFAULT = getConfig(config_classes=[DEFAULT])
......@@ -59,6 +59,19 @@ class Backup:
timestamp_unix = time.mktime(timestamp_datetime.timetuple())
return timestamp_unix
def to_dict(self, meta_data: bool = False) -> Dict:
data = {
"name": self.name,
"creation_time": self.creation_time,
"path": self.path,
}
if meta_data:
data = data | {
"retention_type": self.retention_type,
"database_name": self.database_name,
}
return data
"""
def to_dict(self):
return {
......@@ -123,6 +136,10 @@ class BackupManager:
self.backup_file_suffix = backup_file_suffix
# self.create_pathes_if_not_exists()
def list_backuped_databases(self) -> List[str]:
"""List database names based on available backups and not on available online databases"""
return [dir.name for dir in self.base_path.iterdir() if dir.is_dir()]
def list_backups(
self,
database_name: str,
......
#!/usr/bin/env python3
from email.policy import default
from pydoc import describe
# from email.policy import default
# from pydoc import describe
import click
import os
import sys
from typing import Dict, List, Type
from pathlib import Path, PurePath
import json
from collections import OrderedDict
if __name__ == "__main__":
SCRIPT_DIR = os.path.dirname(
......@@ -26,8 +28,8 @@ from backupper import (
Neo4jOnlineBackupper,
)
from cli_helper import (
format_make_list_of_backups_human_readable,
format_backup_list_human_readable,
backup_list_to_human,
backup_list_to_machine_readable,
)
from log import log
......@@ -380,13 +382,16 @@ def restore_kubernetes(ctx, namespace, all_namespaces, pod_name, backup_name):
@restore_kubernetes.command(name="list")
@click.option(
"--pod-name",
"-p",
"--workload-name",
"-w",
default=None,
help="User to access the database?",
)
@click.option(
"--databases", default=None, help="List only these databases (seperated by comma)"
"--databases",
"-d",
default=None,
help="List only these databases (seperated by comma)",
)
@click.option(
"--namespace",
......@@ -409,59 +414,84 @@ def restore_kubernetes(ctx, namespace, all_namespaces, pod_name, backup_name):
default=None,
help=f"Where to look for the backups. Will default to '{ValidLabels.backup_dir.val}'",
)
@click.option("--yaml", default=False, help="Output list machine readable")
@click.option(
"--output-format",
"-o",
default="human",
help="Define the output format: 'human' an easy to read table, 'json' or 'yaml' for inter-process readability, dict if used in a python script",
type=click.Choice(["human", "json", "yaml", "dict"], case_sensitive=False),
)
def restore_kubernetes_list(
pod_name,
databases,
namespace,
all_namespaces,
yaml,
source_dir,
workload_name, databases, namespace, all_namespaces, source_dir, output_format
):
"""Display available backup files"""
# ToDo: This func is messy :) we sure can improve this.
backup_list: List = {}
data = []
for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
if pod_name and pod_name != pod.name:
if databases:
databases = databases.split(",")
if workload_name and pod.parent["metadata"]["name"] != workload_name:
continue
pod_workload_name = pod.parent["metadata"]["name"]
if pod.coda_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
pod.coda_labels[ValidLabels.database_type].val
]
bu = BackupperClass(
mode="kubernetes",
container_name=pod.name,
host=pod.coda_labels[ValidLabels.database_host].val,
user=pod.coda_labels[ValidLabels.database_username].val,
password=pod.coda_labels[ValidLabels.database_password].val,
kubernetes_all_namespaces=all_namespaces,
kubernetes_namespace=pod.kubernetes_namespace,
backups_base_path=Path(source_dir)
if source_dir
else pod.coda_labels[ValidLabels.backup_dir].val,
)
if databases:
databases = databases.split(",")
elif pod.coda_labels[ValidLabels.database_names].val:
databases = pod.coda_labels[ValidLabels.database_names].val.split(",")
else:
databases = bu.list_databases()
for db in databases:
db_backups = format_make_list_of_backups_human_readable(
bu.manager.list_backups(database_name=db)
bu = BackupperClass(pod)
for database_name in bu.manager.list_backuped_databases():
if databases and not database_name in databases:
continue
namespace = None
for ns in data:
if (
"namespace" in ns
and ns["namespace"] == pod.kubernetes_namespace
):
namespace = ns
if namespace is None:
namespace = {
"namespace": pod.kubernetes_namespace,
"mode": "kubernetes",
"database_containers": [],
}
data.append(namespace)
pod_entry = next(
(
p
for p in namespace["database_containers"]
if p["name"] == pod_workload_name
),
None,
)
if not pod_entry:
pod_entry = {
"name": pod_workload_name,
"basedir": bu.manager.base_path,
"databases": [],
}
namespace["database_containers"].append(pod_entry)
pod_entry["databases"].append(
{
"name": database_name,
"basedir": Path(PurePath(bu.manager.base_path, database_name)),
"backups": bu.manager.list_backups(database_name),
}
)
print(db_backups)
exit()
if db_backups:
if not pod.kubernetes_namespace in backup_list.keys():
backup_list[pod.kubernetes_namespace] = {}
backup_list[pod.kubernetes_namespace][
f'{pod.parent["metadata"]["name"]}/{db}'
] = db_backups
# make human readable or json/yaml
print(backup_list)
if output_format in ["json", "yaml"]:
output = backup_list_to_machine_readable(data, output_format)
print(output)
return output
elif output_format == "human":
output = backup_list_to_human(data)
print(output)
return output
elif output_format == "dict":
return data
def restore_docker():
......
from typing import Dict, List
from ast import Call
from typing import Dict, List, Union, Type, Callable
from datetime import datetime, timedelta
import time
import humanize
from numpy import sort
from pathlib import Path
import tabulate
tabulate.PRESERVE_WHITESPACE = True
from backup_manager import Backup, RetentionType
import json
import yaml
def format_backup_list_human_readable(backup_list: Dict):
def backup_list_to_human(backup_list: Dict):
"""[summary]
Args:
......@@ -40,6 +45,9 @@ def format_backup_list_human_readable(backup_list: Dict):
}
],
"weekly": [],
"monthly": [],
"yearly": [],
"manual": [],
}
],
}
......@@ -50,18 +58,29 @@ def format_backup_list_human_readable(backup_list: Dict):
]
```
"""
data = ""
for namespace in backup_list:
if len(backup_list) > 1:
# we not only have the default namespace. otherwise we could hide it
data = data + f"\n\n🌐 NAMESPACE: {namespace['namespace']}\n\n"
for container in namespace["database_containers"]:
data = data + f" 🖥️ Container: {container['name']}\n\n"
for database in container["databases"]:
data = data + f" 🛢 Database: {database['name']}\n"
data = data + backup_file_list_to_human(database["backups"], 6)
return data
# 🖥️
def format_make_list_of_backups_human_readable(
backups: Dict[RetentionType, List[Backup]]
def backup_file_list_to_human(
backups: Dict[RetentionType, List[Backup]], indent: int = 4
):
headers = ["", "Name", "Date", "Age", "Path"]
s = ""
for retenetion_type, backup_list in backups.items():
s = (
s
+ f"\n\n\n ⌚️ {retenetion_type} {'@ ' + str(backup_list[0].path.parents[1].absolute()) if backup_list else ''}\n\n"
+ f"\n ⌚️ {retenetion_type} {'@ ' + str(backup_list[0].path.parents[1].absolute()) if backup_list else ''}\n"
)
backups_table_struc = []
for backup in backup_list:
......@@ -71,7 +90,7 @@ def format_make_list_of_backups_human_readable(
backups_table_struc.append(
[
" 💾",
"💾",
backup.name,
datetime.utcfromtimestamp(backup.creation_time).strftime(
"%Y-%m-%d %H:%M:%S"
......@@ -82,6 +101,61 @@ def format_make_list_of_backups_human_readable(
backup.path.parents[0].name + "/" + backup.path.name,
]
)
s = s + tabulate.tabulate(backups_table_struc, headers=headers)
table = tabulate.tabulate(backups_table_struc, headers=headers)
indented_table = ""
for line in table.split("\n"):
indented_table = indented_table + " " * indent + line + "\n"
s = s + indented_table
return s
def backup_list_to_machine_readable(backup_list: List, format: str = "json"):
"""Serialize all dict entries of a backup_list"""
if format != "json" and format != "yaml":
raise ValueError(
f"Unknown format style. Expected 'json' or 'yaml' got '{format}'"
)
elif format == "json":
dump_func = json.dumps
elif format == "yaml":
dump_func = lambda input: yaml.dump(input, sort_keys=False)
for namespace in backup_list:
for database_container in namespace["database_containers"]:
for database in database_container["databases"]:
backups_ = {}
for retention, backups in database["backups"].items():
backups_[str(retention.value)] = [bu.to_dict() for bu in backups]
database["backups"] = backups_
backup_list = cast_type_in_nested_dict_or_list(
backup_list, Path, lambda p: str(p.absolute())
)
return dump_func(backup_list)
def cast_type_in_nested_dict_or_list(
data: Union[List, Dict], target_type: Type, cast_func: Callable
) -> Union[List, Dict]:
if isinstance(data, list):
return [
cast_type_in_nested_dict_or_list(item, target_type, cast_func)
for item in data
]
elif isinstance(data, dict):
new_data = {}
for key, val in data.items():
if isinstance(val, list):
new_data[key] = cast_type_in_nested_dict_or_list(
val, target_type, cast_func
)
elif isinstance(val, dict):
new_data[key] = cast_type_in_nested_dict_or_list(
val, target_type, cast_func
)
elif issubclass(type(val), target_type):
new_data[key] = cast_func(val)
else:
new_data[key] = val
return new_data
elif isinstance(data, target_type):
return cast_func(data)
......@@ -89,6 +89,7 @@ Thats it. We now have a directory `./backups/` in front of us, with all database
* (Planned) Email notification
* (Planned) Support of docker and kubernetes secrets for providing database auth
* (Planned) Docker Event listener / Kubectl hooks to react to container started/stopped
* (Planned) Improve human readable output by printing a tree structure (https://www.baeldung.com/java-print-binary-tree-diagram)
* (Idea) Database tools (Create non existing databases, based on labels)
* (Idea) restore by label (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Idea) Matrix notifications
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment