Commit 4d69957f authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

wip

parent eeb624fd
......@@ -36,11 +36,11 @@ class Backup:
if isinstance(path, str):
path = Path(path)
self.path: Path = path
self.backup_name: str = path.stem
self.name: str = path.stem
self.retention_type: RetentionType = RetentionType(path.parent.stem)
self.database_name: str = path.parent.parent.stem
self.creation_time: float = self._parse_filename_for_timestamp(self.backup_name)
self.creation_time: float = self._parse_filename_for_timestamp(self.name)
@property
def age(self):
......@@ -59,6 +59,15 @@ class Backup:
timestamp_unix = time.mktime(timestamp_datetime.timetuple())
return timestamp_unix
"""
def to_dict(self):
return {
"name": self.name,
"path": str(self.path),
"type": str(self.retention_type),
}
"""
class BackupManager:
# mapping of how many backups are kept per retention type
......
......@@ -24,6 +24,10 @@ from backupper import (
Neo4jOfflineBackupper,
Neo4jOnlineBackupper,
)
from cli_helper import (
format_make_list_of_backups_human_readable,
format_backup_list_human_readable,
)
from log import log
from label import ValidLabels
......@@ -150,6 +154,7 @@ def backup_now(
@click.option(
"--all-namespaces",
"-all-namespaces",
is_flag=True,
default=False,
help="If we should search for pods, to be backed up, in all namespaces. When used `--namespace` will be ignored.",
)
......@@ -169,6 +174,9 @@ def backup_kubernetes(namespace, all_namespaces, target_dir):
namespace=namespace, all_namespaces=all_namespaces
):
if pod.backup_labels[ValidLabels.enabled].val:
print(pod)
print(pod.backup_labels["backup.dzd-ev.de/type"])
print(ValidLabels.database_type == "backup.dzd-ev.de/type")
BackupperClass = database_type_backupper_mapping[
pod.backup_labels[ValidLabels.database_type].val
]
......@@ -180,7 +188,7 @@ def backup_kubernetes(namespace, all_namespaces, target_dir):
password=pod.backup_labels[ValidLabels.database_password].val,
kubernetes_all_namespaces=all_namespaces,
kubernetes_namespace=pod.kubernetes_namespace,
backups_base_path=target_dir
backups_base_path=Path(target_dir)
if target_dir
else pod.backup_labels[ValidLabels.backup_dir].val,
)
......@@ -235,7 +243,7 @@ def backup_docker(target_dir):
host=container.backup_labels[ValidLabels.database_host].val,
user=container.backup_labels[ValidLabels.database_username].val,
password=container.backup_labels[ValidLabels.database_password].val,
backups_base_path=target_dir
backups_base_path=Path(target_dir)
if target_dir
else container.backup_labels[ValidLabels.backup_dir].val,
)
......@@ -356,6 +364,7 @@ def restore_now(
@click.option(
"--all-namespaces",
"-all-namespaces",
is_flag=True,
default=False,
help="If we should search for pods, to be backed up, in all namespaces. When used `--namespace` will be ignored.",
)
......@@ -394,6 +403,7 @@ def restore_kubernetes(ctx, namespace, all_namespaces, pod_name, backup_name):
@click.option(
"--all-namespaces",
"-all-namespaces",
is_flag=True,
default=False,
help="If we should search for pods, with backups, in all namespaces. When used `--namespace` will be ignored.",
)
......@@ -413,7 +423,24 @@ def restore_kubernetes_list(
yaml,
source_dir,
):
backup_list: Dict = {}
t = [
{
"namespace": "default",
"database_containers": [
{
"name": "mariadb01",
"databases": [
{
"name": "coda_test",
"backups": [{"daily": [{"name": ""}], "weekly": []}],
}
],
}
],
}
]
backup_list: List = {}
for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
......@@ -431,7 +458,7 @@ def restore_kubernetes_list(
password=pod.backup_labels[ValidLabels.database_password].val,
kubernetes_all_namespaces=all_namespaces,
kubernetes_namespace=pod.kubernetes_namespace,
backups_base_path=source_dir
backups_base_path=Path(source_dir)
if source_dir
else pod.backup_labels[ValidLabels.backup_dir].val,
)
......@@ -443,11 +470,19 @@ def restore_kubernetes_list(
databases = bu.list_databases()
for db in databases:
db_backups = bu.manager.list_backups(database_name=db)
db_backups = format_make_list_of_backups_human_readable(
bu.manager.list_backups(database_name=db)
)
print(db_backups)
exit()
if db_backups:
backup_list[f'{pod.parent["metadata"]["name"]}/{db}'] = db_backups
if not pod.kubernetes_namespace in backup_list.keys():
backup_list[pod.kubernetes_namespace] = {}
backup_list[pod.kubernetes_namespace][
f'{pod.parent["metadata"]["name"]}/{db}'
] = db_backups
# make human readable or json/yaml
click.echo(backup_list)
print(backup_list)
def restore_docker():
......
from typing import Dict, List
from datetime import datetime, timedelta
import time
import humanize
import tabulate
tabulate.PRESERVE_WHITESPACE = True
from backup_manager import Backup, RetentionType
def format_backup_list_human_readable(backup_list: Dict):
pass
# 🖥️
def format_make_list_of_backups_human_readable(
backups: Dict[RetentionType, List[Backup]]
):
headers = ["", "Name", "Date", "Age", "Path"]
s = ""
for retenetion_type, backup_list in backups.items():
s = (
s
+ f"\n\n\n ⌚️ {retenetion_type} {'@ ' + str(backup_list[0].path.parents[1].absolute()) if backup_list else ''}\n\n"
)
backups_table_struc = []
for backup in backup_list:
if not backup.retention_type == retenetion_type:
continue
backups_table_struc.append(
[
" 💾",
backup.name,
datetime.utcfromtimestamp(backup.creation_time).strftime(
"%Y-%m-%d %H:%M:%S"
),
humanize.naturaldelta(
timedelta(seconds=time.time() - backup.creation_time)
),
backup.path.parents[0].name + "/" + backup.path.name,
]
)
s = s + tabulate.tabulate(backups_table_struc, headers=headers)
return s
......@@ -31,10 +31,7 @@ class Executer:
if self.mode.lower() == "docker":
return config.DOCKER_COMMAND + " exec"
elif self.mode.lower() == "kubernetes":
if self.all_namespaces:
self.namespace_arg = "--all-namespaces"
else:
self.namespace_arg = f"-n {self.namespace if self.namespace else config.KUBECTL_DEFAULT_NAMESPACE}"
self.namespace_arg = f"-n {self.namespace if self.namespace else config.KUBECTL_DEFAULT_NAMESPACE}"
return f"{config.KUBECTL_COMMAND} exec {self.namespace_arg}"
elif self.mode.lower() == "native":
return ""
......
......@@ -16,7 +16,7 @@ class Label:
def __init__(
self,
label: Union[str, Dict[str, str]],
type: Type = str,
type_: Type = str,
possible_values: List = None,
default: Any = None,
info: str = None,
......@@ -29,20 +29,29 @@ class Label:
key, *val = label.split("=")
self.key: str = f"{base_label_key + '/' if base_label_key and not key.startswith(base_label_key) else ''}{key}"
try:
if type == bool and isinstance(val, str):
if type_ == bool and isinstance(val, str):
val = util.strtobool(val)
self.val: Union[str, int, bool] = type(val) if val else default
self.val: Union[str, int, bool] = type_(val) if val else default
except:
raise ValueError(
f"Label '{key}={val}' expected to be of type '{type}'. Can not convert '{val}' to {type}"
f"Label '{key}={val}' expected to be of type '{type_}'. Can not convert '{val}' to {type_}"
)
self.type: Type = type
if possible_values and self.val and self.val not in possible_values:
raise ValueError(
f"Label value of '{self.key}' expected to be one of {possible_values} but is '{self.val}' (value type: '{type(self.val)}')"
)
self.type: Type = type_
self.possible_values: List = possible_values
self.default: Union[str, int, bool] = default
self.info: str = info
def __eq__(self, other: "Label"):
return type(other) is type(self) and self.key == other.key
def __eq__(self, other: Union["Label", str]):
return (
type(other) is type(self)
and self.key == other.key
or type(other) == str
and self.key == other
)
def __hash__(self):
return hash(self.key)
......@@ -185,7 +194,7 @@ class ValidLabels:
def label_from_valid_label_as_template(cls, valid_label: Label, val):
return Label(
label={valid_label.key: val},
type=valid_label.type,
type_=valid_label.type,
possible_values=valid_label.possible_values,
default=valid_label.default,
info=valid_label.info,
......
# Create postgres testdata
```bash
docker exec mysql /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@localhost -c "CREATE DATABASE coda_ps_test;" && /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@localhost -c "\
BEGIN; \
create schema coda_ps_test_scheme; \
CREATE TABLE IF NOT EXISTS coda_ps_test_scheme.my_table(id integer PRIMARY KEY, firstname VARCHAR(32)); \
INSERT INTO coda_ps_test_scheme.my_table(id,firstname) VALUES (1,'Anna'); \
INSERT INTO coda_ps_test_scheme.my_table(id,firstname) VALUES (2,'Thomas'); \
commit;"
```
\ No newline at end of file
DZDConfigs
click
\ No newline at end of file
......@@ -14,7 +14,7 @@ setup(
author_email="tim.bleimehl@helmholtz-muenchen.de",
license="MIT",
packages=["CoDaBackup"],
install_requires=["DZDConfigs", "Click"],
install_requires=["DZDConfigs", "Click", "tabulate", "humanize"],
python_requires=">=3.8",
zip_safe=False,
include_package_data=True,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment