Commit ab40dd85 authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

wip

parent 8e4604aa
#!/usr/bin/env python3
from pydoc import describe
import click
import os
import sys
......@@ -176,39 +177,49 @@ def backup_kubernetes(
from container_helper import ContainerHelper
i = 0
for container in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
if container.backup_labels[ValidLabels.enabled].val:
if pod.backup_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
container.backup_labels[ValidLabels.database_type].val
pod.backup_labels[ValidLabels.database_type].val
]
bu = BackupperClass(
executer=Executer(
"kubernetes", container.name, namespace, all_namespaces
),
host=container.backup_labels[ValidLabels.database_host].val,
user=container.backup_labels[ValidLabels.database_username].val,
password=container.backup_labels[ValidLabels.database_password].val,
executer=Executer("kubernetes", pod.name, namespace, all_namespaces),
host=pod.backup_labels[ValidLabels.database_host].val,
user=pod.backup_labels[ValidLabels.database_username].val,
password=pod.backup_labels[ValidLabels.database_password].val,
)
bu.manager.retention_duration = {
RetentionType.DAILY: pod.backup_labels[ValidLabels.retention_daily].val,
RetentionType.WEEKLY: pod.backup_labels[
ValidLabels.retention_weekly
].val,
RetentionType.MONTHLY: pod.backup_labels[
ValidLabels.retention_monthly
].val,
RetentionType.YEARLY: pod.backup_labels[
ValidLabels.retention_yearly
].val,
RetentionType.MANUAL: pod.backup_labels[
ValidLabels.retention_manual
].val,
}
bu.manager.base_path = Path(
PurePath(
target_dir
if target_dir
else container.backup_labels[ValidLabels.backup_dir].val,
container.parent["metadata"]["namespace"]
else pod.backup_labels[ValidLabels.backup_dir].val,
pod.parent["metadata"]["namespace"]
if not exclude_namespace_from_path
else "",
container.parent["metadata"]["name"],
pod.parent["metadata"]["name"],
)
)
if not container.backup_labels[ValidLabels.database_names].val:
if not pod.backup_labels[ValidLabels.database_names].val:
databases = None
else:
databases = container.backup_labels[
ValidLabels.database_names
].val.split(",")
databases = pod.backup_labels[ValidLabels.database_names].val.split(",")
bu.backup(
databases=databases,
retention_type=RetentionType.DAILY,
......@@ -223,35 +234,48 @@ def backup_kubernetes(
def backup_docker(target_dir):
"""Backup databases in a docker environment"""
for container_name in ContainerHelper.docker_get_container_to_be_backed_up():
config_labels = ContainerHelper.docker_get_config_by_labels(
container_name=container_name
)
if config_labels[ValidLabels.enabled].val:
for container in ContainerHelper.docker_get_container_to_be_backed_up(
describe=True
):
if container.backup_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
config_labels[ValidLabels.database_type].val
container.backup_labels[ValidLabels.database_type].val
]
print(container.backup_labels)
bu = BackupperClass(
executer=Executer("docker", container_name),
host=config_labels[ValidLabels.database_host].val,
user=config_labels[ValidLabels.database_username].val,
password=config_labels[ValidLabels.database_password].val,
executer=Executer("docker", container.name),
host=container.backup_labels[ValidLabels.database_host].val,
user=container.backup_labels[ValidLabels.database_username].val,
password=container.backup_labels[ValidLabels.database_password].val,
)
bu.manager.base_path = Path(target_dir)
if not config_labels[ValidLabels.database_names].val:
if not container.backup_labels[ValidLabels.database_names].val:
databases = None
else:
databases = config_labels[ValidLabels.database_names].val.split(",")
databases = container.backup_labels[
ValidLabels.database_names
].val.split(",")
bu.manager.retention_duration = {
RetentionType.DAILY: config_labels[ValidLabels.retention_daily].val,
RetentionType.WEEKLY: config_labels[ValidLabels.retention_weekly].val,
RetentionType.MONTHLY: config_labels[ValidLabels.retention_monthly].val,
RetentionType.YEARLY: config_labels[ValidLabels.retention_yearly].val,
RetentionType.MANUAL: config_labels[ValidLabels.retention_manual].val,
RetentionType.DAILY: container.backup_labels[
ValidLabels.retention_daily
].val,
RetentionType.WEEKLY: container.backup_labels[
ValidLabels.retention_weekly
].val,
RetentionType.MONTHLY: container.backup_labels[
ValidLabels.retention_monthly
].val,
RetentionType.YEARLY: container.backup_labels[
ValidLabels.retention_yearly
].val,
RetentionType.MANUAL: container.backup_labels[
ValidLabels.retention_manual
].val,
}
bu.manager.base_path = Path(
PurePath(
bu.manager.base_path, config_labels[ValidLabels.backup_dir].val
bu.manager.base_path,
container.backup_labels[ValidLabels.backup_dir].val,
)
)
bu.backup(
......@@ -367,54 +391,90 @@ def restore_kubernetes(ctx, namespace, all_namespaces, pod_name, backup_name):
@restore_kubernetes.command(name="list")
@click.option(
"--pod",
"--pod-name",
"-p",
default=None,
prompt="List backups of which container",
help="User to access the database?",
)
@click.option(
"--databases", default=None, help="List only these databases (seperated by comma)"
)
@click.option(
"--namespace",
"-n",
type=str,
default="default",
help="Define the namespace we should search for database pods with backups.",
)
@click.option(
"--all-namespaces",
"-all-namespaces",
default=False,
help="If we should search for pods, with backups, in all namespaces. When used `--namespace` will be ignored.",
)
@click.option(
"--source-dir",
"-s",
type=str,
default=None,
help=f"Where to look for the backups. Will default to '{ValidLabels.backup_dir.val}'",
)
@click.option(
"--exclude-namespace-from-path",
"-e",
default=False,
help=f"By default the kubernetes namespace of a database workload will be included in the backup path to prevent name collisions. Attach this flag to prevent this and have a flatter directory structure, if you are sure your workloads have uniqe names over all namespaces. Alternatively set the label '{str(ValidLabels.backup_dir)}' on a per workload base",
)
@click.option("--yaml", default=False, help="Output list machine readable")
@click.option("--base-path", default=None, help="Define base dir of backups")
def restore_kubernetes_list(pod, databases, yaml, base_path):
backup_list: Dict = []
for container_name in ContainerHelper.kubernetes_get_pod_names_to_be_backed_up():
if pod and pod != container_name:
def restore_kubernetes_list(
pod_name,
databases,
namespace,
all_namespaces,
exclude_namespace_from_path,
yaml,
source_dir,
):
backup_list: Dict = {}
for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
if pod_name and pod_name != pod.name:
continue
config_labels = ContainerHelper.kubernetes_get_config_by_labels(
container_name=container_name
)
if config_labels[ValidLabels.enabled].val:
if pod.backup_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
config_labels[ValidLabels.database_type].val
pod.backup_labels[ValidLabels.database_type].val
]
bu = BackupperClass(
executer=Executer("kubernetes", container_name),
host=config_labels[ValidLabels.database_host].val,
user=config_labels[ValidLabels.database_username].val,
password=config_labels[ValidLabels.database_password].val,
executer=Executer("kubernetes", pod.name, namespace, all_namespaces),
host=pod.backup_labels[ValidLabels.database_host].val,
user=pod.backup_labels[ValidLabels.database_username].val,
password=pod.backup_labels[ValidLabels.database_password].val,
)
if base_path:
bu.manager.base_path = Path(base_path)
if databases:
databases = databases.split(",")
elif config_labels[ValidLabels.database_names].val:
databases = config_labels[ValidLabels.database_names].val.split(",")
else:
databases = bu.list_databases()
bu.manager.base_path = Path(
PurePath(
bu.manager.base_path, config_labels[ValidLabels.backup_dir].val
source_dir
if source_dir
else pod.backup_labels[ValidLabels.backup_dir].val,
pod.parent["metadata"]["namespace"]
if not exclude_namespace_from_path
else "",
pod.parent["metadata"]["name"],
)
)
# YOU ARE HERE. list databases bakcups
if databases:
databases = databases.split(",")
elif pod.backup_labels[ValidLabels.database_names].val:
databases = pod.backup_labels[ValidLabels.database_names].val.split(",")
else:
databases = bu.list_databases()
for db in databases:
db_backups = bu.manager.list_backups(database_name=db)
backup_list[f"{container_name}/{db}"] = db_backups
if db_backups:
backup_list[f'{pod.parent["metadata"]["name"]}/{db}'] = db_backups
# make human readable or json/yaml
click.echo(backup_list)
......
......@@ -24,6 +24,38 @@ class Container:
desc: Dict = None
parent: Dict = None
@classmethod
def from_docker_inspect_dict(cls, container_inspect_result: dict):
# todo validate correct dict type
return cls(
id=container_inspect_result[0]["Id"],
name=container_inspect_result[0]["Name"],
backup_labels=ValidLabels.valid_labels_from_dict(
container_inspect_result[0]["Config"]["Labels"],
add_missing_default_labels=True,
),
other_labels=ValidLabels.non_valid_labels_from_dict(
container_inspect_result[0]["Config"]["Labels"]
),
desc=container_inspect_result[0],
)
@classmethod
def from_kubernetes_get(cls, kubectl_get_item_result: dict):
# todo validate correct item by "kind" field
return cls(
id=kubectl_get_item_result["metadata"]["uid"],
name=kubectl_get_item_result["metadata"]["name"],
backup_labels=ValidLabels.valid_labels_from_dict(
kubectl_get_item_result["metadata"]["labels"],
add_missing_default_labels=True,
),
other_labels=ValidLabels.non_valid_labels_from_dict(
kubectl_get_item_result["metadata"]["labels"]
),
desc=kubectl_get_item_result,
)
class ContainerHelper:
@classmethod
......@@ -60,19 +92,7 @@ class ContainerHelper:
)["items"]
pods: List[Container] = []
for pod_desc in pod_descs:
pods.append(
Container(
id=pod_desc["metadata"]["uid"],
name=pod_desc["metadata"]["name"],
backup_labels=ValidLabels.valid_labels_from_dict(
pod_desc["metadata"]["labels"], add_missing_default_labels=True
),
other_labels=ValidLabels.non_valid_labels_from_dict(
pod_desc["metadata"]["labels"]
),
desc=pod_desc,
)
)
pods.append(Container.from_kubernetes_get(pod_desc))
if describe:
return pods
else:
......@@ -118,12 +138,14 @@ class ContainerHelper:
@classmethod
def kubernetes_get_workloads_to_be_backed_up(
cls, label: str = None, namespace: str = None, all_namespaces: bool = False
cls, namespace: str = None, all_namespaces: bool = False
) -> List[str]:
# kubectl get all
# kubectl get pods -o jsonpath='{range .items[?(.kind=StatefulSet)]}{.metadata.name}{end}'
if not label:
label = str(ValidLabels.enabled.key)
label = ValidLabels.label_from_valid_label_as_template(
ValidLabels.enabled, val=True
)
if all_namespaces:
namespace_arg = "--all-namespaces"
else:
......@@ -143,36 +165,49 @@ class ContainerHelper:
return workloads
@classmethod
def docker_get_container_to_be_backed_up(cls, label: str = None) -> List[str]:
# docker ps --format '{{ .Names }}' --filter "label=color"
import os
# os.environ["CONFIGS_LOG_LEVEL"] = "DEBUG"
# config.refresh_config_from_env_var()
if not label:
label = ValidLabels.enabled.key
pods: List = (
def docker_get_container_to_be_backed_up(
cls, describe: bool = False
) -> Union[List[str], List[Container]]:
return cls.docker_get_containers(
labels=[
ValidLabels.label_from_valid_label_as_template(
ValidLabels.enabled, val=True
)
],
describe=describe,
)
@classmethod
def docker_get_containers(
cls, labels: List[Label] = None, describe: bool = False
) -> Union[List[str], List[Container]]:
filter_args = ""
if labels:
filter_args = " ".join(
['--filter="label=' + str(label) + '"' for label in labels]
)
container_ids = (
Executer.exec(
f'{config.DOCKER_COMMAND} ps --format "{{{{ .ID }}}}" --filter label={label}'
f'{config.DOCKER_COMMAND} ps --format "{{{{ .ID }}}}" {filter_args}'
)
.decode("utf-8")
.splitlines()
)
log.debug(f"Containers found: {pods}")
return pods
@classmethod
def kubernetes_get_pod_sql_type_by_label(cls, pod_name: str, label_key: str = None):
if not label_key:
label_key = config.DATABASE_CONTAINER_LABEL_BASE_KEY
pod: Dict = json.loads(
Executer.exec(
f"{config.KUBECTL_COMMAND} {config.KUBECTL_NAMESPACE_PARAM} get pods {pod_name} -o json"
if not describe:
return container_ids
containers: List[Container] = []
for container_id in container_ids:
container = Container.from_docker_inspect_dict(
json.loads(
Executer.exec(
f"{config.DOCKER_COMMAND} inspect {container_id}"
).decode("utf-8")
)
)
)
for label_k, val in pod["metadata"]["labels"].items():
if label_k == label_key:
return val
containers.append(container)
log.debug(f"Containers found: {containers}")
return containers
@classmethod
def kubernetes_get_config_by_labels(cls, pod_name: str) -> Dict[Label, Label]:
......@@ -192,7 +227,7 @@ class ContainerHelper:
return config_labels_dict
@classmethod
def docker_get_config_by_labels(cls, container_name: str) -> Dict[Label, Label]:
def docker_get_labels(cls, container_name: str, valid=True) -> Dict[Label, Label]:
# config.DATABASE_CONTAINER_LABEL_BASE_KEY
container_labels: Dict = json.loads(
......@@ -200,6 +235,7 @@ class ContainerHelper:
f"{config.DOCKER_COMMAND} inspect --format '{{{{ json .Config.Labels }}}}' {container_name}"
)
)
ValidLabels.valid_labels_from_dict()
config_labels_dict = {}
for label in ValidLabels.iter():
......
......@@ -27,7 +27,7 @@ class Label:
val = list(label.values())[0]
else:
key, *val = label.split("=")
self.key: str = f"{base_label_key + '/' if base_label_key else ''}{key}"
self.key: str = f"{base_label_key + '/' if base_label_key and not key.startswith(base_label_key) else ''}{key}"
try:
if type == bool and isinstance(val, str):
val = util.strtobool(val)
......@@ -48,8 +48,12 @@ class Label:
return hash(self.key)
def __str__(self):
if self.val is not None and self.val != "":
return f"{self.key}={self.val}"
if self.type == bool and self.val is not None:
val = str(self.val).lower()
else:
val = self.val
if val is not None and val != "":
return f"{self.key}={val}"
else:
return self.key
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment