Skip to content
GitLab
Commit 8e4604aa authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

wip

parent 0ee46f17
Hide whitespace changes
Inline Side-by-side
CoDaBackup/backup_manager.py
View file @ 8e4604aa
......@@ -9,7 +9,7 @@ from pathlib import Path, PurePath
from Configs import getConfig
import operator
from CoDaBackup.config import DEFAULT
from config import DEFAULT
config: DEFAULT = getConfig(config_classes=[DEFAULT])
log = logging.getLogger("databasebackupper")
......@@ -233,7 +233,10 @@ class BackupManager:
def rotate_existing_backups(self):
log.info("Start backup rotating...")
if not self.base_path.is_dir():
log.warning.info(f"No backups to rotate at '{str(self.base_path)}'")
for database_base_backup_dir in self.base_path.iterdir():
if not database_base_backup_dir.is_dir():
# something is odd here. There is a file in our directory. lets skip this
continue
......
CoDaBackup/cli.py
View file @ 8e4604aa
......@@ -14,19 +14,19 @@ if __name__ == "__main__":
sys.path.append(os.path.normpath(SCRIPT_DIR))
from CoDaBackup.backup_manager import RetentionType
from CoDaBackup.executer import Executer
from CoDaBackup.backupper import (
from backup_manager import RetentionType
from executer import Executer
from backupper import (
BaseBackupper,
MySQLBackupper,
PostgresBackupper,
Neo4jOfflineBackupper,
Neo4jOnlineBackupper,
)
from CoDaBackup.log import log
from log import log
from CoDaBackup.label import ValidLabels
from CoDaBackup.container_helper import ContainerHelper
from label import ValidLabels
from container_helper import ContainerHelper
# Todo: Skip container related question in native mode. maybe via groups? https://click.palletsprojects.com/en/8.0.x/commands/#nested-handling-and-contexts
......@@ -39,8 +39,15 @@ database_type_backupper_mapping: Dict[str, Type[BaseBackupper]] = {
@click.group()
def cli_root():
pass
@click.option(
"--debug/--no-debug",
help="Enable debug logging",
default=False,
)
def cli_root(debug):
if debug:
click.echo(f"Debug mode is on")
log.setLevel("DEBUG")
@cli_root.group(name="backup")
......@@ -149,34 +156,59 @@ def backup_now(
default=False,
help="If we should search for pods, to be backed up, in all namespaces. When used `--namespace` will be ignored.",
)
def backup_kubernetes(namespace, all_namespaces):
@click.option(
"--target-dir",
"-d",
type=str,
default=None,
help=f"Where to store the backups. Will default to '{ValidLabels.backup_dir.val}'",
)
@click.option(
"--exclude-namespace-from-path",
"-e",
default=False,
help=f"By default the kubernetes namespace of a database workload will be included in the backup path to prevent name collisions. Attach this flag to prevent this and have a flatter directory structure, if you are sure your workloads have uniqe names over all namespaces. Alternatively set the label '{str(ValidLabels.backup_dir)}' on a per workload base",
)
def backup_kubernetes(
namespace, all_namespaces, target_dir, exclude_namespace_from_path
):
"""Backup databases in a kubernetes environment"""
from container_helper import ContainerHelper
if all_namespaces:
os.environ["CONFIGS_KUBECTL_NAMESPACE_PARAM"] = "--all-namespaces"
else:
os.environ["CONFIGS_KUBECTL_NAMESPACE_PARAM"] = f"-n {namespace}"
click.echo(f"Backup all DBs in namespace '{namespace}'")
i = 0
for podname in ContainerHelper.kubernetes_get_pod_names_to_be_backed_up():
config_labels = ContainerHelper.kubernetes_get_config_by_labels(
pod_name=podname
)
if config_labels[ValidLabels.enabled].val:
for container in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
if container.backup_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
config_labels[ValidLabels.database_type].val
container.backup_labels[ValidLabels.database_type].val
]
bu = BackupperClass(
executer=Executer("kubernetes", podname),
host=config_labels[ValidLabels.database_host].val,
user=config_labels[ValidLabels.database_username].val,
password=config_labels[ValidLabels.database_password].val,
executer=Executer(
"kubernetes", container.name, namespace, all_namespaces
),
host=container.backup_labels[ValidLabels.database_host].val,
user=container.backup_labels[ValidLabels.database_username].val,
password=container.backup_labels[ValidLabels.database_password].val,
)
if not config_labels[ValidLabels.database_names].val:
bu.manager.base_path = Path(
PurePath(
target_dir
if target_dir
else container.backup_labels[ValidLabels.backup_dir].val,
container.parent["metadata"]["namespace"]
if not exclude_namespace_from_path
else "",
container.parent["metadata"]["name"],
)
)
if not container.backup_labels[ValidLabels.database_names].val:
databases = None
else:
databases = config_labels[ValidLabels.database_names].val.split(",")
databases = container.backup_labels[
ValidLabels.database_names
].val.split(",")
bu.backup(
databases=databases,
retention_type=RetentionType.DAILY,
......
CoDaBackup/config.py
View file @ 8e4604aa
......@@ -43,8 +43,8 @@ class DEFAULT(ConfigBase):
# `["/bin/bash", "-c"]` will work on the official mysql/mariadb/postgres containers
BASE_EXECUTER_COMMAND: List[str] = ["/bin/bash", "-c"]
DOCKER_COMMAND: str = "docker"
KUBECTL_VALID_WORKLOAD_TYPES: List[str] = ["Deployment", "StatefulSet"]
KUBECTL_COMMAND: str = "kubectl"
KUBECTL_NAMESPACE_PARAM: str = "--all-namespaces"
KUBECTL_DEFAULT_NAMESPACE: str = "default"
# Where should we search for Workloads that could contain run a Database Pod
KUBECTL_VALID_WORKLOAD_TYPES: List[str] = ["Deployment", "StatefulSet"]
CoDaBackup/container_helper.py
View file @ 8e4604aa
......@@ -5,9 +5,9 @@ from Configs import getConfig
import json
from dataclasses import dataclass
from CoDaBackup.config import DEFAULT
from CoDaBackup.label import ValidLabels, Label
from CoDaBackup.executer import Executer
from config import DEFAULT
from label import ValidLabels, Label
from executer import Executer
config: DEFAULT = getConfig(config_classes=[DEFAULT])
log = logging.getLogger("databasebackupper")
......@@ -17,10 +17,10 @@ log = logging.getLogger("databasebackupper")
@dataclass
class Container:
uid: str
id: str
name: str
backup_labels: List[Label]
other_labels: List[Label]
backup_labels: Dict[Label, Label]
other_labels: Dict[Label, Label]
desc: Dict = None
parent: Dict = None
......@@ -62,25 +62,21 @@ class ContainerHelper:
for pod_desc in pod_descs:
pods.append(
Container(
uid=pod_desc["metadata"]["uid"],
id=pod_desc["metadata"]["uid"],
name=pod_desc["metadata"]["name"],
backup_labels=[
Label(lbl)
for lbl in pod_desc["metadata"]["labels"]
if Label(lbl) in ValidLabels.iter()
],
other_labels=[
Label(lbl)
for lbl in pod_desc["metadata"]["labels"]
if Label(lbl) not in ValidLabels.iter()
],
backup_labels=ValidLabels.valid_labels_from_dict(
pod_desc["metadata"]["labels"], add_missing_default_labels=True
),
other_labels=ValidLabels.non_valid_labels_from_dict(
pod_desc["metadata"]["labels"]
),
desc=pod_desc,
)
)
if describe:
return pods
else:
return [pod.uid for pod in pods]
return [pod.id for pod in pods]
@classmethod
def kubernetes_get_pods_to_be_backed_up(
......@@ -89,29 +85,35 @@ class ContainerHelper:
workloads = cls.kubernetes_get_workloads_to_be_backed_up(
label, namespace, all_namespaces
)
pods: List[str] = []
pods: List[Container] = []
for workload in workloads:
if (
"spec" in workload
and "selector" in workload["spec"]
and "matchLables" in workload["spec"]["selector"]
and "matchLabels" in workload["spec"]["selector"]
):
# the Workload specs.selector.matchLabels defines how pods running under this workload have to be labeled
# this provides us the information to find pods belonging to a/this specific workload
labels: List[Label] = [
pod_selector_labels: List[Label] = [
Label(lbl) for lbl in workload["spec"]["selector"]["matchLabels"]
]
wl_pods = cls.kubernetes_get_pods(
labels=labels, namespace=namespace, all_namespaces=all_namespaces
)
for wl_pod in wl_pods:
# we attach the workloads labels because these could contain the metadata to access the database
wl_pod.backup_labels.extend(
[Label(lbl) for lbl in workload["metadata"]["labels"]]
)
# Attach the parent workload data just for good measure... not in any use yet. maybe we can delete this step
wl_pod.parent = workload
pods.extend(wl_pod)
for wl_pod in cls.kubernetes_get_pods(
labels=pod_selector_labels,
namespace=namespace,
all_namespaces=all_namespaces,
describe=True,
):
# we merge the workloads backup config labels into the pod label. usally only the kubernetes workload will have the backup config labels
workload_backup_config_labels = ValidLabels.valid_labels_from_dict(
workload["metadata"]["labels"], add_missing_default_labels=True
)
wl_pod.backup_labels = (
wl_pod.backup_labels | workload_backup_config_labels
)
# Attach the parent workload data just for good measure... not in any use yet. maybe we can delete this step
wl_pod.parent = workload
pods.append(wl_pod)
return pods
@classmethod
......@@ -121,7 +123,7 @@ class ContainerHelper:
# kubectl get all
# kubectl get pods -o jsonpath='{range .items[?(.kind=StatefulSet)]}{.metadata.name}{end}'
if not label:
label = str(ValidLabels.enabled)
label = str(ValidLabels.enabled.key)
if all_namespaces:
namespace_arg = "--all-namespaces"
else:
......
CoDaBackup/executer.py
View file @ 8e4604aa
......@@ -2,14 +2,20 @@ import subprocess
import logging
from Configs import getConfig
from CoDaBackup.config import DEFAULT
from config import DEFAULT
config: DEFAULT = getConfig(config_classes=[DEFAULT])
log = logging.getLogger("databasebackupper")
class Executer:
def __init__(self, mode: str, container_name: str):
def __init__(
self,
mode: str,
container_name: str,
namespace: str = None,
all_namespaces: bool = False,
):
"""[summary]
Args:
......@@ -18,12 +24,18 @@ class Executer:
"""
self.container_name = container_name
self.mode = mode
self.all_namespaces = all_namespaces
self.namespace = namespace
def _get_container_exec_command_base(self):
if self.mode.lower() == "docker":
return config.DOCKER_COMMAND + " exec"
elif self.mode.lower() == "kubernetes":
return f"{config.KUBECTL_COMMAND} exec {config.KUBECTL_NAMESPACE_PARAM}"
if self.all_namespaces:
self.namespace_arg = "--all-namespaces"
else:
self.namespace_arg = f"-n {self.namespace if self.namespace else config.KUBECTL_DEFAULT_NAMESPACE}"
return f"{config.KUBECTL_COMMAND} exec {self.namespace_arg}"
elif self.mode.lower() == "native":
return ""
else:
......@@ -34,7 +46,7 @@ class Executer:
def container_exec(
self, command: str, prefix: str = None, interactive: bool = False
):
exec_command = f"{prefix + ' ' if prefix is not None else ''} {self._get_container_exec_command_base()} {'-it' if interactive else ''} {self.container_name} {command}"
exec_command = f"{prefix + ' ' if prefix is not None else ''} {self._get_container_exec_command_base()} {'-it' if interactive else ''} {self.container_name}{' --' if self.mode.lower() == 'kubernetes' else ''} {command}"
return self.exec(exec_command)
@classmethod
......
CoDaBackup/label.py
View file @ 8e4604aa
from distutils import util
from typing import List, Dict, Type, Any, Union
import copy
import logging
from Configs import getConfig
from CoDaBackup.config import DEFAULT
from config import DEFAULT
config: DEFAULT = getConfig(config_classes=[DEFAULT])
......@@ -13,8 +13,6 @@ log = logging.getLogger("databasebackupper")
class Label:
_BASE_LABEL: str = config.DATABASE_CONTAINER_LABEL_BASE_KEY
def __init__(
self,
label: Union[str, Dict[str, str]],
......@@ -22,16 +20,22 @@ class Label:
possible_values: List = None,
default: Any = None,
info: str = None,
base_label_key: str = None,
):
if isinstance(label, Dict):
key = list(label.keys())[0]
val = list(label.values())[0]
else:
key, *val = label.split("=")
self.key: str = f"{self._BASE_LABEL}/{key}"
if type == bool and isinstance(val, str):
val = util.strtobool(val)
self.val: Union[str, int, bool] = type(val) if val else default
self.key: str = f"{base_label_key + '/' if base_label_key else ''}{key}"
try:
if type == bool and isinstance(val, str):
val = util.strtobool(val)
self.val: Union[str, int, bool] = type(val) if val else default
except:
raise ValueError(
f"Label '{key}={val}' expected to be of type '{type}'. Can not convert '{val}' to {type}"
)
self.type: Type = type
self.possible_values: List = possible_values
self.default: Union[str, int, bool] = default
......@@ -44,7 +48,10 @@ class Label:
return hash(self.key)
def __str__(self):
return f"{self.key}={self.val}"
if self.val is not None and self.val != "":
return f"{self.key}={self.val}"
else:
return self.key
def __repr__(self):
return f"<Label '{self.key}={self.val}'>"
......@@ -56,65 +63,81 @@ class ValidLabels:
bool,
default=False,
info="With this label you can enable or disable the backups for the container",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
database_type: Label = Label(
"type",
str,
["mysql", "postgres", "neo4j"],
info="Set the type of database running in the container",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
database_username: Label = Label(
"username", str, info="Username to access the to be backed up database"
"username",
str,
info="Username to access the to be backed up database",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
database_password: Label = Label(
"password", str, info="Password to access the to be backed up database"
"password",
str,
info="Password to access the to be backed up database",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
database_host: Label = Label(
"host",
str,
default="127.0.0.1",
info="Hostname/IP-Address to access the database. In most container environments this will be the default value '127.0.0.1'",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
database_names: Label = Label(
"databases",
str,
info="A single name or multiple names seperated by commata(','). Only databases with matching names will be backed up. If empty all databases will be backed up ",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
backup_dir: Label = Label(
"backup_dir",
str,
default="",
default=config.BACKUP_DIR,
info="Relative or absolute path to store the backups. Usally you can ignores this setting or if you have multiple database instances with databases that sharing names you have to define sub-directories to avoid interferences.",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
retention_daily: Label = Label(
"retention_daily",
int,
default=config.RETENTION_KEEP_NUMBER_OF_DAILY_BACKUPS,
info="How many daily backups should be kept?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
retention_weekly: Label = Label(
"retention_weekly",
int,
default=config.RETENTION_KEEP_NUMBER_OF_WEEKLY_BACKUPS,
info="How many weekly backups should be kept?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
retention_monthly: Label = Label(
"retention_monthly",
int,
default=config.RETENTION_KEEP_NUMBER_OF_MONTHLY_BACKUPS,
info="How many monthly backups should be kept?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
retention_yearly: Label = Label(
"retention_yearly",
int,
default=config.RETENTION_KEEP_NUMBER_OF_YEARLY_BACKUPS,
info="How many yearly backups should be kept?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
retention_manual: Label = Label(
"retention_manual",
int,
default=config.RETENTION_KEEP_NUMBER_OF_MANUAL_BACKUPS,
info="How many manual backups should be kept?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
@classmethod
......@@ -130,17 +153,20 @@ class ValidLabels:
if not a.startswith("__") and not callable(getattr(cls, a))
]
"""CAN BE REMOVED
@classmethod
def LabelFromString(cls, label_string: str) -> Label:
l = Label(label=label_string)
if cls.IsValid(l):
if cls.check_label_is_valid(l):
for validlabel in cls.iter():
if validlabel.key == l.key:
l.type = validlabel.type
l.possible_values = validlabel.possible_values
"""
"""CAN BE REMOVED
@classmethod
def IsValid(cls, label: Label) -> bool:
def check_label_is_valid(cls, label: Label) -> bool:
for validlabel in cls.iter():
if validlabel.key == label.key:
if not label.val or (
......@@ -149,6 +175,59 @@ class ValidLabels:
and label.val in validlabel.possible_values
):
return True
"""
@classmethod
def label_from_valid_label_as_template(cls, valid_label: Label, val):
return Label(
label={valid_label.key: val},
type=valid_label.type,
possible_values=valid_label.possible_values,
default=valid_label.default,
info=valid_label.info,
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
@classmethod
def valid_labels_from_dict(
cls, labels: Union[List[dict], dict], add_missing_default_labels: bool = False
) -> Dict[Label, Label]:
if isinstance(labels, list):
# convert list of labels to one dict representing all labels
labels_as_dict = {}
for label in labels:
labels_as_dict[list(label.keys())[0]] = list(label.values())[0]
labels = labels_as_dict
lbls = {}
for valid_label in cls.iter():
for key, val in labels.items():
if valid_label.key == key:
lbls[valid_label] = cls.label_from_valid_label_as_template(
valid_label=valid_label, val=val
)
break
else:
# ValidLabel was not in provided label dict. lets add it if desired by caller
if add_missing_default_labels:
lbls[valid_label] = valid_label
return lbls
@classmethod
def non_valid_labels_from_dict(
cls, labels: Union[List[dict], dict]
) -> Dict[Label, Label]:
if isinstance(labels, list):
# convert list of labels to one dict representing all labels
labels_as_dict = {}
for label in labels:
labels_as_dict[list(label.keys())[0]] = list(label.values())[0]
labels = labels_as_dict
lbls = {}
for key, val in labels.items():
if key not in [valid_label.key for valid_label in ValidLabels.iter()]:
label = Label(label={key: val})
lbls[label] = label
return lbls
@classmethod
def list_labels_human_readable(cls):
......
CoDaBackup/log.py
View file @ 8e4604aa
import logging
from Configs import getConfig
from CoDaBackup.config import DEFAULT
from config import DEFAULT
config: DEFAULT = getConfig(config_classes=[DEFAULT])
logging.basicConfig(
......
README.md
View file @ 8e4604aa
......@@ -53,6 +53,7 @@ docker exec mysql /usr/bin/mysql -N -h127.0.0.1 -uroot -pmysuperpw -e "\
CREATE TABLE IF NOT EXISTS coda_test.my_table(id INT AUTO_INCREMENT, firstname VARCHAR(32), PRIMARY KEY (id)); \
INSERT INTO coda_test.my_table(firstname) VALUES ('Anna'); \
INSERT INTO coda_test.my_table(firstname) VALUES ('Thomas'); \
commit;\
"
```
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment