Commit 857a3015 authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

wip

parent 1c1a8341
......@@ -35,7 +35,7 @@ class BaseBackupper:
databases: List[str] = None,
):
self.executer = executer
self.retention = BackupManager(
self.manager = BackupManager(
base_path=self.dir_backups,
backup_file_prefix=self.backup_file_prefix,
backup_file_suffix=f"{self.backup_file_suffix}.gz"
......@@ -57,7 +57,7 @@ class BaseBackupper:
@compress_backup.setter
def compress_backup(self, val: bool):
self.__compress_backup = val
self.retention.backup_file_suffix = (
self.manager.backup_file_suffix = (
f"{self.backup_file_suffix}.gz" if val else self.backup_file_suffix
)
......@@ -98,7 +98,7 @@ class BaseBackupper:
else:
log.error(msg)
return
filepath = self.retention.get_new_backup_path(
filepath = self.manager.get_new_backup_path(
database_name=database, retention_type=retention_type
)
cmd = self.get_backup_command(database, filepath)
......@@ -110,9 +110,7 @@ class BaseBackupper:
def _find_backup_by_name(self, database: str, backup_name: str) -> Dict:
backup: Dict = None
for retention_type, backup_list in self.retention.list_backups(
database
).items():
for retention_type, backup_list in self.manager.list_backups(database).items():
for backup_ in backup_list:
if backup_["backup_name"] == backup_name:
backup = backup_
......
......@@ -39,6 +39,11 @@ database_type_backupper_mapping: Dict[str, Type[BaseBackupper]] = {
@click.group()
def cli_root():
pass
@cli_root.group(name="backup")
@click.option(
"--debug/--no-debug",
help="Enable debug logging",
......@@ -55,8 +60,8 @@ def backup_cli(debug):
"--mode",
default="Docker",
prompt="How is your DB running?",
help="Environment the database is running in: 'Docker', 'k8s', 'native'",
type=click.Choice(["docker", "k8s", "native"], case_sensitive=False),
help="Environment the database is running in: 'Docker', 'kubernetes', 'native'",
type=click.Choice(["docker", "kubernetes", "native"], case_sensitive=False),
)
@click.option(
"--container-identifier",
......@@ -130,7 +135,7 @@ def backup_now(
)
@backup_cli.command()
@backup_cli.command(name="kubernetes")
@click.option(
"--namespace",
"-n",
......@@ -144,7 +149,7 @@ def backup_now(
default=False,
help="If we should search for pods, to be backed up, in all namespaces. When used `--namespace` will be ignored.",
)
def kubernetes(namespace, all_namespaces):
def backup_kubernetes(namespace, all_namespaces):
"""Backup databases in a kubernetes environment"""
from container_helper import ContainerHelper
......@@ -152,14 +157,16 @@ def kubernetes(namespace, all_namespaces):
os.environ["CONFIGS_KUBECTL_NAMESPACE_PARAM"] = "--all-namespaces"
else:
os.environ["CONFIGS_KUBECTL_NAMESPACE_PARAM"] = f"-n {namespace}"
for podname in ContainerHelper.k8s_get_pod_names_to_be_backed_up():
config_labels = ContainerHelper.k8s_get_config_by_labels(pod_name=podname)
for podname in ContainerHelper.kubernetes_get_pod_names_to_be_backed_up():
config_labels = ContainerHelper.kubernetes_get_config_by_labels(
pod_name=podname
)
if config_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
config_labels[ValidLabels.database_type].val
]
bu = BackupperClass(
executer=Executer("k8s", podname),
executer=Executer("kubernetes", podname),
host=config_labels[ValidLabels.database_host].val,
user=config_labels[ValidLabels.database_username].val,
password=config_labels[ValidLabels.database_password].val,
......@@ -172,12 +179,12 @@ def kubernetes(namespace, all_namespaces):
databases=databases,
retention_type=RetentionType.DAILY,
)
bu.retention.rotate_existing_backups()
bu.manager.rotate_existing_backups()
@backup_cli.command()
@backup_cli.command(name="docker")
@click.argument("target-dir", default="backup/")
def docker(target_dir):
def backup_docker(target_dir):
"""Backup databases in a docker environment"""
for container_name in ContainerHelper.docker_get_container_to_be_backed_up():
......@@ -194,31 +201,31 @@ def docker(target_dir):
user=config_labels[ValidLabels.database_username].val,
password=config_labels[ValidLabels.database_password].val,
)
bu.retention.base_path = Path(target_dir)
bu.manager.base_path = Path(target_dir)
if not config_labels[ValidLabels.database_names].val:
databases = None
else:
databases = config_labels[ValidLabels.database_names].val.split(",")
bu.retention.retention_duration = {
bu.manager.retention_duration = {
RetentionType.DAILY: config_labels[ValidLabels.retention_daily].val,
RetentionType.WEEKLY: config_labels[ValidLabels.retention_weekly].val,
RetentionType.MONTHLY: config_labels[ValidLabels.retention_monthly].val,
RetentionType.YEARLY: config_labels[ValidLabels.retention_yearly].val,
RetentionType.MANUAL: config_labels[ValidLabels.retention_manual].val,
}
bu.retention.base_path = Path(
bu.manager.base_path = Path(
PurePath(
bu.retention.base_path, config_labels[ValidLabels.backup_subdir].val
bu.manager.base_path, config_labels[ValidLabels.backup_dir].val
)
)
bu.backup(
databases=databases,
retention_type=RetentionType.DAILY,
)
bu.retention.rotate_existing_backups()
bu.manager.rotate_existing_backups()
@click.group()
@cli_root.group(name="restore")
@click.option(
"--debug/--no-debug",
help="Enable debug logging",
......@@ -235,8 +242,8 @@ def restore_cli(debug):
"--mode",
default="Docker",
prompt="How is your DB running?",
help="Environment the database is running in: 'Docker', 'k8s', 'native'",
type=click.Choice(["docker", "k8s", "native"], case_sensitive=False),
help="Environment the database is running in: 'Docker', 'kubernetes', 'native'",
type=click.Choice(["docker", "kubernetes", "native"], case_sensitive=False),
)
@click.option(
"--container-identifier",
......@@ -289,10 +296,11 @@ def restore_now(
database_name,
backup_name,
):
pass
raise NotImplementedError
@restore_cli.command()
@restore_cli.group(invoke_without_command=True, name="kubernetes")
@click.pass_context
@click.option(
"--namespace",
"-n",
......@@ -316,9 +324,61 @@ def restore_now(
hide_input=False,
help="Name of the backup (find via coda-restore kubernetes list)?",
)
def kubernetes():
pass
def restore_kubernetes(ctx, namespace, all_namespaces, pod_name, backup_name):
if ctx.invoked_subcommand is None:
click.echo("DO restore")
@restore_kubernetes.command(name="list")
@click.option(
"--pod",
default=None,
prompt="List backups of which container",
help="User to access the database?",
)
@click.option(
"--databases", default=None, help="List only these databases (seperated by comma)"
)
@click.option("--yaml", default=False, help="Output list machine readable")
@click.option("--base-path", default=None, help="Define base dir of backups")
def restore_kubernetes_list(pod, databases, yaml, base_path):
backup_list: Dict = []
for container_name in ContainerHelper.kubernetes_get_pod_names_to_be_backed_up():
if pod and pod != container_name:
continue
config_labels = ContainerHelper.kubernetes_get_config_by_labels(
container_name=container_name
)
if config_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
config_labels[ValidLabels.database_type].val
]
bu = BackupperClass(
executer=Executer("kubernetes", container_name),
host=config_labels[ValidLabels.database_host].val,
user=config_labels[ValidLabels.database_username].val,
password=config_labels[ValidLabels.database_password].val,
)
if base_path:
bu.manager.base_path = Path(base_path)
if databases:
databases = databases.split(",")
elif config_labels[ValidLabels.database_names].val:
databases = config_labels[ValidLabels.database_names].val.split(",")
else:
databases = None
bu.manager.base_path = Path(
PurePath(
bu.manager.base_path, config_labels[ValidLabels.backup_dir].val
)
)
# YOU ARE HERE. list databases bakcups
backup_list[f"{container_name}/{da}"]
click.echo(bu.manager.list_backups())
if __name__ == "__main__":
backup_cli()
cli_root()
......@@ -16,23 +16,23 @@ class DEFAULT(ConfigBase):
BACKUP_DIR: str = "./backups"
# execution mode can be either terminal, docker or k8s
# execution mode can be either terminal, docker or kubernetes
EXECUTION_MODE: str = "docker"
# The docker or k8s label key to find databases containers to be backed up. (Value of such a label can be "mysql" or "postgres")
# The docker or kubernetes label key to find databases containers to be backed up. (Value of such a label can be "mysql" or "postgres")
# Info on docker labels https://docs.docker.com/config/labels-custom-metadata/
# Info on k8s labels: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
# Info on kubernetes labels: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
DATABASE_CONTAINER_LABEL_BASE_KEY: str = "backup.dzd-ev.de"
# If you dont want/can controll the backupper via container labels you can also specify a container name to be backed up
DATABASE_CONTAINER_NAME: str = None
# WIP: we need a way to map auths to database identifiers, whcih can be names or labels
# Databases access: A dict/json of database authenikation. key must be container name for docker or service/deployment name for k8s
# Databases access: A dict/json of database authenikation. key must be container name for docker or service/deployment name for kubernetes
# example:
# {"mysql01":{"user":"root","password":"supersecretpw","host":"127.0.0.1"}}
# the key "host" is optional and will default to "127.0.0.1"
# Todo: Create possibility to provide DB auth via docker or k8s secrets
# Todo: Create possibility to provide DB auth via docker or kubernetes secrets
DATABASES_ACCESS: Dict[str, Dict[str, str]] = {}
# TIMESTAMP_FORMAT format code -> https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes
......
......@@ -13,7 +13,7 @@ log = logging.getLogger("databasebackupper")
class ContainerHelper:
@classmethod
def k8s_get_pod_names_to_be_backed_up(cls, label: str = None) -> List[str]:
def kubernetes_get_pod_names_to_be_backed_up(cls, label: str = None) -> List[str]:
# kubectl get pods -n webapps-extern --selector='workload.user.cattle.io/workloadselector=deployment-webapps-extern-gitlab'
if not label:
label = ValidLabels.enabled.key
......@@ -44,7 +44,7 @@ class ContainerHelper:
return pods
@classmethod
def k8s_get_pod_sql_type_by_label(cls, pod_name: str, label_key: str = None):
def kubernetes_get_pod_sql_type_by_label(cls, pod_name: str, label_key: str = None):
if not label_key:
label_key = config.DATABASE_CONTAINER_LABEL_BASE_KEY
pod: Dict = json.loads(
......@@ -57,7 +57,7 @@ class ContainerHelper:
return val
@classmethod
def k8s_get_config_by_labels(cls, pod_name: str) -> Dict[Label, Label]:
def kubernetes_get_config_by_labels(cls, pod_name: str) -> Dict[Label, Label]:
# config.DATABASE_CONTAINER_LABEL_BASE_KEY
container_labels: Dict = json.loads(
Executer.exec(
......
......@@ -13,7 +13,7 @@ class Executer:
"""[summary]
Args:
mode (str): ['Docker','k8s', 'native'] - Which environment the executer works again
mode (str): ['Docker','kubernetes', 'native'] - Which environment the executer works again
container_name (str): name/id of the container/pod we will execute the backup on/in
"""
self.container_name = container_name
......@@ -22,13 +22,13 @@ class Executer:
def _get_container_exec_command_base(self):
if self.mode.lower() == "docker":
return config.DOCKER_COMMAND + " exec"
elif self.mode.lower() == "k8s":
elif self.mode.lower() == "kubernetes":
return f"{config.KUBECTL_COMMAND} exec {config.KUBECTL_NAMESPACE_PARAM}"
elif self.mode.lower() == "native":
return ""
else:
raise ValueError(
"Unknown executer mode '{mode}'. Expected on string of ['Docker','k8s', 'native']"
"Unknown executer mode '{mode}'. Expected on string of ['Docker','kubernetes', 'native']"
)
def container_exec(
......
from typing import List, Dict, Type, Any
from distutils import util
from typing import List, Dict, Type, Any, Union
import logging
from Configs import getConfig
from CoDaBackup.config import DEFAULT
......@@ -19,13 +21,17 @@ class Label:
type: Type,
possible_values: List = None,
default: Any = None,
info: str = None,
):
key, *val = label.split("=")
self.key = f"{self._BASE_LABEL}/{key}"
self.val = val if val else default
self.type = type
self.possible_values = possible_values
self.default = default
self.key: str = f"{self._BASE_LABEL}/{key}"
if type == bool and isinstance(val, str):
val = util.strtobool(val)
self.val: Union[str, int, bool] = type(val) if val else default
self.type: Type = type
self.possible_values: List = possible_values
self.default: Union[str, int, bool] = default
self.info: str = info
def __eq__(self, other: "Label"):
return type(other) is type(self) and self.key == other.key
......@@ -41,33 +47,70 @@ class Label:
class ValidLabels:
enabled: Label = Label("enabled", str, ["false", "true"], "false")
database_type: Label = Label("type", str, ["mysql", "postgres", "neo4j"])
database_username: Label = Label("username", str)
database_password: Label = Label("password", str)
database_host: Label = Label("host", str, default="127.0.0.1")
database_names: Label = Label("databases", str)
backup_subdir: Label = Label("backup_subdir", str, default="")
enabled: Label = Label(
"enabled",
bool,
default=False,
info="With this label you can enable or disable the backups for the container",
)
database_type: Label = Label(
"type",
str,
["mysql", "postgres", "neo4j"],
info="Set the type of database running in the container",
)
database_username: Label = Label(
"username", str, info="Username to access the to be backed up database"
)
database_password: Label = Label(
"password", str, info="Password to access the to be backed up database"
)
database_host: Label = Label(
"host",
str,
default="127.0.0.1",
info="Hostname/IP-Address to access the database. In most container environments this will be the default value '127.0.0.1'",
)
database_names: Label = Label(
"databases",
str,
info="A single name or multiple names seperated by commata(','). Only databases with matching names will be backed up. If empty all databases will be backed up ",
)
backup_dir: Label = Label(
"backup_dir",
str,
default="",
info="Relative or absolute path to store the backups. Usally you can ignores this setting or if you have multiple database instances with databases that sharing names you have to define sub-directories to avoid interferences.",
)
retention_daily: Label = Label(
"retention_daily", int, default=config.RETENTION_KEEP_NUMBER_OF_DAILY_BACKUPS
"retention_daily",
int,
default=config.RETENTION_KEEP_NUMBER_OF_DAILY_BACKUPS,
info="How many daily backups should be kept?",
)
retention_weekly: Label = Label(
"retention_weekly", int, default=config.RETENTION_KEEP_NUMBER_OF_WEEKLY_BACKUPS
"retention_weekly",
int,
default=config.RETENTION_KEEP_NUMBER_OF_WEEKLY_BACKUPS,
info="How many weekly backups should be kept?",
)
retention_monthly: Label = Label(
"retention_monthly",
int,
default=config.RETENTION_KEEP_NUMBER_OF_MONTHLY_BACKUPS,
info="How many monthly backups should be kept?",
)
retention_yearly: Label = Label(
"retention_yearly",
int,
default=config.RETENTION_KEEP_NUMBER_OF_YEARLY_BACKUPS,
info="How many yearly backups should be kept?",
)
retention_manual: Label = Label(
"retention_manual",
int,
default=config.RETENTION_KEEP_NUMBER_OF_MANUAL_BACKUPS,
info="How many manual backups should be kept?",
)
@classmethod
......@@ -102,3 +145,18 @@ class ValidLabels:
and label.val in validlabel.possible_values
):
return True
@classmethod
def list_labels_human_readable(cls):
# ToDo: make this better readable for humans
s = ""
for label in cls.iter():
s += f"\n{label.key} \n"
s += f" type: {label.type.__name__}\n"
if label.default is not None:
s += f" default: {label.default}\n"
if label.possible_values:
s += f" possible_values: {label.possible_values}\n"
if label.info:
s += f" {label.info}\n"
return s
......@@ -2,20 +2,20 @@ import logging
from Configs import getConfig
from config import DEFAULT
from backupper import DatabaseBackupper
from executer import Executer
from backup_manager import BackupManager, RetentionType
from container_helper import ContainerHelper
from log import log
config: DEFAULT = getConfig(config_classes=[DEFAULT])
from label import ValidLabels
if __name__ == "__main__":
# r = BackupManager(base_path="/tmp")
# log.info(r.list_backups())
# exit()
"""
manager = Executer("docker", "mysql")
mysql = DatabaseBackupper(
executer=manager, host="127.0.0.1", user="root", password="498zrthfwejfef"
......@@ -25,3 +25,5 @@ if __name__ == "__main__":
mysql.backup(retention_type=RetentionType.DAILY)
mysql.retention.rotate_existing_backups()
# mysql.restore("redcado","mysqlbackup_2021-12-29_14-19-40")
"""
print(ValidLabels.list_labels_human_readable())
......@@ -10,7 +10,7 @@ Status: Alpha
CoDaBackup helps you to automate backups, if your databases are running in container environment (kubernetes, docker)
It relies heavily on configuration by labels ([docker-labels](https://docs.docker.com/config/labels-custom-metadata/), [k8s-labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/))
It relies heavily on configuration by labels ([docker-labels](https://docs.docker.com/config/labels-custom-metadata/), [kubernetes-labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/))
This means you only have to attach the right labels to your database containers and they will be included in your daily backup.
......@@ -83,10 +83,11 @@ Thats it. We now have a directory `./backups/` in front of us, with all database
* (WIP) Restore Wizard
* (WIP) Neo4j support
* (WIP) Database auth via Docker/K8s Secrets
* (WIP) Database auth via Docker/kubernetes Secrets
* (Planned) Create database if not existent via label conf (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Planned) Email notification
* (Planned) Support of docker and kubernetes secrets for providing database auth
* (Planned) Docker Event listener / Kubectl hooks to react to container started/stopped
* (Idea) restore by label (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Idea) Matrix notifications
* (Idea) [Podman](https://podman.io/) support. Your help is greatly appreciated. Should be easy or maybe no work at all
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment