Commit 87372661 authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

wip

parent 04d62b56
......@@ -23,6 +23,21 @@ class BaseBackupper:
backup_file_suffix: str = None
ignore_databases: List[str] = []
def get_list_database_command() -> List[str]:
raise NotImplementedError
def get_list_user_command() -> List[str]:
raise NotImplementedError
def get_backup_command(self, database_name, target_filepath) -> str:
raise NotImplementedError
def get_restore_command(self, database_name, source_filepath) -> str:
raise NotImplementedError
def get_create_database_and_user_if_not_exists_command(self, database_name) -> str:
raise NotImplementedError
def __init__(self, db_container: Container, target_base_dir: Path = None):
self.executer = Executer(
mode=db_container.mode,
......@@ -70,6 +85,7 @@ class BaseBackupper:
self.host = db_container.coda_labels[ValidLabels.database_host].val
self.user = db_container.coda_labels[ValidLabels.database_username].val
self.password = db_container.coda_labels[ValidLabels.database_password].val
self.container = db_container
databases = db_container.coda_labels[ValidLabels.database_names].val
if isinstance(databases, str):
self.databases = databases.split(",")
......@@ -87,12 +103,9 @@ class BaseBackupper:
f"{self.backup_file_suffix}.gz" if val else self.backup_file_suffix
)
def get_list_database_command():
raise NotImplementedError
def list_databases(self, show_all: bool = False) -> List[str]:
result = self.executer.container_exec(command=self.get_list_database_command())
return [
database.strip()
for database in result.decode("utf-8").splitlines()
......@@ -100,6 +113,15 @@ class BaseBackupper:
and database.strip() != ""
]
def list_users(self) -> List[str]:
result = self.executer.container_exec(command=self.get_list_user_command())
return [
user.strip()
for user in result.decode("utf-8").splitlines()
if user.strip() != ""
]
def backup(
self,
databases: List[str] = None,
......@@ -138,9 +160,6 @@ class BaseBackupper:
backup_pathes.append(filepath)
return backup_pathes
def get_backup_command(self, database_name, target_filepath):
raise NotImplementedError
def _find_backup_by_name(self, database_name: str, backup_name: str) -> Dict:
backup: Backup = None
for retention_type, backups in self.manager.list_backups(
......@@ -155,19 +174,16 @@ class BaseBackupper:
)
return backup
def get_restore_command(self, database_name, source_filepath):
raise NotImplementedError
def restore(self, database: str, backup_name: str, dry_run: bool = False):
log.debug(
f"Restore order for database '{database}'. backup_name: '{backup_name}'"
)
backup: Dict = self._find_backup_by_name(database, backup_name)
backup: Backup = self._find_backup_by_name(database, backup_name)
log.debug(f"Found backup to restore: {backup}")
com = self.get_restore_command(database_name=database)
self.executer.container_exec(
command=com,
prefix=f"{'zcat' if self.compress_backup else 'cat'} <{backup['path'].absolute()} |",
prefix=f"{'zcat' if self.compress_backup else 'cat'} <{backup.path.absolute()} |",
dry_run=dry_run,
)
......@@ -183,9 +199,10 @@ class MySQLBackupper(BaseBackupper):
"mysql",
"sys",
]
custom_extra_options: Dict = {"backup": ["--add-drop-table", "--add-drop-database"]}
def get_backup_command(self, database_name, target_filepath):
return f"{self.bin_dump} {'--add-drop-table' if self.add_drop_table else ''} --databases {'--add-drop-database' if self.add_drop_database else ''} -h{self.host} -u{self.user} --password={self.password} {database_name} {'| gzip -9' if self.compress_backup else ''} >{target_filepath}"
return f"{self.bin_dump} {' '.join(self.custom_extra_options['backup'])} -h{self.host} -u{self.user} --password={self.password} --databases {database_name} {'| gzip -9' if self.compress_backup else ''} >{target_filepath}"
def get_restore_command(self, database_name):
return f"""{self.bin_cmd} -u{self.user} -p{self.password} {database_name}"""
......@@ -206,8 +223,11 @@ class PostgresBackupper(BaseBackupper):
"template1",
]
# https://www.postgresql.org/docs/10/app-pgdump.html
custom_extra_options: Dict = {"backup": ["--clean"]}
def get_backup_command(self, database_name, target_filepath):
return f"env PGPASSWORD={self.password} {self.bin_dump} {'--clean' if self.add_drop_database else ''} -h {self.host} -U {self.user} {database_name} {'| gzip -9' if self.compress_backup else ''} >{target_filepath}"
return f"env PGPASSWORD={self.password} {self.bin_dump} {' '.join(self.custom_extra_options['backup'])} -h {self.host} -U {self.user} -d {database_name} {'| gzip -9' if self.compress_backup else ''} >{target_filepath}"
def get_restore_command(self, database_name):
return f"""env PGPASSWORD={self.password} {self.bin_cmd} -U {self.user} -d {database_name}"""
......@@ -215,6 +235,16 @@ class PostgresBackupper(BaseBackupper):
def get_list_database_command(self):
return f"""env PGPASSWORD={self.password} {self.bin_cmd} -U {self.user} -h {self.host} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;" """
def get_list_user_command(self):
return f"""env PGPASSWORD={self.password} {self.bin_cmd} -U {self.user} -h {self.host} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;" """
def get_create_database_and_user_if_not_exists_command(self, database_name):
admin_user = self.container.coda_label[ValidLabels.auto_create_user_name].val
admin_pw = self.container.coda_label[ValidLabels.auto_create_user_password].val
users = self.executer.
databases = self.list_databases()
return f""""""
class Neo4jOnlineBackupper(BaseBackupper):
bin_dump: str = "/usr/bin/neo4j-admin backup"
......
......@@ -28,7 +28,11 @@ from backupper import (
Neo4jOfflineBackupper,
Neo4jOnlineBackupper,
)
from cli_helper import list_backups, format_n_output_backup_list
from cli_helper import (
list_backups,
format_n_output_backup_list,
click_check_required_params_not_None,
)
from log import log
from label import ValidLabels, Label
......@@ -359,13 +363,11 @@ def restore_now(
)
@click.option(
"--workload-name",
required=True,
type=str,
help="Name of the pod/container the to be restored database runs in?",
)
@click.option(
"--backup-name",
required=True,
type=str,
help="Name of the backup (find via coda-restore kubernetes list)?",
)
......@@ -373,15 +375,20 @@ def restore_now(
"--database-name",
type=str,
help="name of the database instance to be restored",
required=True,
)
def restore_kubernetes(ctx, namespace, workload_name, backup_name, database_name):
def restore_kubernetes(
ctx: click.Context, namespace, workload_name, backup_name, database_name
):
if ctx.invoked_subcommand is None:
# you are here
click_check_required_params_not_None(
ctx, ["workload_name", "backup_name", "database_name"]
)
workload = next(
(
wl
for wl in ContainerHelper.kubernetes_get_workloads(namespace=namespace)
for wl in ContainerHelper.kubernetes_get_workloads(
namespace=namespace, describe=True
)
if wl["metadata"]["name"] == workload_name
),
None,
......@@ -398,7 +405,7 @@ def restore_kubernetes(ctx, namespace, workload_name, backup_name, database_name
pod.coda_labels[ValidLabels.database_type].val
]
bu = BackupperClass(db_container=pod)
bu.restore(database=database_name, backup_name=backup_name, dry_run=True)
bu.restore(database=database_name, backup_name=backup_name, dry_run=False)
@restore_kubernetes.command(name="list")
......
from typing import Dict, List, Union, Type, Callable
from datetime import datetime, timedelta
import click
import time
import humanize
from pathlib import Path, PurePath
......@@ -286,3 +287,27 @@ def format_n_output_backup_list(data, output_format):
return output
elif output_format == "dict":
return data
def click_check_required_params_not_None(
context: click.Context, required_names: List[str]
):
missing_opts = []
for param, val in context.params.items():
missing_opt = next(
(
option
for option in context.command.get_params(context)
if option.name == param
and option.name in required_names
and val is None
),
None,
)
if missing_opt is not None:
missing_opts.append(missing_opt.opts[0])
if missing_opts:
click.echo(context.get_help() + "\n")
raise click.ClickException(f"missing option(s): {', '.join(missing_opts)}")
......@@ -29,6 +29,7 @@ class Container:
@classmethod
def from_docker_inspect_dict(cls, container_inspect_result: Dict):
# todo validate correct dict type
container = cls(
mode="docker",
......@@ -144,19 +145,7 @@ class ContainerHelper:
pods: List[Container] = []
for workload in workloads:
for wl_pod in cls.kubernetes_get_pods_by_workload(workload):
# we merge the workloads backup config labels into the pod label. usally only the kubernetes workload will have the backup config labels
workload_backup_config_labels = ValidLabels.valid_labels_from_dict(
workload["metadata"]["labels"], add_missing_default_labels=True
)
wl_pod.coda_labels = wl_pod.coda_labels | workload_backup_config_labels
if (
ValidLabels.backup_name in wl_pod.coda_labels
and not wl_pod.coda_labels[ValidLabels.backup_name].val
):
# if the backup subdir is not defined by a label we override the default (pod name) with the parent workload name to get a more consistent name
wl_pod.backup_name = workload["metadata"]["name"]
wl_pod = cls._attach_parent_workload_metadata_to_pod(wl_pod, workload)
pods.append(wl_pod)
return pods
......@@ -189,11 +178,30 @@ class ContainerHelper:
all_namespaces=False,
describe=True,
):
# Attach the parent workload data just for good measure... not in any use yet. maybe we can delete this step
wl_pod.parent = workload
wl_pod = cls._attach_parent_workload_metadata_to_pod(wl_pod, workload)
pods.append(wl_pod)
return pods
@classmethod
def _attach_parent_workload_metadata_to_pod(
cls, pod: Container, parent_workload: Dict
) -> Container:
workload_backup_config_labels = ValidLabels.valid_labels_from_dict(
parent_workload["metadata"]["labels"], add_missing_default_labels=True
)
pod.coda_labels = pod.coda_labels | workload_backup_config_labels
if (
ValidLabels.backup_name in pod.coda_labels
and not pod.coda_labels[ValidLabels.backup_name].val
):
# if the backup subdir is not defined by a label we override the default (pod name) with the parent workload name to get a more consistent name
pod.backup_name = parent_workload["metadata"]["name"]
# Attach the parent workload data just for good measure... not in any use yet. maybe we can delete this step
pod.parent = parent_workload
return pod
@classmethod
def kubernetes_get_workloads(
cls,
......
......@@ -30,7 +30,7 @@ class Executer:
return config.DOCKER_COMMAND + " exec"
elif self.mode.lower() == "kubernetes":
self.namespace_arg = f"-n {self.namespace if self.namespace else config.KUBECTL_DEFAULT_NAMESPACE}"
return f"{config.KUBECTL_COMMAND} exec {self.namespace_arg}"
return f"{config.KUBECTL_COMMAND} exec -i {self.namespace_arg}"
elif self.mode.lower() == "native":
return ""
else:
......
......@@ -105,7 +105,7 @@ class ValidLabels:
database_username: Label = Label(
"username",
str,
info="Username to access the to be backed up database",
info="Username to access the to be backed up database(s)",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
database_password: Label = Label(
......@@ -170,6 +170,49 @@ class ValidLabels:
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
auto_create_enables: Label = Label(
"auto-create",
bool,
default=False,
info="If the database does not exists, create it?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
auto_create_user_name: Label = Label(
"auto-create-user",
str,
default=None,
info="The user to create a missing database (must have the priviledge to create a database)?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
auto_create_user_password: Label = Label(
"auto-create-user-password",
bool,
default=None,
info="The password for the user that is creating a missing database?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
auto_create_encoding: Label = Label(
"auto-create-encoding",
str,
default=None,
info="Which database encoding should be used?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
auto_create_collation: Label = Label(
"auto-create-collation",
str,
default=None,
info="Which database encoding should be used?",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
auto_create_databases: Label = Label(
"auto-create-databases",
str,
default=None,
info="Name of the databases to be created",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
@classmethod
def iter(cls) -> List[Label]:
"""iterate all existing labels
......
......@@ -83,6 +83,7 @@ Thats it. We now have a directory `./backups/` in front of us, with all database
* (WIP) Restore Wizard
* (WIP) Neo4j support
* (WIP) Auto database creation if not exists
* (Planned) Database auth via Docker/kubernetes Secrets
* (Planned) Create database if not existent via label conf (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Planned) Email notification
......@@ -102,6 +103,15 @@ Thats it. We now have a directory `./backups/` in front of us, with all database
* implement RETENTION_ON_COLLISION_KEEP_NEWEST_BACKUP
* Restore CLI API
* write docs
* Timestamp is not in current timezone?
# limitations
## Auto database creation
* All databases in one instance/container must have the same encoding and collation. Atm there is no way of configuring this on a per database level
* All databases in one instance/container must share on backup user. Atm there is no way of having multiple users to access different databases in one instance/container
# Docker Volumes Pathes
`/var/run/docker.sock`
`/.kube/config`
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment