Commit 3c299e5b authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

wip

parent 3bb20056
......@@ -70,6 +70,8 @@ class Backup:
class BackupManager:
"""Manages (List backup, Rotate existing backup, provide file names for new backups) of a single database instance"""
# mapping of how many backups are kept per retention type
retention_duration = {
RetentionType.DAILY: config.RETENTION_KEEP_NUMBER_OF_DAILY_BACKUPS,
......
......@@ -30,22 +30,15 @@ class BaseBackupper:
password: str,
databases: List[str] = None,
kubernetes_namespace: str = None,
kubernetes_all_namespaces: bool = False,
backups_base_path: Path = config.BACKUP_DIR,
):
self.executer = Executer(
mode=mode,
container_name=container_name,
namespace=kubernetes_namespace,
all_namespaces=kubernetes_all_namespaces,
)
base_path = Path(
PurePath(
backups_base_path,
kubernetes_namespace if kubernetes_namespace else "",
)
)
print(backups_base_path)
base_path = Path(backups_base_path)
self.manager = BackupManager(
base_path=base_path,
......@@ -91,7 +84,9 @@ class BaseBackupper:
databases: List[str] = None,
retention_type: RetentionType = RetentionType.MANUAL,
) -> List[Path]:
log.info("Start database backup...")
log.info(
f"Start backup of instance '{self.executer.container_name}' databases..."
)
if not databases:
# no certain databases specified. We backup all available databases
......@@ -102,6 +97,8 @@ class BaseBackupper:
databases = [databases]
backup_pathes = []
log.debug(f"Following databases will be backuped: {databases}")
if len(databases) == 0:
log.warning("No Databases found to backup...")
for database in databases:
if not database in self.list_databases():
msg = f"Database {database} does not exists @ '{self.host}' with user '{self.user}'"
......@@ -118,8 +115,6 @@ class BaseBackupper:
self.executer.container_exec(command=cmd)
backup_pathes.append(filepath)
else:
log.warning("No Databases found to backup...")
return backup_pathes
def get_backup_command(self, database_name, target_filepath):
......
......@@ -174,41 +174,42 @@ def backup_kubernetes(namespace, all_namespaces, target_dir):
for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
if pod.backup_labels[ValidLabels.enabled].val:
if pod.coda_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
pod.backup_labels[ValidLabels.database_type].val
pod.coda_labels[ValidLabels.database_type].val
]
target_dir_full = Path(
PurePath(
target_dir
if target_dir
else pod.coda_labels[ValidLabels.backup_dir].val,
pod.kubernetes_namespace,
pod.backup_name,
)
)
bu = BackupperClass(
mode="kubernetes",
container_name=pod.name,
host=pod.backup_labels[ValidLabels.database_host].val,
user=pod.backup_labels[ValidLabels.database_username].val,
password=pod.backup_labels[ValidLabels.database_password].val,
kubernetes_all_namespaces=all_namespaces,
host=pod.coda_labels[ValidLabels.database_host].val,
user=pod.coda_labels[ValidLabels.database_username].val,
password=pod.coda_labels[ValidLabels.database_password].val,
kubernetes_namespace=pod.kubernetes_namespace,
backups_base_path=Path(target_dir)
if target_dir
else pod.backup_labels[ValidLabels.backup_dir].val,
backups_base_path=target_dir_full,
)
bu.manager.retention_duration = {
RetentionType.DAILY: pod.backup_labels[ValidLabels.retention_daily].val,
RetentionType.WEEKLY: pod.backup_labels[
ValidLabels.retention_weekly
].val,
RetentionType.MONTHLY: pod.backup_labels[
RetentionType.DAILY: pod.coda_labels[ValidLabels.retention_daily].val,
RetentionType.WEEKLY: pod.coda_labels[ValidLabels.retention_weekly].val,
RetentionType.MONTHLY: pod.coda_labels[
ValidLabels.retention_monthly
].val,
RetentionType.YEARLY: pod.backup_labels[
ValidLabels.retention_yearly
].val,
RetentionType.MANUAL: pod.backup_labels[
ValidLabels.retention_manual
].val,
RetentionType.YEARLY: pod.coda_labels[ValidLabels.retention_yearly].val,
RetentionType.MANUAL: pod.coda_labels[ValidLabels.retention_manual].val,
}
if not pod.backup_labels[ValidLabels.database_names].val:
if not pod.coda_labels[ValidLabels.database_names].val:
databases = None
else:
databases = pod.backup_labels[ValidLabels.database_names].val.split(",")
databases = pod.coda_labels[ValidLabels.database_names].val.split(",")
backup_files = bu.backup(
databases=databases,
retention_type=RetentionType.DAILY,
......@@ -235,44 +236,44 @@ def backup_docker(target_dir):
for container in ContainerHelper.docker_get_container_to_be_backed_up(
describe=True
):
if container.backup_labels[ValidLabels.enabled].val:
if container.coda_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
container.backup_labels[ValidLabels.database_type].val
container.coda_labels[ValidLabels.database_type].val
]
bu = BackupperClass(
mode="docker",
container_name=container.name,
host=container.backup_labels[ValidLabels.database_host].val,
user=container.backup_labels[ValidLabels.database_username].val,
password=container.backup_labels[ValidLabels.database_password].val,
host=container.coda_labels[ValidLabels.database_host].val,
user=container.coda_labels[ValidLabels.database_username].val,
password=container.coda_labels[ValidLabels.database_password].val,
backups_base_path=Path(target_dir)
if target_dir
else container.backup_labels[ValidLabels.backup_dir].val,
else container.coda_labels[ValidLabels.backup_dir].val,
)
bu.manager.retention_duration = {
RetentionType.DAILY: container.backup_labels[
RetentionType.DAILY: container.coda_labels[
ValidLabels.retention_daily
].val,
RetentionType.WEEKLY: container.backup_labels[
RetentionType.WEEKLY: container.coda_labels[
ValidLabels.retention_weekly
].val,
RetentionType.MONTHLY: container.backup_labels[
RetentionType.MONTHLY: container.coda_labels[
ValidLabels.retention_monthly
].val,
RetentionType.YEARLY: container.backup_labels[
RetentionType.YEARLY: container.coda_labels[
ValidLabels.retention_yearly
].val,
RetentionType.MANUAL: container.backup_labels[
RetentionType.MANUAL: container.coda_labels[
ValidLabels.retention_manual
].val,
}
if not container.backup_labels[ValidLabels.database_names].val:
if not container.coda_labels[ValidLabels.database_names].val:
databases = None
else:
databases = container.backup_labels[
ValidLabels.database_names
].val.split(",")
databases = container.coda_labels[ValidLabels.database_names].val.split(
","
)
bu.backup(
databases=databases,
retention_type=RetentionType.DAILY,
......@@ -426,48 +427,32 @@ def restore_kubernetes_list(
source_dir,
):
t = [
{
"namespace": "default",
"database_containers": [
{
"name": "mariadb01",
"databases": [
{
"name": "coda_test",
"backups": [{"daily": [{"name": ""}], "weekly": []}],
}
],
}
],
}
]
backup_list: List = {}
for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
if pod_name and pod_name != pod.name:
continue
if pod.backup_labels[ValidLabels.enabled].val:
if pod.coda_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
pod.backup_labels[ValidLabels.database_type].val
pod.coda_labels[ValidLabels.database_type].val
]
bu = BackupperClass(
mode="kubernetes",
container_name=pod.name,
host=pod.backup_labels[ValidLabels.database_host].val,
user=pod.backup_labels[ValidLabels.database_username].val,
password=pod.backup_labels[ValidLabels.database_password].val,
host=pod.coda_labels[ValidLabels.database_host].val,
user=pod.coda_labels[ValidLabels.database_username].val,
password=pod.coda_labels[ValidLabels.database_password].val,
kubernetes_all_namespaces=all_namespaces,
kubernetes_namespace=pod.kubernetes_namespace,
backups_base_path=Path(source_dir)
if source_dir
else pod.backup_labels[ValidLabels.backup_dir].val,
else pod.coda_labels[ValidLabels.backup_dir].val,
)
if databases:
databases = databases.split(",")
elif pod.backup_labels[ValidLabels.database_names].val:
databases = pod.backup_labels[ValidLabels.database_names].val.split(",")
elif pod.coda_labels[ValidLabels.database_names].val:
databases = pod.coda_labels[ValidLabels.database_names].val.split(",")
else:
databases = bu.list_databases()
......
......@@ -9,7 +9,47 @@ from backup_manager import Backup, RetentionType
def format_backup_list_human_readable(backup_list: Dict):
pass
"""[summary]
Args:
backup_list (Dict): [description]
Returns:
[type]: [description]
backup_list input e.g.:
```json
[
{
"namespace": "default",
"database_containers": [
{
"name": "mariadb01",
"databases": [
{
"name": "coda_test",
"basedir": "/backups/default/coda_test",
"backups": [
{
"daily": [
{
"name": "sqlbackup_2022-01-26_10-18-49.sql",
"date": "2022-01-26 09:18:49",
"path": "daily/sqlbackup_2022-01-26_10-18-49.sql.gz",
"full_path": "/backups/default/coda_test/daily/sqlbackup_2022-01-26_10-18-49.sql.gz",
}
],
"weekly": [],
}
],
}
],
}
],
}
]
```
"""
# 🖥️
......
......@@ -19,19 +19,21 @@ log = logging.getLogger("databasebackupper")
class Container:
id: str
name: str
backup_labels: Dict[Label, Label]
backup_name: str
coda_labels: Dict[Label, Label]
other_labels: Dict[Label, Label]
desc: Dict = None
parent: Dict = None
kubernetes_namespace: str = None
@classmethod
def from_docker_inspect_dict(cls, container_inspect_result: dict):
def from_docker_inspect_dict(cls, container_inspect_result: Dict):
# todo validate correct dict type
return cls(
container = cls(
id=container_inspect_result[0]["Id"],
name=container_inspect_result[0]["Name"],
backup_labels=ValidLabels.valid_labels_from_dict(
backup_name=container_inspect_result[0]["Name"],
coda_labels=ValidLabels.valid_labels_from_dict(
container_inspect_result[0]["Config"]["Labels"],
add_missing_default_labels=True,
),
......@@ -40,14 +42,22 @@ class Container:
),
desc=container_inspect_result[0],
)
if (
ValidLabels.backup_name in container.coda_labels
and container.coda_labels[ValidLabels.backup_name].val
):
container.name = container.coda_labels[ValidLabels.backup_name].val
return container
@classmethod
def from_kubernetes_get_dict(cls, kubectl_get_item_result: dict):
def from_kubernetes_get_dict(cls, kubectl_get_item_result: Dict):
# todo validate correct item by "kind" field
return cls(
container = cls(
id=kubectl_get_item_result["metadata"]["uid"],
name=kubectl_get_item_result["metadata"]["name"],
backup_labels=ValidLabels.valid_labels_from_dict(
backup_name=kubectl_get_item_result["metadata"]["name"],
coda_labels=ValidLabels.valid_labels_from_dict(
kubectl_get_item_result["metadata"]["labels"],
add_missing_default_labels=True,
),
......@@ -57,6 +67,12 @@ class Container:
desc=kubectl_get_item_result,
kubernetes_namespace=kubectl_get_item_result["metadata"]["namespace"],
)
if (
ValidLabels.backup_name in container.coda_labels
and container.coda_labels[ValidLabels.backup_name].val
):
container.backup_name = container.coda_labels[ValidLabels.backup_name].val
return container
class ContainerHelper:
......@@ -143,10 +159,17 @@ class ContainerHelper:
workload_backup_config_labels = ValidLabels.valid_labels_from_dict(
workload["metadata"]["labels"], add_missing_default_labels=True
)
wl_pod.backup_labels = (
wl_pod.backup_labels | workload_backup_config_labels
wl_pod.coda_labels = (
wl_pod.coda_labels | workload_backup_config_labels
)
if (
ValidLabels.backup_name in wl_pod.coda_labels
and not wl_pod.coda_labels[ValidLabels.backup_name].val
):
# if the backup subdir is not defined by a label we override the default (pod name) with the parent workload name to get a more consistent name
wl_pod.backup_name = workload["metadata"]["name"]
# Attach the parent workload data just for good measure... not in any use yet. maybe we can delete this step
wl_pod.parent = workload
pods.append(wl_pod)
......
......@@ -14,7 +14,6 @@ class Executer:
mode: str,
container_name: str,
namespace: str = None,
all_namespaces: bool = False,
):
"""[summary]
......@@ -24,7 +23,6 @@ class Executer:
"""
self.container_name = container_name
self.mode = mode
self.all_namespaces = all_namespaces
self.namespace = namespace
def _get_container_exec_command_base(self):
......
......@@ -88,10 +88,17 @@ class ValidLabels:
info="With this label you can enable or disable the backups for the container",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
backup_name: Label = Label(
"backup_name",
str,
default=None,
info="With this label you can define the sub-directory name for the the specific database. If not set, CoDaBackup will determine the name by a containers docker name or kubernetes deployment name.",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
database_type: Label = Label(
"type",
str,
["mysql", "postgres", "neo4j"],
possible_values=["mysql", "postgres", "neo4j"],
info="Set the type of database running in the container",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
......@@ -124,7 +131,7 @@ class ValidLabels:
"backup_dir",
str,
default=config.BACKUP_DIR,
info="Relative or absolute path to store the backups. Usally you can ignores this setting or if you have multiple database instances with databases that sharing names you have to define sub-directories to avoid interferences.",
info="Relative or absolute path to store the backups. Usally you can ignores this setting, you only need to specifiy this if you, for example, want to store your databases on different mounts/directories on your backup host",
base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
)
retention_daily: Label = Label(
......
......@@ -9,4 +9,4 @@ logging.basicConfig(
handlers=[logging.StreamHandler()],
level=config.LOG_LEVEL,
)
log = logging.getLogger("databasebackupper")
log: logging.Logger = logging.getLogger("databasebackupper")
......@@ -2,7 +2,7 @@
```bash
docker exec mysql /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@localhost -c "CREATE DATABASE coda_ps_test;" && /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@localhost -c "\
docker exec postgres /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@localhost -c "CREATE DATABASE coda_ps_test;" && /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@localhost -c "\
BEGIN; \
create schema coda_ps_test_scheme; \
CREATE TABLE IF NOT EXISTS coda_ps_test_scheme.my_table(id integer PRIMARY KEY, firstname VARCHAR(32)); \
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment