Commit 87aef0ea authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

wip

parent 3c299e5b
......@@ -5,6 +5,8 @@ from pathlib import PurePath, Path
from config import DEFAULT
from executer import Executer
from backup_manager import BackupManager, RetentionType
from container_helper import Container
from label import ValidLabels, Label
config: DEFAULT = getConfig(config_classes=[DEFAULT])
# log = logging.getLogger("BaseBackupper")
......@@ -21,37 +23,39 @@ class BaseBackupper:
backup_file_suffix: str = None
ignore_databases: List[str] = []
def __init__(
self,
mode,
container_name,
host: str,
user: str,
password: str,
databases: List[str] = None,
kubernetes_namespace: str = None,
backups_base_path: Path = config.BACKUP_DIR,
):
def __init__(self, db_container: Container, target_base_dir: Path = None):
self.executer = Executer(
mode=mode,
container_name=container_name,
namespace=kubernetes_namespace,
mode=db_container.mode,
container_name=db_container.name,
namespace=db_container.kubernetes_namespace,
)
# generate target path for backups
target_dir = Path(
PurePath(
target_base_dir
if target_base_dir
else db_container.coda_labels[ValidLabels.backup_dir].val,
db_container.kubernetes_namespace
if db_container.kubernetes_namespace
else "",
db_container.backup_name,
)
)
print(backups_base_path)
base_path = Path(backups_base_path)
self.manager = BackupManager(
base_path=base_path,
base_path=target_dir,
backup_file_prefix=self.backup_file_prefix,
backup_file_suffix=f"{self.backup_file_suffix}.gz"
if self.compress_backup
else self.backup_file_suffix,
)
self.host = host
self.user = user
self.password = password
self.host = db_container.coda_labels[ValidLabels.database_host].val
self.user = db_container.coda_labels[ValidLabels.database_username].val
self.password = db_container.coda_labels[ValidLabels.database_password].val
databases = db_container.coda_labels[ValidLabels.database_names].val
if isinstance(databases, str):
self.databases = [databases]
self.databases = databases.split(",")
else:
self.databases = databases
......
......@@ -30,8 +30,8 @@ from cli_helper import (
)
from log import log
from label import ValidLabels
from container_helper import ContainerHelper
from label import ValidLabels, Label
from container_helper import ContainerHelper, Container
# Todo: Skip container related question in native mode. maybe via groups? https://click.palletsprojects.com/en/8.0.x/commands/#nested-handling-and-contexts
......@@ -70,18 +70,21 @@ def backup_cli(debug):
@backup_cli.command(name="now")
@click.option(
"--mode",
"-m",
default="Docker",
prompt="How is your DB running?",
help="Environment the database is running in: 'Docker', 'kubernetes', 'native'",
type=click.Choice(["docker", "kubernetes", "native"], case_sensitive=False),
help="Environment the database is running in: 'Docker', 'kubernetes'",
type=click.Choice(["docker", "kubernetes"], case_sensitive=False),
)
@click.option(
"--container-identifier",
prompt="(k8s-namespace)/Containername? e.g.: 'default/mariadb01-0' in k8s or 'mariadb01' in docker",
"-c",
prompt="Containername or k8s-namespace/Workload-Name? e.g.: 'mariadb01' in docker or 'default/mariadb01' in k8s ",
help="The Docker container name/id (as in `docker ps`) or kubernetes pod name. Split by ',' if multiple",
)
@click.option(
"--database-type",
"-t",
default="mysql",
prompt="Database type?",
help="Do we backup a mysql/maria db or a postgres db?",
......@@ -89,26 +92,30 @@ def backup_cli(debug):
)
@click.option(
"--database-host",
"-h",
default="127.0.0.1",
prompt="Database host?",
help="Hostname or IP where the database is running. In non `native`-mode this will be likely the 'localhost'/' 127.0.0.1' (default value)",
)
@click.option(
"--database-user",
"-u",
default="root",
prompt="Database user?",
help="User to access the database?",
)
@click.option(
"--database-password",
"-p",
hide_input=True,
prompt="Database password?",
help="Password to access the database?",
)
@click.option(
"--database-names",
"-n",
default="",
prompt="Database(s) to backup ('mydb' or ['mydb01', 'mydb02'])? Leave empty for all DBs.",
prompt="Database(s) to backup ('mydb' or 'mydb01,mydb02')? Leave empty for all DBs.",
help="The Databases to backup. We can select a specific database by name . e.g. 'my_database01' or a list of databases e.b. ['mydb01', 'mydb02']. If left empty all databses accessable for the user will be backuped.",
)
def backup_now(
......@@ -124,19 +131,41 @@ def backup_now(
container_names = container_identifier.split(",")
for container_name in container_names:
namespace = None
if mode == "kubernetes":
namespace, container_name_ = container_name.split("/")
container_name = container_name_
# You are here. you need to find the workload ny name and then find the pod in the workload. maybe we need some more container helper funcs
container_name = next(
p
for p in ContainerHelper.kubernetes_get_pods(
namespace=namespace, describe=True
)
).name
print(container_name)
exit()
BackupperClass = database_type_backupper_mapping[database_type]
bu = BackupperClass(
container = Container(
mode=mode,
container_name=container_name,
host=database_host,
user=database_user,
password=database_password,
id="",
name=container_name,
backup_name=container_name,
coda_labels=ValidLabels.valid_labels_from_dict(
{
ValidLabels.backup_name.key: container_name,
ValidLabels.database_type.key: database_type,
ValidLabels.database_host.key: database_host,
ValidLabels.database_password.key: database_password,
ValidLabels.database_username.key: database_user,
ValidLabels.database_names.key: database_names,
},
add_missing_default_labels=True,
),
other_labels={},
kubernetes_namespace=namespace,
)
BackupperClass = database_type_backupper_mapping[database_type]
bu = BackupperClass(container)
bu.backup(
databases=database_names,
retention_type=RetentionType.MANUAL,
......@@ -178,25 +207,7 @@ def backup_kubernetes(namespace, all_namespaces, target_dir):
BackupperClass = database_type_backupper_mapping[
pod.coda_labels[ValidLabels.database_type].val
]
target_dir_full = Path(
PurePath(
target_dir
if target_dir
else pod.coda_labels[ValidLabels.backup_dir].val,
pod.kubernetes_namespace,
pod.backup_name,
)
)
bu = BackupperClass(
mode="kubernetes",
container_name=pod.name,
host=pod.coda_labels[ValidLabels.database_host].val,
user=pod.coda_labels[ValidLabels.database_username].val,
password=pod.coda_labels[ValidLabels.database_password].val,
kubernetes_namespace=pod.kubernetes_namespace,
backups_base_path=target_dir_full,
)
bu = BackupperClass(pod)
bu.manager.retention_duration = {
RetentionType.DAILY: pod.coda_labels[ValidLabels.retention_daily].val,
RetentionType.WEEKLY: pod.coda_labels[ValidLabels.retention_weekly].val,
......
......@@ -10,18 +10,19 @@ from label import ValidLabels, Label
from executer import Executer
config: DEFAULT = getConfig(config_classes=[DEFAULT])
log = logging.getLogger("databasebackupper")
from log import log
# TodO: You are here. Make the docker* func work with new Container class
@dataclass
class Container:
mode: str
id: str
name: str
backup_name: str
coda_labels: Dict[Label, Label]
other_labels: Dict[Label, Label]
coda_labels: Dict[str, Label]
other_labels: Dict[str, Label]
desc: Dict = None
parent: Dict = None
kubernetes_namespace: str = None
......@@ -30,6 +31,7 @@ class Container:
def from_docker_inspect_dict(cls, container_inspect_result: Dict):
# todo validate correct dict type
container = cls(
mode="docker",
id=container_inspect_result[0]["Id"],
name=container_inspect_result[0]["Name"],
backup_name=container_inspect_result[0]["Name"],
......@@ -54,6 +56,7 @@ class Container:
# todo validate correct item by "kind" field
container = cls(
mode="kubernetes",
id=kubectl_get_item_result["metadata"]["uid"],
name=kubectl_get_item_result["metadata"]["name"],
backup_name=kubectl_get_item_result["metadata"]["name"],
......@@ -102,6 +105,7 @@ class ContainerHelper:
namespace_arg = (
f"-n {namespace if namespace else config.KUBECTL_DEFAULT_NAMESPACE}"
)
selector_arg = ""
if pod_name:
selector_arg = pod_name
elif labels:
......@@ -176,15 +180,20 @@ class ContainerHelper:
return pods
@classmethod
def kubernetes_get_workloads_to_be_backed_up(
cls, namespace: str = None, all_namespaces: bool = False
def kubernetes_get_workloads(
cls,
namespace: str = None,
all_namespaces: bool = False,
labels: List[Label] = None,
describe: bool = False,
) -> List[str]:
# kubectl get all
# kubectl get pods -o jsonpath='{range .items[?(.kind=StatefulSet)]}{.metadata.name}{end}'
selector_arg = ""
if labels:
labels_str: str = ",".join([str(label) for label in labels])
selector_arg: str = f"--selector '{labels_str}'" if labels_str else ""
label = ValidLabels.label_from_valid_label_as_template(
ValidLabels.enabled, val=True
)
if all_namespaces:
namespace_arg = "--all-namespaces"
else:
......@@ -195,14 +204,34 @@ class ContainerHelper:
for workload_type in config.KUBECTL_VALID_WORKLOAD_TYPES:
results_items: Dict = json.loads(
Executer.exec(
f"{config.KUBECTL_COMMAND} get all {namespace_arg} --selector={label} -o json"
f"{config.KUBECTL_COMMAND} get all {namespace_arg} {selector_arg} -o json"
)
)["items"]
for res_item in results_items:
if "kind" in res_item and res_item["kind"] == workload_type:
workloads.append(res_item)
if describe:
workloads.append(res_item)
else:
workloads.append(res_item["metadata"]["name"])
return workloads
@classmethod
def kubernetes_get_workloads_to_be_backed_up(
cls, namespace: str = None, all_namespaces: bool = False
) -> List[Dict]:
# kubectl get all
# kubectl get pods -o jsonpath='{range .items[?(.kind=StatefulSet)]}{.metadata.name}{end}'
backup_enabled_label = ValidLabels.label_from_valid_label_as_template(
ValidLabels.enabled, val=True
)
return cls.kubernetes_get_workloads(
namespace=namespace,
all_namespaces=all_namespaces,
labels=[backup_enabled_label],
describe=True,
)
@classmethod
def docker_get_container_to_be_backed_up(
cls, describe: bool = False
......@@ -261,7 +290,7 @@ class ContainerHelper:
for label in ValidLabels.iter():
if label.key in container_labels["metadata"]["labels"].keys():
label.val = container_labels["metadata"]["labels"][label.key]
config_labels_dict[label] = label
config_labels_dict[label.key] = label
return config_labels_dict
......@@ -280,5 +309,5 @@ class ContainerHelper:
for label in ValidLabels.iter():
if label.key in container_labels.keys():
label.val = container_labels[label.key]
config_labels_dict[label] = label
config_labels_dict[label.key] = label
return config_labels_dict
......@@ -239,7 +239,7 @@ class ValidLabels:
else:
# ValidLabel was not in provided label dict. lets add it if desired by caller
if add_missing_default_labels:
lbls[valid_label] = valid_label
lbls[valid_label.key] = valid_label
return lbls
@classmethod
......@@ -256,7 +256,7 @@ class ValidLabels:
for key, val in labels.items():
if key not in [valid_label.key for valid_label in ValidLabels.iter()]:
label = Label(label={key: val})
lbls[label] = label
lbls[label.key] = label
return lbls
@classmethod
......
......@@ -10,3 +10,10 @@ docker exec postgres /usr/bin/psql -Atx postgresql://postgres:mysupersavepw@loca
INSERT INTO coda_ps_test_scheme.my_table(id,firstname) VALUES (2,'Thomas'); \
commit;"
```
# Run Backupnow test
```bash
python3 CoDaBackup/cli.py backup now --mode=docker --container-identifier=mysql --database-type=mysql --database-host=127.0.0.1 --database-user=root --database-names=coda_test
```
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment