Skip to content
GitLab
Commit 54d9834d authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

auto create k8s rc1

parent 993be7f1
Hide whitespace changes
Inline Side-by-side
CoDaBackup/backupper.py
View file @ 54d9834d
from typing import List, Dict, Union
import logging
import json
from Configs import getConfig
from pathlib import PurePath, Path
from config import DEFAULT
......@@ -38,7 +39,6 @@ class BaseBackupper:
def get_create_database_command(
self,
database_name,
enabled_user_name,
encoding,
collation,
executive_db_user_name=None,
......@@ -55,6 +55,15 @@ class BaseBackupper:
):
raise NotImplementedError
def get_grant_access_to_user_command(
self,
username,
database,
executive_db_user_name=None,
executive_db_user_pw=None,
):
raise NotImplementedError
def __init__(self, db_container: Container, target_base_dir: Path = None):
self.executer = Executer(
mode=db_container.mode,
......@@ -204,11 +213,14 @@ class BaseBackupper:
dry_run=dry_run,
)
def auto_create(self):
def auto_create(self) -> List[str]:
"""Create databases and users if not existent"""
if not self.container.coda_labels[ValidLabels.auto_create_enabled].val:
log.debug(
f"Skip auto-create for {self.container.name} no '{ValidLabels.auto_create_enabled.key}'-Label"
)
return
created_dbs: List[str] = []
# First: Lets gather the database username and pw for the user that will execute the database and user creation
executive_user = self.container.coda_labels[
ValidLabels.auto_create_user_name
......@@ -222,34 +234,39 @@ class BaseBackupper:
executive_user_pw = self.password
# Lets find which databases we have to create, or lets say 'should exists'?
to_be_existing_databases_data = self.container.coda_labels[
ValidLabels.auto_create_databases
].val.split(",")
try:
to_be_existing_databases_data = json.loads(
self.container.coda_labels[ValidLabels.auto_create_databases].val
)
except:
log.error(
f"Can not parse '{ValidLabels.auto_create_databases.key}'-Label. Valid format is json list of obects. got {self.container.coda_labels[ValidLabels.auto_create_databases]}"
)
raise
for to_exist_db in to_be_existing_databases_data:
# Lets find out which databases allready exists
existing_databases = self.get_list_database_command()
existing_databases = self.list_databases()
existing_users = self.list_users()
# example value for to_exist_db: 'mydb/myuser/supersavepassw,otherdb/otheruser/savepw'
db_props: List = to_exist_db.split("/")
if len(db_props) == 3:
database_name, user_name, user_password = db_props
elif len(db_props) == 1:
database_name = db_props[0]
database_name = None
user_name = None
user_password = None
if "database" in to_exist_db:
database_name = to_exist_db["database"].lower()
else:
raise ValueError(
f"Unexpected format in {ValidLabels.auto_create_databases.key}. Got '{to_be_existing_databases_data}'.\nLabel help: {ValidLabels.auto_create_databases.info}"
f"Expeted property 'database' in auto-create config object in label {ValidLabels.auto_create_databases.key}. got '{to_exist_db}'"
)
if "user" in to_exist_db:
user_name = to_exist_db["user"].lower()
if "password" in to_exist_db:
user_password = to_exist_db["password"]
if not user_name:
user_name = self.user
user_password = self.password
# clean input data
database_name = database_name.strip()
user_name = user_name.strip()
user_password = user_password.strip()
# lets go
touched_user_or_database: bool = False
if not user_name in existing_users:
# user does not exists. lets create it
log.info(f"Create user '{user_name}'@'{self.container.name}' ")
......@@ -261,13 +278,14 @@ class BaseBackupper:
executive_db_user_pw=executive_user_pw,
)
)
touched_user_or_database = True
if not database_name in existing_databases:
# database does not exists. lets create it
log.info(f"Create database '{database_name}'@'{self.container.name}' ")
self.executer.container_exec(
command=self.get_create_database_command(
database_name,
enabled_user_name=user_name,
encoding=self.container.coda_labels[
ValidLabels.auto_create_encoding
].val,
......@@ -278,6 +296,21 @@ class BaseBackupper:
executive_db_user_pw=executive_user_pw,
),
)
touched_user_or_database = True
created_dbs.append(database_name)
if touched_user_or_database:
# Grant access for user to DB
self.executer.container_exec(
command=self.get_grant_access_to_user_command(
database=database_name,
username=user_name,
executive_db_user_name=executive_user,
executive_db_user_pw=executive_user_pw,
),
)
return created_dbs
class MySQLBackupper(BaseBackupper):
......@@ -308,14 +341,14 @@ class MySQLBackupper(BaseBackupper):
def get_create_database_command(
self,
database_name,
enabled_user_name,
encoding,
collation,
executive_db_user_name=None,
executive_db_user_pw=None,
):
# character set UTF8mb4 collate utf8mb4_bin
return f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e "CREATE DATABASE `{database_name}` {"CHARACTER SET = '" + encoding + "'" if encoding else ""} {"COLLATE = '" + collation + "'" if collation else ""}; GRANT ALL privileges ON `{database_name}`.* TO '{enabled_user_name}'@'%';FLUSH PRIVILEGES;" """
command_base = f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e """
return f"""{command_base} "CREATE DATABASE {database_name} {"CHARACTER SET = '" + encoding + "'" if encoding else ""} {"COLLATE = '" + collation + "'" if collation else ""};" """
def get_create_user_command(
self,
......@@ -324,9 +357,21 @@ class MySQLBackupper(BaseBackupper):
executive_db_user_name=None,
executive_db_user_pw=None,
):
log.warning("Create mysql user with not network restrictions (`@'%'`)")
log.warning("Create mysql user with no network restrictions (@'%')")
return f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e "CREATE USER '{username}'@'%' IDENTIFIED BY '{password}';" """
def get_grant_access_to_user_command(
self,
username,
database,
executive_db_user_name=None,
executive_db_user_pw=None,
):
log.warning(
f"Grant access to '{database}' for user '{username}' with no network restrictions (@'%')"
)
return f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e "GRANT ALL privileges ON {database}.* TO '{username}'@'%';FLUSH PRIVILEGES;" """
class PostgresBackupper(BaseBackupper):
bin_dump: str = "/usr/bin/pg_dump"
......@@ -358,13 +403,15 @@ class PostgresBackupper(BaseBackupper):
def get_create_database_command(
self,
database_name,
enabled_user_name,
encoding,
collation,
executive_db_user_name=None,
executive_db_user_pw=None,
):
return f"""env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password} {self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "CREATE DATABASE {database_name} {"ENCODING '"+ encoding + "'" if encoding else ''} {"LC_COLLATE = '"+ collation + "'" if collation else ''}; GRANT ALL PRIVILEGES ON DATABASE {database_name} TO {enabled_user_name};" """
password_export = f"env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password}"
command_base = f"{self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "
create_database_transaction = f"""CREATE DATABASE {database_name} {"ENCODING '"+ encoding + "'" if encoding else ''} {"LC_COLLATE = '"+ collation + "'" if collation else ''};"""
return f"""{password_export} {command_base} "{create_database_transaction}" """
def get_create_user_command(
self,
......@@ -375,6 +422,23 @@ class PostgresBackupper(BaseBackupper):
):
return f"""env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password} {self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "CREATE USER {username} WITH PASSWORD '{password}'; " """
def get_grant_access_to_user_command(
self,
username,
database,
executive_db_user_name=None,
executive_db_user_pw=None,
):
log.warning(
f"Grant access to {database} for user {username} with no network restrictions (@'%')"
)
password_export = f"env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password}"
command_base = f"{self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "
grant_privs_transaction = (
f"""GRANT ALL PRIVILEGES ON DATABASE {database} TO {username};"""
)
return f"""{password_export} {command_base} "{grant_privs_transaction}" """
class Neo4jOnlineBackupper(BaseBackupper):
bin_dump: str = "/usr/bin/neo4j-admin backup"
......
CoDaBackup/cli.py
View file @ 54d9834d
......@@ -555,18 +555,27 @@ def auto_create_kubernetes(namespace, all_namespaces):
count_db_instances = 0
count_dbs = 0
for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
count_db_instances += 1
if pod.coda_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
pod.coda_labels[ValidLabels.database_type].val
]
bu: BaseBackupper = BackupperClass(pod)
bu.auto_create()
click.echo(
f"\nBackuped {count_dbs} database(s) in {count_db_instances} instance(s)"
)
created_dbs = bu.auto_create()
if created_dbs:
count_dbs += len(created_dbs)
click.echo(f"Created {count_dbs} database(s) in {count_db_instances} instance(s)")
@cli_root.command(name="list-labels")
def list_labels():
print("")
click.echo(ValidLabels.list_labels_human_readable())
if __name__ == "__main__":
......
README.md
View file @ 54d9834d
......@@ -84,20 +84,19 @@ Thats it. We now have a directory `./backups/` in front of us, with all database
* (WIP) Restore Wizard
* (WIP) Neo4j support
* (WIP) Auto database creation if not exists
* (WIP) Create database if not existent via label conf (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Planned) Database auth via Docker/kubernetes Secrets
* (Planned) Create database if not existent via label conf (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Planned) Email notification
* (Planned) Support of docker and kubernetes secrets for providing database auth
* (Planned) Docker Event listener / Kubectl hooks to react to container started/stopped
* (Planned) Improve human readable output by printing a tree structure (https://www.baeldung.com/java-print-binary-tree-diagram)
* (Planned) Compression optional
* (Idea) Database tools (Create non existing databases, based on labels)
* (Planned) Backup compression optional
* (Idea) Switch to https://github.com/kubernetes-client/python and https://docker-py.readthedocs.io/en/stable/ (or create alternative modules)
* (Idea) restore by label (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Idea) restore by label (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) (needs a metadata store :/ )
* (Idea) Matrix notifications
* (Idea) [Podman](https://podman.io/) support. Your help is greatly appreciated. Should be easy or maybe no work at all
* (Idea) [LCX](https://linuxcontainers.org/) support. Help needed!
* (Idea) Suppord pod with more than one container https://kubernetes.io/docs/tasks/debug-application-cluster/get-shell-running-container/#opening-a-shell-when-a-pod-has-more-than-one-container
* (Idea) Provide auto-create params in https://kubernetes.io/docs/concepts/configuration/configmap/ instead of k8s annotation
# Current ToDo
* implement RETENTION_ON_COLLISION_KEEP_NEWEST_BACKUP
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment