Commit 54d9834d authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

auto create k8s rc1

parent 993be7f1
from typing import List, Dict, Union
import logging
import json
from Configs import getConfig
from pathlib import PurePath, Path
from config import DEFAULT
......@@ -38,7 +39,6 @@ class BaseBackupper:
def get_create_database_command(
self,
database_name,
enabled_user_name,
encoding,
collation,
executive_db_user_name=None,
......@@ -55,6 +55,15 @@ class BaseBackupper:
):
raise NotImplementedError
def get_grant_access_to_user_command(
self,
username,
database,
executive_db_user_name=None,
executive_db_user_pw=None,
):
raise NotImplementedError
def __init__(self, db_container: Container, target_base_dir: Path = None):
self.executer = Executer(
mode=db_container.mode,
......@@ -204,11 +213,14 @@ class BaseBackupper:
dry_run=dry_run,
)
def auto_create(self):
def auto_create(self) -> List[str]:
"""Create databases and users if not existent"""
if not self.container.coda_labels[ValidLabels.auto_create_enabled].val:
log.debug(
f"Skip auto-create for {self.container.name} no '{ValidLabels.auto_create_enabled.key}'-Label"
)
return
created_dbs: List[str] = []
# First: Lets gather the database username and pw for the user that will execute the database and user creation
executive_user = self.container.coda_labels[
ValidLabels.auto_create_user_name
......@@ -222,34 +234,39 @@ class BaseBackupper:
executive_user_pw = self.password
# Lets find which databases we have to create, or lets say 'should exists'?
to_be_existing_databases_data = self.container.coda_labels[
ValidLabels.auto_create_databases
].val.split(",")
try:
to_be_existing_databases_data = json.loads(
self.container.coda_labels[ValidLabels.auto_create_databases].val
)
except:
log.error(
f"Can not parse '{ValidLabels.auto_create_databases.key}'-Label. Valid format is json list of obects. got {self.container.coda_labels[ValidLabels.auto_create_databases]}"
)
raise
for to_exist_db in to_be_existing_databases_data:
# Lets find out which databases allready exists
existing_databases = self.get_list_database_command()
existing_databases = self.list_databases()
existing_users = self.list_users()
# example value for to_exist_db: 'mydb/myuser/supersavepassw,otherdb/otheruser/savepw'
db_props: List = to_exist_db.split("/")
if len(db_props) == 3:
database_name, user_name, user_password = db_props
elif len(db_props) == 1:
database_name = db_props[0]
database_name = None
user_name = None
user_password = None
if "database" in to_exist_db:
database_name = to_exist_db["database"].lower()
else:
raise ValueError(
f"Unexpected format in {ValidLabels.auto_create_databases.key}. Got '{to_be_existing_databases_data}'.\nLabel help: {ValidLabels.auto_create_databases.info}"
f"Expeted property 'database' in auto-create config object in label {ValidLabels.auto_create_databases.key}. got '{to_exist_db}'"
)
if "user" in to_exist_db:
user_name = to_exist_db["user"].lower()
if "password" in to_exist_db:
user_password = to_exist_db["password"]
if not user_name:
user_name = self.user
user_password = self.password
# clean input data
database_name = database_name.strip()
user_name = user_name.strip()
user_password = user_password.strip()
# lets go
touched_user_or_database: bool = False
if not user_name in existing_users:
# user does not exists. lets create it
log.info(f"Create user '{user_name}'@'{self.container.name}' ")
......@@ -261,13 +278,14 @@ class BaseBackupper:
executive_db_user_pw=executive_user_pw,
)
)
touched_user_or_database = True
if not database_name in existing_databases:
# database does not exists. lets create it
log.info(f"Create database '{database_name}'@'{self.container.name}' ")
self.executer.container_exec(
command=self.get_create_database_command(
database_name,
enabled_user_name=user_name,
encoding=self.container.coda_labels[
ValidLabels.auto_create_encoding
].val,
......@@ -278,6 +296,21 @@ class BaseBackupper:
executive_db_user_pw=executive_user_pw,
),
)
touched_user_or_database = True
created_dbs.append(database_name)
if touched_user_or_database:
# Grant access for user to DB
self.executer.container_exec(
command=self.get_grant_access_to_user_command(
database=database_name,
username=user_name,
executive_db_user_name=executive_user,
executive_db_user_pw=executive_user_pw,
),
)
return created_dbs
class MySQLBackupper(BaseBackupper):
......@@ -308,14 +341,14 @@ class MySQLBackupper(BaseBackupper):
def get_create_database_command(
self,
database_name,
enabled_user_name,
encoding,
collation,
executive_db_user_name=None,
executive_db_user_pw=None,
):
# character set UTF8mb4 collate utf8mb4_bin
return f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e "CREATE DATABASE `{database_name}` {"CHARACTER SET = '" + encoding + "'" if encoding else ""} {"COLLATE = '" + collation + "'" if collation else ""}; GRANT ALL privileges ON `{database_name}`.* TO '{enabled_user_name}'@'%';FLUSH PRIVILEGES;" """
command_base = f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e """
return f"""{command_base} "CREATE DATABASE {database_name} {"CHARACTER SET = '" + encoding + "'" if encoding else ""} {"COLLATE = '" + collation + "'" if collation else ""};" """
def get_create_user_command(
self,
......@@ -324,9 +357,21 @@ class MySQLBackupper(BaseBackupper):
executive_db_user_name=None,
executive_db_user_pw=None,
):
log.warning("Create mysql user with not network restrictions (`@'%'`)")
log.warning("Create mysql user with no network restrictions (@'%')")
return f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e "CREATE USER '{username}'@'%' IDENTIFIED BY '{password}';" """
def get_grant_access_to_user_command(
self,
username,
database,
executive_db_user_name=None,
executive_db_user_pw=None,
):
log.warning(
f"Grant access to '{database}' for user '{username}' with no network restrictions (@'%')"
)
return f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e "GRANT ALL privileges ON {database}.* TO '{username}'@'%';FLUSH PRIVILEGES;" """
class PostgresBackupper(BaseBackupper):
bin_dump: str = "/usr/bin/pg_dump"
......@@ -358,13 +403,15 @@ class PostgresBackupper(BaseBackupper):
def get_create_database_command(
self,
database_name,
enabled_user_name,
encoding,
collation,
executive_db_user_name=None,
executive_db_user_pw=None,
):
return f"""env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password} {self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "CREATE DATABASE {database_name} {"ENCODING '"+ encoding + "'" if encoding else ''} {"LC_COLLATE = '"+ collation + "'" if collation else ''}; GRANT ALL PRIVILEGES ON DATABASE {database_name} TO {enabled_user_name};" """
password_export = f"env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password}"
command_base = f"{self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "
create_database_transaction = f"""CREATE DATABASE {database_name} {"ENCODING '"+ encoding + "'" if encoding else ''} {"LC_COLLATE = '"+ collation + "'" if collation else ''};"""
return f"""{password_export} {command_base} "{create_database_transaction}" """
def get_create_user_command(
self,
......@@ -375,6 +422,23 @@ class PostgresBackupper(BaseBackupper):
):
return f"""env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password} {self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "CREATE USER {username} WITH PASSWORD '{password}'; " """
def get_grant_access_to_user_command(
self,
username,
database,
executive_db_user_name=None,
executive_db_user_pw=None,
):
log.warning(
f"Grant access to {database} for user {username} with no network restrictions (@'%')"
)
password_export = f"env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password}"
command_base = f"{self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "
grant_privs_transaction = (
f"""GRANT ALL PRIVILEGES ON DATABASE {database} TO {username};"""
)
return f"""{password_export} {command_base} "{grant_privs_transaction}" """
class Neo4jOnlineBackupper(BaseBackupper):
bin_dump: str = "/usr/bin/neo4j-admin backup"
......
......@@ -555,18 +555,27 @@ def auto_create_kubernetes(namespace, all_namespaces):
count_db_instances = 0
count_dbs = 0
for pod in ContainerHelper.kubernetes_get_pods_to_be_backed_up(
namespace=namespace, all_namespaces=all_namespaces
):
count_db_instances += 1
if pod.coda_labels[ValidLabels.enabled].val:
BackupperClass = database_type_backupper_mapping[
pod.coda_labels[ValidLabels.database_type].val
]
bu: BaseBackupper = BackupperClass(pod)
bu.auto_create()
click.echo(
f"\nBackuped {count_dbs} database(s) in {count_db_instances} instance(s)"
)
created_dbs = bu.auto_create()
if created_dbs:
count_dbs += len(created_dbs)
click.echo(f"Created {count_dbs} database(s) in {count_db_instances} instance(s)")
@cli_root.command(name="list-labels")
def list_labels():
print("")
click.echo(ValidLabels.list_labels_human_readable())
if __name__ == "__main__":
......
......@@ -84,20 +84,19 @@ Thats it. We now have a directory `./backups/` in front of us, with all database
* (WIP) Restore Wizard
* (WIP) Neo4j support
* (WIP) Auto database creation if not exists
* (WIP) Create database if not existent via label conf (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Planned) Database auth via Docker/kubernetes Secrets
* (Planned) Create database if not existent via label conf (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Planned) Email notification
* (Planned) Support of docker and kubernetes secrets for providing database auth
* (Planned) Docker Event listener / Kubectl hooks to react to container started/stopped
* (Planned) Improve human readable output by printing a tree structure (https://www.baeldung.com/java-print-binary-tree-diagram)
* (Planned) Compression optional
* (Idea) Database tools (Create non existing databases, based on labels)
* (Planned) Backup compression optional
* (Idea) Switch to https://github.com/kubernetes-client/python and https://docker-py.readthedocs.io/en/stable/ (or create alternative modules)
* (Idea) restore by label (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/)
* (Idea) restore by label (checked/executed when pod starts via https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) (needs a metadata store :/ )
* (Idea) Matrix notifications
* (Idea) [Podman](https://podman.io/) support. Your help is greatly appreciated. Should be easy or maybe no work at all
* (Idea) [LCX](https://linuxcontainers.org/) support. Help needed!
* (Idea) Suppord pod with more than one container https://kubernetes.io/docs/tasks/debug-application-cluster/get-shell-running-container/#opening-a-shell-when-a-pod-has-more-than-one-container
* (Idea) Provide auto-create params in https://kubernetes.io/docs/concepts/configuration/configmap/ instead of k8s annotation
# Current ToDo
* implement RETENTION_ON_COLLISION_KEEP_NEWEST_BACKUP
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment