wip

CoDaBackup/backupper.py

@@ -35,7 +35,24 @@ class BaseBackupper:
     def get_restore_command(self, database_name, source_filepath) -> str:
         raise NotImplementedError
 
-    def get_create_database_and_user_if_not_exists_command(self, database_name) -> str:
+    def get_create_database_command(
+        self,
+        database_name,
+        enabled_user_name,
+        encoding,
+        collation,
+        executive_db_user_name=None,
+        executive_db_user_pw=None,
+    ):
+        raise NotImplementedError
+
+    def get_create_user_command(
+        self,
+        username,
+        password,
+        executive_db_user_name=None,
+        executive_db_user_pw=None,
+    ):
         raise NotImplementedError
 
     def __init__(self, db_container: Container, target_base_dir: Path = None):
@@ -119,7 +136,7 @@ class BaseBackupper:
         return [
             user.strip()
             for user in result.decode("utf-8").splitlines()
-            if  user.strip() != ""
+            if user.strip() != ""
         ]
 
     def backup(
@@ -187,6 +204,79 @@ class BaseBackupper:
             dry_run=dry_run,
         )
 
+    def auto_create(self):
+        """Create databases and users if not existent"""
+        if not self.container.coda_labels[ValidLabels.auto_create_enabled].val:
+            return
+
+        # First: Lets gather the database username and pw for the user that will execute the database and user creation
+        executive_user = self.container.coda_labels[
+            ValidLabels.auto_create_user_name
+        ].val
+        executive_user_pw = self.container.coda_labels[
+            ValidLabels.auto_create_user_password
+        ].val
+        if not executive_user:
+            # there is no extra user configures to execute the DB and user creation. we take the current user from label 'username'
+            executive_user = self.user
+            executive_user_pw = self.password
+
+        # Lets find which databases we have to create, or lets say 'should exists'?
+        to_be_existing_databases_data = self.container.coda_labels[
+            ValidLabels.auto_create_databases
+        ].val.split(",")
+
+        for to_exist_db in to_be_existing_databases_data:
+            # Lets find out which databases allready exists
+            existing_databases = self.get_list_database_command()
+
+            existing_users = self.list_users()
+            # example value for to_exist_db: 'mydb/myuser/supersavepassw,otherdb/otheruser/savepw'
+            db_props: List = to_exist_db.split("/")
+            if len(db_props) == 3:
+                database_name, user_name, user_password = db_props
+            elif len(db_props) == 1:
+                database_name = db_props[0]
+            else:
+                raise ValueError(
+                    f"Unexpected format in {ValidLabels.auto_create_databases.key}. Got '{to_be_existing_databases_data}'.\nLabel help: {ValidLabels.auto_create_databases.info}"
+                )
+
+            if not user_name:
+                user_name = self.user
+                user_password = self.password
+            # clean input data
+            database_name = database_name.strip()
+            user_name = user_name.strip()
+            user_password = user_password.strip()
+            # lets go
+            if not user_name in existing_users:
+                # user does not exists. lest create it
+                self.executer.container_exec(
+                    command=self.get_create_user_command(
+                        username=user_name,
+                        password=user_password,
+                        executive_db_user_name=executive_user,
+                        executive_db_user_pw=executive_user_pw,
+                    )
+                )
+            if not database_name in existing_databases:
+                # database does not exists. lets create it
+                self.executer.container_exec(
+                    command=self.get_create_database_command(
+                        database_name,
+                        enabled_user_name=user_name,
+                        encoding=self.container.coda_labels[
+                            ValidLabels.auto_create_encoding
+                        ].val,
+                        collation=self.container.coda_labels[
+                            ValidLabels.auto_create_collation
+                        ].val,
+                        executive_db_user_name=executive_user,
+                        executive_db_user_pw=executive_user_pw,
+                    ),
+                )
+
 
 class MySQLBackupper(BaseBackupper):
     bin_dump: str = "/usr/bin/mysqldump"
@@ -208,7 +298,32 @@ class MySQLBackupper(BaseBackupper):
         return f"""{self.bin_cmd} -u{self.user} -p{self.password} {database_name}"""
 
     def get_list_database_command(self):
-        return f"""{self.bin_cmd} -N -h{self.host} -u{self.user} -p{self.password} -e "SHOW DATABASES" """
+        return f"""{self.bin_cmd} -N -h{self.host} -u{self.user} -p{self.password} -e "SHOW DATABASES;" """
+
+    def get_list_user_command(self):
+        return f"""{self.bin_cmd} -N -h{self.host} -u{self.user} -p{self.password} -e "SELECT User FROM mysql.user;" """
+
+    def get_create_database_command(
+        self,
+        database_name,
+        enabled_user_name,
+        encoding,
+        collation,
+        executive_db_user_name=None,
+        executive_db_user_pw=None,
+    ):
+        # character set UTF8mb4 collate utf8mb4_bin
+        return f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e "CREATE DATABASE `{database_name}` {"CHARACTER SET = '" + encoding + "'" if encoding else ""} {"COLLATE = '" + collation + "'" if collation else ""}; GRANT ALL privileges ON `{database_name}`.* TO '{enabled_user_name}'@'%';FLUSH PRIVILEGES;" """
+
+    def get_create_user_command(
+        self,
+        username,
+        password,
+        executive_db_user_name=None,
+        executive_db_user_pw=None,
+    ):
+        log.warning("Create mysql user with not network restrictions (`@'%'`)")
+        return f"""{self.bin_cmd} -N -h{self.host} -u{executive_db_user_name if executive_db_user_name else self.user} -p{executive_db_user_pw if executive_db_user_name else self.password} -e "CREATE USER '{username}'@'%' IDENTIFIED BY '{password}';" """
 
 
 class PostgresBackupper(BaseBackupper):
@@ -236,14 +351,27 @@ class PostgresBackupper(BaseBackupper):
         return f"""env PGPASSWORD={self.password} {self.bin_cmd} -U {self.user} -h {self.host} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;" """
 
     def get_list_user_command(self):
-    return f"""env PGPASSWORD={self.password} {self.bin_cmd} -U {self.user} -h {self.host} -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;" """
-
-    def get_create_database_and_user_if_not_exists_command(self, database_name):
-        admin_user = self.container.coda_label[ValidLabels.auto_create_user_name].val
-        admin_pw = self.container.coda_label[ValidLabels.auto_create_user_password].val
-        users = self.executer.
-        databases = self.list_databases()
-        return f""""""
+        return f"""env PGPASSWORD={self.password} {self.bin_cmd} -U {self.user} -h {self.host} -t -c "SELECT usename FROM pg_catalog.pg_user;" """
+
+    def get_create_database_command(
+        self,
+        database_name,
+        enabled_user_name,
+        encoding,
+        collation,
+        executive_db_user_name=None,
+        executive_db_user_pw=None,
+    ):
+        return f"""env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password} {self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "CREATE DATABASE {database_name} {"ENCODING '"+ encoding + "'" if encoding else ''} {"LC_COLLATE = '"+ collation + "'" if collation else ''}; GRANT ALL PRIVILEGES ON DATABASE {database_name} TO {enabled_user_name};" """
+
+    def get_create_user_command(
+        self,
+        username,
+        password,
+        executive_db_user_name=None,
+        executive_db_user_pw=None,
+    ):
+        return f"""env PGPASSWORD={executive_db_user_pw if executive_db_user_name else self.password} {self.bin_cmd} -U {executive_db_user_name if executive_db_user_name else self.user} -h {self.host} -t -c "CREATE USER {username} WITH PASSWORD '{password}'; " """
 
 
 class Neo4jOnlineBackupper(BaseBackupper):

CoDaBackup/cli.py

@@ -522,5 +522,23 @@ def restore_docker_list(container_name, databases, source_dir, output_format):
     return format_n_output_backup_list(data, output_format)
 
 
+@cli_root.group(name="auto-create")
+@click.option(
+    "--debug/--no-debug",
+    help="Enable debug logging",
+    default=False,
+)
+def auto_create(debug):
+    if debug:
+        click.echo(f"Debug mode is on")
+        log.setLevel("DEBUG")
+
+
+@auto_create.command(name="now")
+def auto_create_now():
+    # you are here. maybe we dont new "now" command and instead a kubernetes and a docker version
+    raise NotImplementedError
+
+
 if __name__ == "__main__":
     cli_root()

CoDaBackup/container_helper.py

@@ -62,7 +62,8 @@ class Container:
             name=kubectl_get_item_result["metadata"]["name"],
             backup_name=kubectl_get_item_result["metadata"]["name"],
             coda_labels=ValidLabels.valid_labels_from_dict(
-                kubectl_get_item_result["metadata"]["labels"],
+                kubectl_get_item_result["metadata"]["labels"]
+                | kubectl_get_item_result["metadata"]["annotations"],
                 add_missing_default_labels=True,
             ),
             other_labels=ValidLabels.non_valid_labels_from_dict(

CoDaBackup/label.py

@@ -170,7 +170,7 @@ class ValidLabels:
         base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
     )
 
-    auto_create_enables: Label = Label(
+    auto_create_enabled: Label = Label(
         "auto-create",
         bool,
         default=False,
@@ -181,7 +181,7 @@ class ValidLabels:
         "auto-create-user",
         str,
         default=None,
-        info="The user to create a missing database (must have the priviledge to create a database)?",
+        info=f"The user to create a missing database (must have the priviledge to create a database). Defaults to backup user from label `{database_username.key}`?",
         base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
     )
     auto_create_user_password: Label = Label(
@@ -209,7 +209,7 @@ class ValidLabels:
         "auto-create-databases",
         str,
         default=None,
-        info="Name of the databases to be created",
+        info="Databases, Users and Password to create. Multiple entries seperated by comma are possible. \nformat:\n'<databasename>/<username>/<password>'\nExample:\n'mydb/myuser/supersave,otherdb/otheruser/savepw'",
         base_label_key=config.DATABASE_CONTAINER_LABEL_BASE_KEY,
     )
 

setup.py

@@ -15,7 +15,7 @@ setup(
     license="MIT",
     packages=["CoDaBackup"],
     install_requires=["DZDConfigs", "Click", "tabulate", "humanize"],
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     zip_safe=False,
     include_package_data=True,
     use_scm_version={