Commit 9297236e authored by Tim's avatar Tim
Browse files

wip

parent 0c0bd875
from Configs import ConfigBase
class DEFAULT(ConfigBase):
NEO4J = {}
# docker.DockerClient params as dict. https://docker-py.readthedocs.io/en/stable/client.html#client-reference
DOCKER_CLIENT = {"base_url": "unix:///var/run/docker.sock"}
LOG_PATH = "./"
PIPELINE_DEF_FILE = "./pipeline.yaml"
PIPELINE_STATE_NODES_LABEL = "_PipelineLogNode"
"""Motherlode
https://git.connect.dzd-ev.de/dzdconnectpipeline/motherlode/
Copyright 2020 Deutsches Zentrum für Diabetesforschung e.V.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
import os
import sys
import py2neo
import logging
from Configs import getConfig
if __name__ == "__main__":
SCRIPT_DIR = os.path.dirname(
os.path.realpath(os.path.join(
os.getcwd(), os.path.expanduser(__file__)))
)
SCRIPT_DIR = os.path.join(SCRIPT_DIR, "..")
sys.path.insert(0, os.path.normpath(SCRIPT_DIR))
from Motherlode.config import DEFAULT
config: DEFAULT = getConfig()
graph: py2neo.Graph = py2neo.Graph(config.NEO4J)
log = logging.getLogger(__name__)
def parameter_validation():
if config.PIPELINE_DEF_FILE is None:
raise FileNotFoundError(
"No pipeline definition file provided. Use 'PIPELINE_DEF_FILE' to provide a file.")
if not os.path.isfile(config.PIPELINE_DEF_FILE):
raise FileNotFoundError(
"Pipeline definition file is not a file: 'PIPELINE_DEF_FILE'='{}'".format(config.PIPELINE_DEF_FILE))
try:
graph.run("MATCH (n) return n limit 1")
except Exception as e:
log.exception(
"Can not connect to neo4j. py2neo connection dict: '{}'".format(config.NEO4J))
from docker.client import DockerClient
import py2neo
import docker
import datetime
from typing import List
from Configs import getConfig
from copili import Pipeline, ContainerManager
config = getConfig()
class Motherlode:
def __init__(self):
self.pipeline = Pipeline(
config.PIPELINE_DEF_FILE, docker_client=docker.DockerClient(**config.DOCKER_CLIENT))
self.pipeline.container_pre_run_callback = self._callback_attach_container_registry_data
self.pipeline.container_post_run_callback = self._callback_create_container_log_node
self.pipeline.container_did_run_check_override_callback = self._callback_did_container_allready_run_check
self.run()
def _callback_did_container_allready_run_check(self, container_manager: ContainerManager, finished_container_managers: List[ContainerManager]):
if self._query_log_nodes(container_manager):
return True
else:
return False
def _callback_attach_container_registry_data(self, container_manager: ContainerManager):
container_manager.image_reg_data = container_manager.docker_client.images.get_registry_data(
"{}:{}".format(container_manager.container_desc.image_repo, container_manager.container_desc.tag))
def _callback_create_container_log_node(self, container_manager: ContainerManager):
n = py2neo.Node(config.PIPELINE_STATE_NODES_LABEL)
n["loader"] = container_manager.container_desc.name
n["container_image_repo"] = container_manager.container_desc.image_repo
n["container_image_tag"] = container_manager.container_desc.tag
n["container_image_hash"] = container_manager.image_reg_data.id
n["container_short_image_hash"] = container_manager.image_reg_data.short_id
n["container_finished_at"] = str(datetime.datetime.now(tz=None))
tx = py2neo.Graph(config.NEO4J).begin()
tx.create(n)
tx.commit()
def _query_log_nodes(self, container_manager: ContainerManager):
return list(py2neo.Graph(config.NEO4J).run(
"""MATCH (n:{NODE_LABEL})
WHERE loader={LOADER_NAME}
AND container_image_repo={IMAGE_REPO}
AND container_image_tag={IMAGE_TAG}
AND container_image_hash={IMAGE_HASH}
RETURN n""".format(NODE_LABEL=config.PIPELINE_STATE_NODES_LABEL,
LOADER_NAME=container_manager.container_desc.name,
IMAGE_REPO=container_manager.container_desc.image_repo,
IMAGE_TAG=container_manager.container_desc.tag,
IMAGE_HASH=container_manager.image_reg_data.id)).data())
def run(self):
self.pipeline.run()
# motherlode
# Motherlode
Python script to manage the build up, update a graph with multiple datasources.
\ No newline at end of file
Python script to manage the creation and updating a neo4j graph with multiple datasources running in docker/podman containers.
Motherlode builds up on [copili](https://git.connect.dzd-ev.de/dzdtools/pythonmodules/-/tree/master/copili)
**Maintainer**: tim.bleimehl@dzd-ev.de
**Licence**: MIT
**issue tracker**: https://git.connect.dzd-ev.de/dzdconnectpipeline/motherlode/-/issues
\ No newline at end of file
Motherlode
https://git.connect.dzd-ev.de/dzdconnectpipeline/motherlode/
Copyright 2020 Deutsches Zentrum für Diabetesforschung e.V.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
docker
py2neo
git+https://git.connect.dzd-ev.de/dzdtools/pythonmodules.git#subdirectory=copili
git+https://git.connect.dzd-ev.de/dzdtools/pythonmodules.git#subdirectory=Configs
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment