Commit 6359ee8a authored by Tim Bleimehl's avatar Tim Bleimehl 🤸🏼
Browse files

rename and whole clustered schema visualize

parent 9a346546
from NeoMetaLogger.neo_meta_logger import NeoMetaLogger
from NeoMetaLogger.visualizer import GraphvizVisualizer
from NeoMetaLogger.visualizer.graphiz_visualizer import GraphvizVisualizer
from py2neo import Subgraph
from NeoMetaLogger.visualizer._base_visualizer import BaseVisualizer
# sudo apt-get install graphviz graphviz-dev
# pip install pygraphviz
import pygraphviz as pgv
class GraphvizVisualizer(BaseVisualizer):
def generate_file(self, path):
self._py2neo_subgraph_to_graphviz(
py2neo_subgraph=self.neo_meta_logger_graph_schema
).write(path)
def generate_object(self) -> bytes:
return (
self._py2neo_subgraph_to_graphviz(
py2neo_subgraph=self.neo_meta_logger_graph_schema
)
.string()
.encode("utf-8")
)
def _py2neo_subgraph_to_graphviz(
self,
py2neo_subgraph: Subgraph,
) -> pgv.AGraph:
# https://pygraphviz.github.io/documentation/stable/tutorial.html#graphs
graph: pgv.AGraph = pgv.AGraph(strict=False, directed=True, landscape=False)
for node in py2neo_subgraph.nodes:
graph.add_node(node["__label_name"], **dict(node))
for rel in py2neo_subgraph.relationships:
graph.add_edge(
rel.start_node["__label_name"],
rel.end_node["__label_name"],
fontsize=8,
minlen=2,
# key=type(rel).__name__,
label=type(rel).__name__,
)
return graph
from NeoMetaTracker.neo_meta_logger import NeoMetaTracker
from NeoMetaTracker.visualizer import GraphvizVisualizer
from __future__ import annotations
from typing import Dict, TYPE_CHECKING
from NeoMetaLogger.graph_scheme import GraphSchema
from NeoMetaTracker.graph_scheme import GraphSchema
import time
if TYPE_CHECKING:
from NeoMetaLogger.neo_meta_logger import NeoMetaLogger
from NeoMetaTracker.neo_meta_logger import NeoMetaTracker
class CapturePoint:
......@@ -12,7 +12,7 @@ class CapturePoint:
def __init__(
self,
parent_logger: NeoMetaLogger,
parent_logger: NeoMetaTracker,
name: str,
labels_count: Dict[str, int],
relations_count: Dict[str, int],
......@@ -26,7 +26,7 @@ class CapturePoint:
self.schema: GraphSchema = GraphSchema.from_neo4j_schema_vis_data(
neo4j_schema_vis_data,
parent_capture_point=self,
extra_props={"name": name} if name else {},
extra_props={"capture_name": name} if name else {},
)
def __hash__(self):
......
......@@ -2,14 +2,16 @@ from __future__ import annotations
from typing import Union, List, Dict, Generic, TYPE_CHECKING
if TYPE_CHECKING:
from NeoMetaLogger.capture_point import CapturePoint
from NeoMetaTracker.capture_point import CapturePoint
from pathlib import Path
import py2neo
from NeoMetaLogger.visualizer._base_visualizer import BaseVisualizer
from NeoMetaTracker.visualizer._base_visualizer import BaseVisualizer
class GraphSchema(py2neo.Subgraph):
parent_capture_point: CapturePoint
def visualize(
self,
visualizer_class: BaseVisualizer,
......@@ -61,9 +63,7 @@ class GraphSchema(py2neo.Subgraph):
rel_nodes[0] = clean_rel_node
elif node.identity == rel.end_node.identity:
rel_nodes[1] = clean_rel_node
rel_ident = (
f"{list(rel_nodes[0])[0]}_{type(rel).__name__}_{list(rel_nodes[1])[0]}"
)
rel_ident = f"{list(rel_nodes[0].labels)[0]}_{type(rel).__name__}_{list(rel_nodes[1].labels)[0]}"
if rel_ident in parent_capture_point.parent_logger.all_schema_rels:
clean_rel = parent_capture_point.parent_logger.all_schema_rels[
rel_ident
......@@ -94,7 +94,9 @@ class GraphSchema(py2neo.Subgraph):
# workaround:
sb: py2neo.Subgraph = schema_graph | cls(single_nodes_cleaned)
return GraphSchema(nodes=sb.nodes, relationships=sb.relationships)
gs = GraphSchema(nodes=sb.nodes, relationships=sb.relationships)
gs.parent_capture_point = parent_capture_point
return gs
@classmethod
def _get_or_create_unbound_schema_node(
......
......@@ -2,11 +2,12 @@ from typing import Union, Dict, List, Generic
import time
import py2neo
from pathlib import Path
from NeoMetaLogger.capture_point import CapturePoint
from NeoMetaLogger.graph_scheme import GraphSchema
from NeoMetaTracker.capture_point import CapturePoint
from NeoMetaTracker.graph_scheme import GraphSchema
from NeoMetaTracker.visualizer._base_visualizer import BaseVisualizer
class NeoMetaLogger:
class NeoMetaTracker:
def __init__(self, connection: Union[py2neo.Graph, Dict]):
self.capture_points: List[CapturePoint] = []
self.all_schema_nodes: Dict[tuple, py2neo.Node] = None
......@@ -34,6 +35,23 @@ class NeoMetaLogger:
)
)
def visualize(self, visualizer_class: BaseVisualizer, to_file: Union[str, Path]):
schemas_changes = []
for index, to_cp in enumerate(self.capture_points):
if index < 1:
continue
from_cp = self.capture_points[index - 1]
changes = self.get_schemagraph_changes(
from_capture=from_cp, to_capture=to_cp
)
changes.parent_capture_point = to_cp
schemas_changes.append(changes)
visualizer: BaseVisualizer = visualizer_class(schemas_changes)
if to_file:
visualizer.generate_file(to_file)
else:
return visualizer.generate_object()
def _get_neo4j_schema(self):
# call db.schema.visualization
return self.graph.run(
......
from NeoMetaTracker.visualizer.graphiz_visualizer import GraphvizVisualizer
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, List
if TYPE_CHECKING:
from NeoMetaLogger.graph_scheme import GraphSchema
from NeoMetaTracker.graph_scheme import GraphSchema
class BaseVisualizer:
def __init__(self, neo_meta_logger_graph_schema: GraphSchema):
self.neo_meta_logger_graph_schema = neo_meta_logger_graph_schema
def __init__(self, neo_meta_logger_graph_schemas: List[GraphSchema]):
if not isinstance(neo_meta_logger_graph_schemas, list):
self.neo_meta_logger_graph_schemas = [neo_meta_logger_graph_schemas]
else:
self.neo_meta_logger_graph_schemas = neo_meta_logger_graph_schemas
def generate_file(self, path):
raise NotImplementedError
......
from __future__ import annotations
from typing import List, TYPE_CHECKING
import py2neo
from NeoMetaTracker.visualizer._base_visualizer import BaseVisualizer
if TYPE_CHECKING:
from NeoMetaTracker.graph_scheme import GraphSchema
# sudo apt-get install graphviz graphviz-dev
# pip install pygraphviz
import pygraphviz as pgv
class GraphvizVisualizer(BaseVisualizer):
def generate_file(self, path):
self._py2neo_subgraph_to_graphviz(
neometatracker_graphschemas=self.neo_meta_logger_graph_schemas
).write(path)
def generate_object(self) -> bytes:
return (
self._py2neo_subgraph_to_graphviz(
neometatracker_graphschemas=self.neo_meta_logger_graph_schemas
)
.string()
.encode("utf-8")
)
def _py2neo_subgraph_to_graphviz(
self,
neometatracker_graphschemas: List[GraphSchema],
) -> pgv.AGraph:
# https://pygraphviz.github.io/documentation/stable/tutorial.html#graphs
graph: pgv.AGraph = pgv.AGraph(strict=False, directed=True, landscape=False)
node_track_list: List[py2neo.Node] = []
for nmt_graphschema in neometatracker_graphschemas:
if len(neometatracker_graphschemas) == 1:
graphviz_subg = graph
else:
sb_name = (
nmt_graphschema.parent_capture_point.name
if nmt_graphschema.parent_capture_point.name
else ""
)
graphviz_subg = graph.add_subgraph(
name=sb_name,
label=sb_name,
cluster=True,
style="filled",
color="#E0E0E0",
)
for node in nmt_graphschema.nodes:
if node not in node_track_list:
node_track_list.append(node)
graphviz_subg.add_node(node["__label_name"], **dict(node))
for rel in nmt_graphschema.relationships:
graph.add_edge(
rel.start_node["__label_name"],
rel.end_node["__label_name"],
fontsize=8,
minlen=2,
# key=type(rel).__name__,
label=type(rel).__name__ + " ",
)
return graph
from py2neo import Subgraph
import networkx
from NeoMetaLogger.visualizer._base_visualizer import BaseVisualizer
from NeoMetaTracker.visualizer._base_visualizer import BaseVisualizer
import matplotlib.pyplot as plt
......
......@@ -17,7 +17,7 @@ Lets create a sample graph with python and Neo4J
```python
import py2neo
from NeoMetaLogger import NeoMetaLogger
from NeoMetaTracker import NeoMetaTracker
g = py2neo.Graph(name="test_graph")
......@@ -43,7 +43,7 @@ and the schema will look like this:
Lets capture the current status to analyse the changes later.
```python
meta_logger = NeoMetaLogger(test_graph)
meta_logger = NeoMetaTracker(test_graph)
meta_logger.capture()
```
......@@ -65,7 +65,7 @@ The schema still looks the same. We wont be able to recognize changes without an
!["docs/03_schema.png"](docs/03_schema.png)
Here come `NeoMetaLogger` for the rescue
Here come `NeoMetaTracker` for the rescue
Lets do another capture to compare the changes we did:
......
......@@ -5,7 +5,7 @@ this_directory = Path(__file__).parent
long_description = (this_directory / "README.md").read_text()
setup(
name="NeoMetaLogger",
name="NeoMetaTracker",
description="Track and visualize changes in a Neo4j database schema",
long_description=long_description,
long_description_content_type="text/markdown",
......@@ -13,7 +13,7 @@ setup(
author="Tim Bleimehl",
author_email="tim.bleimehl@helmholtz-muenchen.de",
license="MIT",
packages=["NeoMetaLogger"],
packages=["NeoMetaTracker"],
install_requires=["py2neo", "DZDUtils"],
python_requires=">=3.9",
zip_safe=False,
......
......@@ -10,41 +10,52 @@ if __name__ == "__main__":
os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__)))
)
SCRIPT_DIR = os.path.join(SCRIPT_DIR, "..")
print(os.path.normpath(SCRIPT_DIR))
sys.path.insert(0, os.path.normpath(SCRIPT_DIR))
from NeoMetaLogger import NeoMetaLogger
from NeoMetaLogger.visualizer import GraphvizVisualizer
from NeoMetaTracker import NeoMetaTracker
from NeoMetaTracker.visualizer import GraphvizVisualizer
NEO4J: Dict = json.loads(os.getenv("NEO4J", "{}"))
wait_for_db_boot(NEO4J)
current_dir = os.path.dirname(__file__)
graph = py2neo.Graph(**NEO4J)
graph.run("CREATE OR REPLACE DATABASE schematest")
graph.run("CREATE OR REPLACE DATABASE test")
test_graph = py2neo.Graph(**(NEO4J | {"name": "test"}))
# test_graph = py2neo.Graph(**(NEO4J))
mlog = NeoMetaLogger(test_graph)
mlog = NeoMetaTracker(test_graph)
mlog.capture(name="Init")
test_graph.run("CREATE (:Human{name:'Amina Okujewa'})")
test_graph.run("CREATE (as:Human{name:'Aaron Swartz'})")
mlog.capture(name="HumansCluster")
test_graph.run("CREATE (:Cat{name:'Grumpy Cat'})")
test_graph.run("CREATE (:Pig{name:'Miss Piggy'})")
mlog.capture("AnimalCluster")
test_graph.run("CREATE (:World {name: 'Earth'})")
test_graph.run("CREATE (:World {name: 'Internet'})")
test_graph.run("CREATE (:World{name:'House'})")
mlog.capture("WorldCluster")
test_graph.run(
"CREATE p = (:Human{name:'Amina Okujewa'})-[:LIVES_ON]->(:World {name: 'Earth'})"
"MATCH (wE:World{name:'Earth'}),(ao:Human{name:'Amina Okujewa'}) CREATE (ao)-[:LIVES_ON]->(wE)"
)
test_graph.run(
"CREATE p = (:Cat{name:'Grumpy Cat'})-[:LIVES_ON]->(:World {name: 'Internet'})"
"MATCH (wE:World{name:'Earth'}),(c:Cat{name:'Grumpy Cat'}) CREATE (c)-[:LIVES_ON]->(wE)"
)
test_graph.run(
"MATCH (wI:World{name:'Internet'}),(wE:World{name:'Earth'}),(as:Human{name:'Aaron Swartz'}) CREATE (as)-[:LIVES_ON]->(wI), (as)-[:LIVES_ON]->(wE)"
)
test_graph.run(
"MATCH (wI:World{name:'Internet'}),(wE:World{name:'Earth'}) CREATE (wI)-[:EXISTS_ON]->(wE)"
)
mlog.capture()
test_graph.run(
"MATCH (wI:World{name:'Internet'}),(wE:World{name:'Earth'}) CREATE (as:Human{name:'Aaron Swartz'})-[:LIVES_ON]->(wI), (as)-[:LIVES_ON]->(wE)"
"MATCH (wH:World{name:'House'}),(wE:World{name:'Earth'}) CREATE (wH)-[:EXISTS_ON]->(wE)"
)
mlog.capture()
test_graph.run("CREATE (wH:World{name:'House'})")
test_graph.run(
"MATCH (wH:World{name:'House'}),(wE:World{name:'Earth'}) CREATE (wH)-[:EXISTS_ON]->(wE)"
"MATCH (wH:World{name:'House'}),(wE:World{name:'Earth'}),(mp:Pig{name:'Miss Piggy'}) CREATE (mp)-[:LIVES_ON]->(wH),(mp)-[:LIVES_ON]->(wE)"
)
mlog.capture()
mlog.capture(name="RelationCluster")
print("###get_numeric_last_changes###\n", mlog.get_numeric_last_changes())
print("###get_schemagraph_last_changes###\n", mlog.get_schemagraph_last_changes())
print(
......@@ -53,8 +64,9 @@ print(
)
print(
mlog.get_schemagraph_last_changes().visualize(
GraphvizVisualizer, to_file="file.dot"
GraphvizVisualizer, to_file=os.path.join(current_dir, "last_change.dot")
)
)
# schema_graph = py2neo.Graph(**(NEO4J | {"name": "schematest"}))
# schema_graph.merge(mlog.get_schemagraph_last_changes())
print(mlog.visualize(GraphvizVisualizer, to_file=os.path.join(current_dir, "all.dot")))
schema_graph = py2neo.Graph(**(NEO4J | {"name": "schematest"}))
schema_graph.merge(mlog.get_schemagraph_last_changes())
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment