exp(moviekg): updates to moviekg evaluation

Author: Vehnem

experiments/moviekg/.gitignore

@@ -0,0 +1 @@
+*.yaml

experiments/moviekg/eval.sh

@@ -0,0 +1,10 @@
+kgpipe eval -c metric_config.yaml \
+	-m ReferenceTripleAlignmentMetricSoftEV \
+	-m entity_count \
+	-m incorrect_relation_direction \
+	-m incorrect_relation_cardinality \
+	-m incorrect_relation_range \
+	-m incorrect_relation_domain \
+	-m incorrect_datatype \
+	-m incorrect_datatype_format \
+	data/out/small/rdf_a/stage_3/result.nt

experiments/moviekg/src/moviekg/evaluation/test_sensitivity.py

@@ -1,7 +1,14 @@
 from dataclasses import dataclass
 from typing import List
-from kgpipe.evaluation.aspects.reference import ReferenceEvaluator, ReferenceConfig
-
+from kgpipe.common import KgPipe, Data, DataFormat, KG
+from pathlib import Path
+from kgpipe.common.models import KgPipePlan
+from kgpipe.evaluation.aspects.reference import (
+    ReferenceEvaluator, ReferenceConfig, 
+    ER_EntityMatchMetric, ER_RelationMatchMetric, 
+    TE_ExpectedEntityLinkMetric, TE_ExpectedRelationLinkMetric
+)
+import os
 @dataclass
 class BinaryClassifier:
     tp: int
@@ -21,20 +28,117 @@ class ThresholdSensitivityResult:
     threshold: float
     result: BinaryClassifier
 
+benchdata = Path("/home/marvin/phd/kgpipe/experiments/moviekg/data/datasets/film_10k/")
+seed_path = benchdata / "split_0/kg/seed/data.nt"
+rdf_path = benchdata / "split_1/sources/rdf/data.nt"
+result_dir_path = Path(f"data/moviekg/threshold_sensitivity/")
 
+# reference_evaluator = ReferenceEvaluator()
 
-reference_evaluator = ReferenceEvaluator()
+def run_paris_pipeline(pipeline_name: str, threshold: float) -> List[ThresholdSensitivityResult]:
+    from kgpipe_tasks.tasks import paris_entity_matching, paris_exchange
 
-def paris_threshold_sensitivity(pipeline_name: str, threshold: float) -> List[ThresholdSensitivityResult]:
-    ReferenceConfig(
-        ENTITY_MATCH_THRESHOLD=threshold
-        RELATION_MATCH_THRESHOLD=threshold        
-    ) # TODO get config from dataset
-    # kg = KG(path=Path(f"data/moviekg/paris/{pipeline_name}.nt"))
-    # reference_kg = KG(path=Path("data/moviekg/paris/reference.nt"))
-    # result = reference_evaluator.evaluate(kg, reference_kg)
-    # return result
-    pass
+    pipe_result_dir_path = result_dir_path / f"{pipeline_name}"
+    pipeline = KgPipe(
+        name="paris pipeline",
+        tasks=[paris_entity_matching, paris_exchange],
+        seed=Data(path=seed_path, format=DataFormat.RDF_NTRIPLES),
+        data_dir=pipe_result_dir_path / "tmp"
+    )
+    plan = pipeline.build(
+        source=Data(path=rdf_path, format=DataFormat.RDF_NTRIPLES),
+        result=Data(path=pipe_result_dir_path / "result.json", format=DataFormat.ER_JSON)
+    )
+
+    os.makedirs(pipe_result_dir_path, exist_ok=True)
+
+    with open(pipe_result_dir_path / "exec-plan.json", "w") as f:
+        f.write(plan.model_dump_json(indent=4))
+
+    pipeline.run()
+
+def paris_er_threshold_sensitivity(pipeline_name: str, threshold: float) -> List[ThresholdSensitivityResult]:
+    config = ReferenceConfig(
+        name="paris config",
+        ENTITY_MATCH_THRESHOLD=threshold,
+        RELATION_MATCH_THRESHOLD=threshold,
+        GT_MATCHES=benchdata / "split_1/sources/rdf/meta/verified_matches.csv",
+        GT_MATCHES_TARGET_DATASET="split_0/kg/seed"
+    ) 
+
+    plan = KgPipePlan.model_validate_json(open(result_dir_path / f"{pipeline_name}" / "exec-plan.json").read())
+
+    kg = KG(id="paris", name="paris", path=Path(f"data/moviekg/paris/{pipeline_name}.nt"), format=DataFormat.RDF_NTRIPLES, plan=plan)
+
+    metric_result = ER_EntityMatchMetric().compute(kg, config=config)
+    # print(metric_result)
+
+    return metric_result
+
+def paris_om_threshold_sensitivity(pipeline_name: str, threshold: float) -> List[ThresholdSensitivityResult]:
+    config = ReferenceConfig(
+        name="paris config",
+        ENTITY_MATCH_THRESHOLD=threshold,
+        RELATION_MATCH_THRESHOLD=threshold,
+        GT_MATCHES=benchdata / "split_1/sources/rdf/meta/verified_matches.csv",
+        GT_MATCHES_TARGET_DATASET="split_0/kg/seed"
+    ) 
+
+    plan = KgPipePlan.model_validate_json(open(result_dir_path / f"{pipeline_name}" / "exec-plan.json").read())
+
+    kg = KG(id="paris", name="paris", path=Path(f"data/moviekg/paris/{pipeline_name}.nt"), format=DataFormat.RDF_NTRIPLES, plan=plan)
+
+    metric_result = ER_RelationMatchMetric().compute(kg, config=config)
+    # print(metric_result)
+
+    return metric_result
+
+def test_paris():
+    # run_paris_pipeline("paris", 0.99)
+    range_of_thresholds = [0.0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1.0]
+    
+    er_results = []
+    for threshold in range_of_thresholds:
+        result = paris_er_threshold_sensitivity("paris", threshold)
+        er_results.append([threshold, result.normalized_score, result.details])
+
+    print()
+    print("ER Results:")
+    for r in er_results:
+        print(r[0], r[1], r[2])
+
+    om_results = []
+    for threshold in range_of_thresholds:
+        result = paris_om_threshold_sensitivity("paris", threshold)
+        om_results.append([threshold, result.normalized_score, result.details])
+
+    print("OM Results:")
+    for r in om_results:
+        print(r[0], r[1], r[2])
+
+# def paris_threshold_sensitivity(pipeline_name: str, threshold: float) -> List[ThresholdSensitivityResult]:
+#     result = run_paris_pipeline(pipeline_name, threshold)
+
+#     pipeline.run(
+#         input=[Data(path=Path(f"data/moviekg/paris/{pipeline_name}.nt"), format=DataFormat.RDF_NTRIPLES)],
+#         output=[Data(path=Path(f"data/moviekg/paris/{pipeline_name}.paris_csv"), format=DataFormat.PARIS_CSV)]
+#     )
+
+#     config = ReferenceConfig(
+#         name="paris config",
+#         ENTITY_MATCH_THRESHOLD=threshold,
+#         RELATION_MATCH_THRESHOLD=threshold        
+#     ) 
+
+
+
+
+#     # TODO get config from dataset
+#     # kg = KG(path=Path(f"data/moviekg/paris/{pipeline_name}.nt"))
+#     # reference_kg = KG(path=Path("data/moviekg/paris/reference.nt"))
+#     # result = reference_evaluator.evaluate(kg, reference_kg)
+#     # return result
+#     pass
 
 def jedai_threshold_sensitivity(pipeline_name: str, threshold: float) -> List[ThresholdSensitivityResult]:
     pass

experiments/moviekg/src/moviekg/paper/helpers/getter.py

@@ -168,7 +168,8 @@ def ref_source_typed_entity_p(df: pd.DataFrame):
     res: pipeline_stage_dict = defaultdict[pipeline_name, defaultdict[stage_name, metric_value]](lambda: defaultdict[stage_name, metric_value](lambda: None))
     for row in df.itertuples():
         details = json.loads(row.details)
-        precision = details["precision"]
+        # print(details)
+        precision = details.get("fn", -1)
         res[row.pipeline][row.stage] = precision
     return res
 
@@ -177,7 +178,7 @@ def ref_source_typed_entity_r(df: pd.DataFrame):
     res: pipeline_stage_dict = defaultdict[pipeline_name, defaultdict[stage_name, metric_value]](lambda: defaultdict[stage_name, metric_value](lambda: None))
     for row in df.itertuples():
         details = json.loads(row.details)
-        recall = details["recall"]
+        recall = details.get("recall", -1)
         res[row.pipeline][row.stage] = recall
     return res
 

experiments/moviekg/src/moviekg/paper/test_figtab.py

@@ -719,4 +719,64 @@ def test_new_ranking_table():
     # # metric_df = metric_df.reset_index(drop=True)
     # # metric_df = metric_df.pivot(index="pipeline", columns="metric", values="normalized")
     # # metric_df = metric_df.reset_index()
-    # metric_df.to_csv(OUTPUT_ROOT / "paper/test_tab_8_new_ranking_table.csv", sep="\t")
+    # metric_df.to_csv(OUTPUT_ROOT / "paper/test_tab_8_new_ranking_table.csv", sep="\t")
+
+
+def test_new_quality_table():
+
+    metric_df = load_metrics_from_file(OUTPUT_ROOT / "all_metrics.csv")
+    metric_df["pipeline"] = metric_df["pipeline"].map(map_pipeline_name_pretty)
+    from moviekg.paper.helpers.getter import (
+        get_pipeline_stage_metric_dict, 
+        sta_entity_count, sta_fact_count, sta_type_count, sta_relation_count, sta_shallow_entity_count, sta_denisity, sta_duration,
+        ref_kg_f1, ref_kg_p, ref_kg_r, 
+        ref_source_entity_f1, ref_source_entity_p, ref_source_entity_r, 
+        ref_source_typed_entity_r, ref_source_typed_entity_p,
+        sem_disjoint_domain, sem_incorrect_relation_direction, sem_incorrect_relation_cardinality, sem_incorrect_relation_range, sem_incorrect_relation_domain, sem_incorrect_datatype, sem_incorrect_datatype_format,
+    )
+
+    metrics = [
+        sta_entity_count.__name__, sta_fact_count.__name__, sta_type_count.__name__, sta_relation_count.__name__, sta_shallow_entity_count.__name__, sta_denisity.__name__, sta_duration.__name__,
+        ref_kg_f1.__name__, ref_kg_p.__name__, 
+        ref_kg_r.__name__, ref_source_entity_f1.__name__, 
+        ref_source_entity_p.__name__, ref_source_entity_r.__name__, 
+        ref_source_typed_entity_r.__name__, ref_source_typed_entity_p.__name__,
+        sem_disjoint_domain.__name__, sem_incorrect_relation_direction.__name__, sem_incorrect_relation_cardinality.__name__, sem_incorrect_relation_range.__name__, sem_incorrect_relation_domain.__name__, sem_incorrect_datatype.__name__, sem_incorrect_datatype_format.__name__,
+    ]
+
+    psmd = get_pipeline_stage_metric_dict(metric_df, metrics)
+    # import json
+    # json.dump(psmd, open(OUTPUT_ROOT / "paper/test_tab_6_metrics.json", "w"), indent=4)
+
+    rows = []
+
+    round_to = 3
+
+    for pipeline, stage_dict in psmd.items():
+        if pipeline in ["reference", "seed"]:
+            continue
+
+        for stage, metric_dict in stage_dict.items():
+            ec = round(metric_dict.get(sta_entity_count.__name__, -1), round_to)
+            kg_p = round(metric_dict.get(ref_kg_p.__name__, -1), round_to)
+            kg_r = round(metric_dict.get(ref_kg_r.__name__, -1), round_to)
+            se_p = round(metric_dict.get(ref_source_entity_p.__name__, -1), round_to)
+            se_r= round(metric_dict.get(ref_source_entity_r.__name__, -1), round_to)
+            ste_p = round(metric_dict.get(ref_source_typed_entity_p.__name__, -1), round_to)
+            ste_r = round(metric_dict.get(ref_source_typed_entity_r.__name__, -1), round_to)
+            o_dt = round(metric_dict.get(sem_disjoint_domain.__name__, -1), round_to)
+            o_d = round(metric_dict.get(sem_incorrect_relation_domain.__name__, -1), round_to)
+            o_r = round(metric_dict.get(sem_incorrect_relation_range.__name__, -1), round_to)
+            o_rd = round(metric_dict.get(sem_incorrect_relation_direction.__name__, -1), round_to)
+            o_lt = round(metric_dict.get(sem_incorrect_datatype.__name__, -1), round_to)
+            o_lf = round(metric_dict.get(sem_incorrect_datatype_format.__name__, -1), round_to)
+        
+            rows.append({
+                "pipeline": pipeline, "stage": stage,
+                "EC": ec,
+                "kg_p": kg_p, "kg_r": kg_r, "se_p": se_p, "se_r": se_r, "ste_p": ste_p, "ste_r": ste_r, 
+                "O_DT": o_dt, "O_D": o_d, "O_R": o_r, "O_RD": o_rd, "O_LT": o_lt, "O_LF": o_lf
+            })
+
+    df = pd.DataFrame(rows)
+    df.to_csv(OUTPUT_ROOT / "paper/test_tab_9_new_quality_table.csv", sep="\t")