diff --git a/CI/physmon/tests/conftest.py b/CI/physmon/tests/conftest.py new file mode 100644 index 00000000000..dfac9e44f53 --- /dev/null +++ b/CI/physmon/tests/conftest.py @@ -0,0 +1,305 @@ +from pathlib import Path +import re +from typing import List, IO, Tuple, Optional +import threading +import csv +from datetime import datetime +import time +import shutil + +import pytest +import psutil +from pytest_check import check + +import acts +import acts.examples +from common import getOpenDataDetectorDirectory +from acts.examples.odd import getOpenDataDetector + + +@pytest.fixture(scope="session") +def output_path(request): + path: Path = request.config.getoption("--physmon-output-path").resolve() + path.mkdir(parents=True, exist_ok=True) + + return path + + +@pytest.fixture(scope="session") +def reference_path(request): + path: Path = request.config.getoption("--physmon-reference-path").resolve() + + return path + + +class Physmon: + detector: "acts.examples.dd4hep.DD4hepDetector" + trackingGeometry: acts.TrackingGeometry + decorators: List[acts.IMaterialDecorator] + field: acts.MagneticFieldProvider + digiConfig: Path + geoSel: Path + output_path: Path + reference_path: Path + update_references: bool + tmp_path: Path + name: str + + def __init__( + self, + detector, + trackingGeometry, + decorators, + field, + digiConfig, + geoSel, + output_path, + reference_path, + update_references, + tmp_path, + name, + ): + self.detector = detector + self.trackingGeometry = trackingGeometry + self.decorators = decorators + self.field = field + self.digiConfig = digiConfig + self.geoSel = geoSel + self.output_path = output_path + self.reference_path = reference_path + self.update_references = update_references + self.tmp_path = tmp_path + self.name = name + + self.test_output_path.mkdir(exist_ok=True, parents=True) + self.test_reference_path.mkdir(exist_ok=True, parents=True) + + @property + def test_output_path(self) -> Path: + return self.output_path / self.name + + @property + def test_reference_path(self) -> Path: + return self.reference_path / self.name + + def add_output_file(self, filename: str, rename: Optional[str] = None): + __tracebackhide__ = True + tmp = self.tmp_path / filename + assert tmp.exists(), f"Output file {tmp} does not exist" + outname = rename if rename else filename + shutil.copy(tmp, self.test_output_path / outname) + + def histogram_comparison( + self, filename: Path, title: str, config_path: Optional[Path] = None + ): + __tracebackhide__ = True + monitored = self.test_output_path / filename + reference = self.test_reference_path / filename + + assert monitored.exists(), f"Output file {monitored} does not exist" + + if self.update_references: + shutil.copy(monitored, reference) + assert reference.exists(), f"Reference file {reference} does not exist" + + from histcmp.console import Console + from histcmp.report import make_report + from histcmp.checks import Status + from histcmp.config import Config + from histcmp.github import is_github_actions, github_actions_marker + from histcmp.cli import print_summary + + from histcmp.compare import compare, Comparison + + from rich.panel import Panel + from rich.console import Group + from rich.pretty import Pretty + + import yaml + + console = Console() + + console.print( + Panel( + Group(f"Monitored: {monitored}", f"Reference: {reference}"), + title="Comparing files:", + ) + ) + + if config_path is None: + config = Config.default() + else: + with config_path.open() as fh: + config = Config(**yaml.safe_load(fh)) + + console.print(Panel(Pretty(config), title="Configuration")) + + # filter_path = Path(_filter) + # if filter_path.exists(): + # with filter_path.open() as fh: + # filters = fh.read().strip().split("\n") + # else: + # filters = [_filter] + filters = [] + comparison = compare( + config, monitored, reference, filters=filters, console=console + ) + + comparison.label_monitored = "monitored" + comparison.label_reference = "reference" + comparison.title = title + + status = print_summary(comparison, console) + + plots = self.test_output_path / "plots" + plots.mkdir(exist_ok=True, parents=True) + report_file = self.test_output_path / f"{monitored.stem}.html" + make_report(comparison, report_file, console, plots, format="pdf") + + for item in comparison.items: + msg = f"{item.key} failures: " + ", ".join( + [c.name for c in item.checks if c.status == Status.FAILURE] + ) + check.equal(item.status, Status.SUCCESS, msg=msg) + + +@pytest.fixture(scope="session") +def _physmon_prereqs(): + srcdir = Path(__file__).resolve().parent.parent.parent.parent + + matDeco = acts.IMaterialDecorator.fromFile( + srcdir / "thirdparty/OpenDataDetector/data/odd-material-maps.root", + level=acts.logging.INFO, + ) + + detector, trackingGeometry, decorators = getOpenDataDetector( + getOpenDataDetectorDirectory(), matDeco + ) + + return srcdir, detector, trackingGeometry, decorators + + +def _sanitize_test_name(name: str): + name = re.sub(r"^test_", "", name) + name = re.sub(r"\]$", "", name) + name = re.sub(r"[\[\]]", "_", name) + return name + + +@pytest.fixture() +def physmon( + output_path: Path, reference_path: Path, _physmon_prereqs, tmp_path, request +): + u = acts.UnitConstants + + srcdir, detector, trackingGeometry, decorators = _physmon_prereqs + + setup = Physmon( + detector=detector, + trackingGeometry=trackingGeometry, + decorators=decorators, + digiConfig=srcdir + / "thirdparty/OpenDataDetector/config/odd-digi-smearing-config.json", + geoSel=srcdir / "thirdparty/OpenDataDetector/config/odd-seeding-config.json", + field=acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T)), + output_path=output_path, + reference_path=reference_path, + update_references=request.config.getoption("--physmon-update-references"), + tmp_path=tmp_path, + name=_sanitize_test_name(request.node.name), + ) + return setup + + +# @TODO: try using spyral directly + + +class Monitor: + interval: float + + max_rss: float = 0 + max_vms: float = 0 + + terminate: bool = False + + exception = None + + def __init__( + self, + output: IO[str], + interval: float = 0.5, + ): + self.interval = interval + self.writer = csv.writer(output) + self.writer.writerow(("time", "rss", "vms")) + + self.time: List[float] = [0] + self.rss: List[float] = [0] + self.vms: List[float] = [0] + + @staticmethod + def _get_memory(p: psutil.Process) -> Tuple[float, float]: + rss = p.memory_info().rss + vms = p.memory_info().vms + for subp in p.children(recursive=True): + try: + rss += subp.memory_info().rss + vms += subp.memory_info().vms + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + return rss, vms + + def run(self, p: psutil.Process): + try: + start = datetime.now() + while p.is_running() and p.status() in ( + psutil.STATUS_RUNNING, + psutil.STATUS_SLEEPING, + ): + if self.terminate: + return + + delta = (datetime.now() - start).total_seconds() + + rss, vms = self._get_memory(p) + + self.rss.append(rss / 1e6) + self.vms.append(vms / 1e6) + self.time.append(delta) + self.max_rss = max(rss, self.max_rss) + self.max_vms = max(vms, self.max_vms) + + self.writer.writerow((delta, rss, vms)) + + time.sleep(self.interval) + except (psutil.NoSuchProcess, psutil.AccessDenied): + return + except Exception as e: + self.exception = e + raise e + + +@pytest.fixture(autouse=True) +def monitor(physmon: Physmon, request, capsys): + import psutil + + p = psutil.Process() + + memory = physmon.output_path / "memory" + memory.mkdir(exist_ok=True) + name = _sanitize_test_name(request.node.name) + with (memory / f"mem_{name}.csv").open("w") as fh: + mon = Monitor(output=fh, interval=0.1) + + t = threading.Thread(target=mon.run, args=(p,)) + t.start() + + yield + + mon.terminate = True + t.join() + + # @TODO: Add plotting + + # with capsys.disabled(): + # print("MONITORING") diff --git a/CI/physmon/tests/test_ckf_tracking.py b/CI/physmon/tests/test_ckf_tracking.py new file mode 100644 index 00000000000..58bc24dc98f --- /dev/null +++ b/CI/physmon/tests/test_ckf_tracking.py @@ -0,0 +1,201 @@ +from pathlib import Path + +import pytest + +import acts.examples + +from acts.examples.simulation import ( + addParticleGun, + MomentumConfig, + EtaConfig, + PhiConfig, + ParticleConfig, + addFatras, + addDigitization, +) + +from acts.examples.reconstruction import ( + addSeeding, + TruthSeedRanges, + ParticleSmearingSigmas, + SeedFinderConfigArg, + SeedFinderOptionsArg, + SeedingAlgorithm, + TruthEstimatedSeedingAlgorithmConfigArg, + addCKFTracks, + addAmbiguityResolution, + AmbiguityResolutionConfig, + addVertexFitting, + VertexFinder, + TrackSelectorConfig, +) + +from helpers import failure_threshold + +u = acts.UnitConstants + + +@pytest.mark.parametrize( + "seeding_algorithm", + [ + SeedingAlgorithm.Default, + SeedingAlgorithm.TruthSmeared, + SeedingAlgorithm.TruthEstimated, + SeedingAlgorithm.Orthogonal, + ], +) +def test_ckf_tracking(seeding_algorithm, physmon: "Physmon"): + s = acts.examples.Sequencer( + events=10, + numThreads=-1, + logLevel=acts.logging.INFO, + fpeMasks=acts.examples.Sequencer.FpeMask.fromFile( + Path(__file__).parent.parent / "fpe_masks.yml" + ), + ) + + for d in physmon.decorators: + s.addContextDecorator(d) + + rnd = acts.examples.RandomNumbers(seed=42) + + addParticleGun( + s, + MomentumConfig(1.0 * u.GeV, 10.0 * u.GeV, transverse=True), + EtaConfig(-3.0, 3.0), + PhiConfig(0.0, 360.0 * u.degree), + ParticleConfig(4, acts.PdgParticle.eMuon, randomizeCharge=True), + vtxGen=acts.examples.GaussianVertexGenerator( + mean=acts.Vector4(0, 0, 0, 0), + stddev=acts.Vector4(0.0125 * u.mm, 0.0125 * u.mm, 55.5 * u.mm, 1.0 * u.ns), + ), + multiplicity=50, + rnd=rnd, + ) + + addFatras( + s, + physmon.trackingGeometry, + physmon.field, + enableInteractions=True, + rnd=rnd, + ) + + addDigitization( + s, + physmon.trackingGeometry, + physmon.field, + digiConfigFile=physmon.digiConfig, + rnd=rnd, + ) + + addSeeding( + s, + physmon.trackingGeometry, + physmon.field, + TruthSeedRanges(pt=(500 * u.MeV, None), nHits=(9, None)), + ParticleSmearingSigmas(pRel=0.01), # only used by SeedingAlgorithm.TruthSmeared + SeedFinderConfigArg( + r=(33 * u.mm, 200 * u.mm), + deltaR=(1 * u.mm, 60 * u.mm), + collisionRegion=(-250 * u.mm, 250 * u.mm), + z=(-2000 * u.mm, 2000 * u.mm), + maxSeedsPerSpM=1, + sigmaScattering=5, + radLengthPerSeed=0.1, + minPt=500 * u.MeV, + impactMax=3 * u.mm, + ), + SeedFinderOptionsArg(bFieldInZ=2 * u.T), + TruthEstimatedSeedingAlgorithmConfigArg(deltaR=(10.0 * u.mm, None)), + seedingAlgorithm=seeding_algorithm, + geoSelectionConfigFile=physmon.geoSel, + rnd=rnd, # only used by SeedingAlgorithm.TruthSmeared + outputDirRoot=physmon.tmp_path, + ) + + addCKFTracks( + s, + physmon.trackingGeometry, + physmon.field, + TrackSelectorConfig( + pt=(500 * u.MeV, None), + loc0=(-4.0 * u.mm, 4.0 * u.mm), + nMeasurementsMin=6, + ), + outputDirRoot=physmon.tmp_path, + ) + + stems = [ + "performance_ckf", + "tracksummary_ckf", + ] + + associatedParticles = "particles_input" + if seeding_algorithm in [SeedingAlgorithm.Default, SeedingAlgorithm.Orthogonal]: + addAmbiguityResolution( + s, + AmbiguityResolutionConfig(maximumSharedHits=3), + outputDirRoot=physmon.tmp_path, + ) + + associatedParticles = None + + stems += ["performance_ambi"] + + addVertexFitting( + s, + physmon.field, + associatedParticles=associatedParticles, + outputProtoVertices="ivf_protovertices", + outputVertices="ivf_fittedVertices", + vertexFinder=VertexFinder.Iterative, + outputDirRoot=physmon.tmp_path / "ivf", + ) + + addVertexFitting( + s, + physmon.field, + associatedParticles=associatedParticles, + outputProtoVertices="amvf_protovertices", + outputVertices="amvf_fittedVertices", + vertexFinder=VertexFinder.AMVF, + outputDirRoot=physmon.tmp_path / "amvf", + ) + + with failure_threshold(acts.logging.FATAL): + s.run() + del s + + # @TODO: Add plotting into ROOT file for vertexing and ckf tracksummary + + for vertexing in ["ivf", "amvf"]: + target = f"performance_{vertexing}.root" + physmon.add_output_file( + f"{vertexing}/performance_vertexing.root", + rename=target, + ) + physmon.histogram_comparison(target, f"Performance {vertexing}") + + if seeding_algorithm in [ + SeedingAlgorithm.Default, + SeedingAlgorithm.Orthogonal, + SeedingAlgorithm.TruthEstimated, + ]: + stems += ["performance_seeding"] + + for stem in stems: + target = f"{stem}.root" + physmon.add_output_file(f"{stem}.root", rename=target) + physmon.histogram_comparison(target, f"Performance {physmon.name} {stem}") + + # ] + ( + # ["performance_seeding", "performance_ambi"] + # if label in ["seeded", "orthogonal"] + # else ["performance_seeding"] + # if label == "truth_estimated" + # else [] + # ): + # perf_file = physmon.tmp_path / f"{stem}.root" + # assert perf_file.exists(), "Performance file not found" + # shutil.copy(perf_file, physmon.outdir / f"{stem}_{label}.root") diff --git a/CI/physmon/tests/test_truth_tracking.py b/CI/physmon/tests/test_truth_tracking.py new file mode 100644 index 00000000000..f965ca83b71 --- /dev/null +++ b/CI/physmon/tests/test_truth_tracking.py @@ -0,0 +1,63 @@ +from pathlib import Path + +import acts.examples + +from truth_tracking_kalman import runTruthTrackingKalman +from truth_tracking_gsf import runTruthTrackingGsf + +from helpers import failure_threshold + + +def test_truth_tracking_kalman(physmon: "Physmon"): + s = acts.examples.Sequencer( + events=100, + numThreads=-1, + logLevel=acts.logging.INFO, + fpeMasks=acts.examples.Sequencer.FpeMask.fromFile( + Path(__file__).parent.parent / "fpe_masks.yml" + ), + ) + + runTruthTrackingKalman( + physmon.trackingGeometry, + physmon.field, + digiConfigFile=physmon.digiConfig, + outputDir=physmon.tmp_path, + s=s, + ) + + s.run() + del s + + physmon.add_output_file( + "performance_track_fitter.root", rename="performance_truth_tracking.root" + ) + physmon.histogram_comparison( + "performance_truth_tracking.root", title="Truth tracking KF" + ) + + +def test_truth_tracking_gsf(physmon: "Physmon"): + s = acts.examples.Sequencer( + events=500, + numThreads=-1, + logLevel=acts.logging.INFO, + fpeMasks=acts.examples.Sequencer.FpeMask.fromFile( + Path(__file__).parent.parent / "fpe_masks.yml" + ), + ) + + runTruthTrackingGsf( + physmon.trackingGeometry, + physmon.digiConfig, + physmon.field, + outputDir=physmon.tmp_path, + s=s, + ) + + with failure_threshold(acts.logging.FATAL): + s.run() + del s + + physmon.add_output_file("performance_gsf.root", rename="performance_gsf.root") + physmon.histogram_comparison("performance_gsf.root", title="Truth tracking GSF") diff --git a/Examples/Python/tests/conftest.py b/Examples/Python/tests/conftest.py index 3daec9c6a30..31b04f61e5c 100644 --- a/Examples/Python/tests/conftest.py +++ b/Examples/Python/tests/conftest.py @@ -1,60 +1,28 @@ -import multiprocessing from pathlib import Path -import sys -import os -import tempfile -import shutil from typing import Dict -import warnings -import pytest_check as check +import shutil +import tempfile +import os +import multiprocessing from collections import namedtuple - sys.path = [ str(Path(__file__).parent.parent.parent.parent / "Examples/Scripts/Python/"), str(Path(__file__).parent), ] + sys.path +import pytest import helpers import helpers.hash_root -import pytest - import acts import acts.examples from acts.examples.odd import getOpenDataDetector, getOpenDataDetectorDirectory from acts.examples.simulation import addParticleGun, EtaConfig, ParticleConfig -try: - import ROOT - - ROOT.gSystem.ResetSignals() -except ImportError: - pass - -try: - if acts.logging.getFailureThreshold() != acts.logging.WARNING: - acts.logging.setFailureThreshold(acts.logging.WARNING) -except RuntimeError: - # Repackage with different error string - errtype = ( - "negative" - if acts.logging.getFailureThreshold() < acts.logging.WARNING - else "positive" - ) - warnings.warn( - "Runtime log failure threshold could not be set. " - "Compile-time value is probably set via CMake, i.e. " - f"`ACTS_LOG_FAILURE_THRESHOLD={acts.logging.getFailureThreshold().name}` is set, " - "or `ACTS_ENABLE_LOG_FAILURE_THRESHOLD=OFF`. " - f"The pytest test-suite can produce false-{errtype} results in this configuration" - ) - - u = acts.UnitConstants - class RootHashAssertionError(AssertionError): def __init__( self, file: Path, key: str, exp_hash: str, act_hash: str, *args, **kwargs diff --git a/Examples/Python/tests/test_examples_vertexing.py b/Examples/Python/tests/test_examples_vertexing.py new file mode 100644 index 00000000000..0b9f442552a --- /dev/null +++ b/Examples/Python/tests/test_examples_vertexing.py @@ -0,0 +1,133 @@ +if False: + from pathlib import Path + + import pytest + + import acts + import acts.examples + from acts.examples import Sequencer, GenericDetector, RootParticleWriter + + from helpers import ( + dd4hepEnabled, + pythia8Enabled, + AssertCollectionExistsAlg, + assert_csv_output, + assert_entries, + assert_has_entries, + ) + + from acts.examples.odd import getOpenDataDetector + from common import getOpenDataDetectorDirectory + + u = acts.UnitConstants + + @pytest.mark.skipif(not dd4hepEnabled, reason="DD4hep not set up") + @pytest.mark.skipif(not pythia8Enabled, reason="Pythia8 not set up") + @pytest.mark.slow + @pytest.mark.odd + @pytest.mark.filterwarnings("ignore::UserWarning") + def test_vertex_fitting(tmp_path): + detector, trackingGeometry, decorators = getOpenDataDetector( + getOpenDataDetectorDirectory() + ) + + field = acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T)) + + from vertex_fitting import runVertexFitting, VertexFinder + + s = Sequencer(events=100) + + runVertexFitting( + field, + vertexFinder=VertexFinder.Truth, + outputDir=tmp_path, + s=s, + ) + + alg = AssertCollectionExistsAlg(["fittedVertices"], name="check_alg") + s.addAlgorithm(alg) + + s.run() + assert alg.events_seen == s.config.events + + @pytest.mark.parametrize( + "finder,inputTracks,entries", + [ + ("Truth", False, 100), + # ("Truth", True, 0), # this combination seems to be not working + ("Iterative", False, 100), + ("Iterative", True, 100), + ("AMVF", False, 100), + ("AMVF", True, 100), + ], + ) + @pytest.mark.filterwarnings("ignore::UserWarning") + @pytest.mark.flaky(reruns=2) + def test_vertex_fitting_reading( + tmp_path, ptcl_gun, rng, finder, inputTracks, entries, assert_root_hash + ): + ptcl_file = tmp_path / "particles.root" + + detector, trackingGeometry, decorators = GenericDetector.create() + field = acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T)) + + from vertex_fitting import runVertexFitting, VertexFinder + + inputTrackSummary = None + if inputTracks: + from truth_tracking_kalman import runTruthTrackingKalman + + s2 = Sequencer(numThreads=1, events=100) + runTruthTrackingKalman( + trackingGeometry, + field, + digiConfigFile=Path( + Path(__file__).parent.parent.parent.parent + / "Examples/Algorithms/Digitization/share/default-smearing-config-generic.json" + ), + outputDir=tmp_path, + s=s2, + ) + s2.run() + del s2 + inputTrackSummary = tmp_path / "tracksummary_fitter.root" + assert inputTrackSummary.exists() + assert ptcl_file.exists() + else: + s0 = Sequencer(events=100, numThreads=1) + evGen = ptcl_gun(s0) + s0.addWriter( + RootParticleWriter( + level=acts.logging.INFO, + inputParticles=evGen.config.outputParticles, + filePath=str(ptcl_file), + ) + ) + s0.run() + del s0 + + assert ptcl_file.exists() + + finder = VertexFinder[finder] + + s3 = Sequencer(numThreads=1) + + runVertexFitting( + field, + inputParticlePath=ptcl_file, + inputTrackSummary=inputTrackSummary, + outputDir=tmp_path, + vertexFinder=finder, + s=s3, + ) + + alg = AssertCollectionExistsAlg(["fittedVertices"], name="check_alg") + s3.addAlgorithm(alg) + + s3.run() + + vertexing_file = tmp_path / "performance_vertexing.root" + assert vertexing_file.exists() + + assert_entries(vertexing_file, "vertexing", entries) + assert_root_hash(vertexing_file.name, vertexing_file) diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000000..f50062f1286 --- /dev/null +++ b/conftest.py @@ -0,0 +1,59 @@ +from pathlib import Path +import sys +import warnings +import pytest_check as check + + +sys.path += [ + str(Path(__file__).parent / "Examples/Scripts"), + str(Path(__file__).parent / "Examples/Scripts/Python"), + str(Path(__file__).parent / "Examples/Python/tests"), +] + + +import pytest + +import acts + +try: + import ROOT + + ROOT.gSystem.ResetSignals() + ROOT.gROOT.SetBatch(ROOT.kTRUE) +except ImportError: + pass + +try: + if acts.logging.getFailureThreshold() != acts.logging.WARNING: + acts.logging.setFailureThreshold(acts.logging.WARNING) +except RuntimeError: + # Repackage with different error string + errtype = ( + "negative" + if acts.logging.getFailureThreshold() < acts.logging.WARNING + else "positive" + ) + warnings.warn( + "Runtime log failure threshold could not be set. " + "Compile-time value is probably set via CMake, i.e. " + f"`ACTS_LOG_FAILURE_THRESHOLD={acts.logging.getFailureThreshold().name}` is set, " + "or `ACTS_ENABLE_LOG_FAILURE_THRESHOLD=OFF`. " + f"The pytest test-suite can produce false-{errtype} results in this configuration" + ) + + +def pytest_addoption(parser): + parser.addoption( + "--physmon-output-path", + action="store", + default=Path.cwd() / "physmon", + type=Path, + ) + parser.addoption( + "--physmon-reference-path", + action="store", + default=Path(__file__).parent / "CI/physmon/reference", + type=Path, + ) + + parser.addoption("--physmon-update-references", action="store_true") diff --git a/pytest.ini b/pytest.ini index 6e1657f482e..c581e301e68 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,8 @@ [pytest] testpaths = Examples/Python/tests + CI/physmon/tests +addopts = --import-mode=importlib norecursedirs=Examples/Python/tests/helpers markers = csv