diff --git a/.gitignore b/.gitignore index 3ab4b839..0832ad6b 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,18 @@ nohup.out artifacts/ data/ docs/site/ +postgres_data/ +postgres_lock/ + +# Singularity files +*.sif +*.def + +# Postgres directories +postgres_config +postgres_run +postgres_log +postgres_data + +# tokenizer data +nltk_data diff --git a/Dockerfile b/Dockerfile index 226bf401..82af74ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,8 +43,7 @@ RUN conda install -y psycopg2 tqdm && \ wandb==0.16.6 chardet==5.2.0 nltk==3.8.1 igraph==0.11.5 \ cairocffi==1.7.0 wget==3.2 -RUN conda install -y pytorch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1 \ - pytorch-cuda=11.7 -c pytorch -c nvidia +RUN pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu117 RUN pip install torch_geometric==2.5.3 --no-cache-dir && \ pip install pyg_lib==0.2.0 torch_scatter==2.1.1 torch_sparse==0.6.17 \ diff --git a/compose-postgres.yml b/compose-postgres.yml index 5553d5ef..fe074d44 100644 --- a/compose-postgres.yml +++ b/compose-postgres.yml @@ -12,7 +12,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data - ./postgres/init-create-empty-databases.sh:/docker-entrypoint-initdb.d/init-create-empty-databases.sh - - ./settings/scripts:/scripts + - ./scripts:/scripts - ${INPUT_DIR:-/data}:/data healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] diff --git a/config/default.yml b/config/default.yml index 544bd512..09910c33 100644 --- a/config/default.yml +++ b/config/default.yml @@ -92,6 +92,7 @@ detection: activation: prelu custom_mlp: architecture_str: none + x_is_tuple: False decoder: predict_edge_type: decoder: edge_mlp diff --git a/config/velox.yml b/config/velox.yml index 372f0b23..98892957 100644 --- a/config/velox.yml +++ b/config/velox.yml @@ -9,3 +9,4 @@ detection: gnn_training: encoder: used_methods: none # uses only a linear layer + x_is_tuple: True diff --git a/docs/docs/create-db-from-scratch.md b/docs/docs/create-db-from-scratch.md index 7bfb6bd3..d17e8da1 100644 --- a/docs/docs/create-db-from-scratch.md +++ b/docs/docs/create-db-from-scratch.md @@ -7,7 +7,7 @@ You can download all required files directly by running: pip install gdown ``` ```shell -./settings/scripts/download_{dataset}.sh {data_folder} +./scripts/download_{dataset}.sh {data_folder} ``` where `{dataset}` can be either `clearscope_e3`, `cadets_e3`, `theia_e3`, `clearscope_e5`, `cadets_e5` or `theia_e5` and `{data_folder}` is the absolute path to the output folder where all raw files will be downloaded. @@ -26,14 +26,18 @@ sudo docker compose exec pids bash 4. Convert the DARPA files ```shell -./settings/scripts/uncompress_darpa_files.sh /data/ +./scripts/uncompress_darpa_files.sh /data/ ``` > [!NOTE] > This may take multiple hours depending on the dataset. ### Optional configurations -- optionally, if using a specific postgres database instead of the postgres docker, update the connection config by setting `DATABASE_DEFAULT_CONFIG` within `pidsmaker/config.py`. +- optionally, if using a specific postgres database instead of the postgres docker, pass the details as command line arguments to the python scripts + - `--database_host`: the host machine where the database is located (default: `postgres`) + - `--database_user`: the database user to connect to the database (default: `postgres`) + - `--database_password`: the password for the database user (default: `postgres`) + - `--database_port`: the port number for Postgres (default: `5432`) - optionaly, if you want to change the output folder where generated files are stored, update accordingly the volume by uncommenting `./artifacts:/home/artifacts` in `compose.yml`. diff --git a/docs/docs/docs/docs/introduction.md b/docs/docs/docs/docs/introduction.md new file mode 100644 index 00000000..e69de29b diff --git a/docs/docs/singularity_install.md b/docs/docs/singularity_install.md new file mode 100644 index 00000000..c54fb620 --- /dev/null +++ b/docs/docs/singularity_install.md @@ -0,0 +1,39 @@ +# Install Framework using Singularity + +For quick installation on environments where Docker is not available (such as HPC clusters), you can use Singularity. This guide assumes Singularity is already installed on your system. + +## Setup Process + +### 1. Database Setup +The Makefile in `./scripts/Makefile` provides easy environment setup: + +```bash +make full-setup +``` + +This command will: +- Download and run a PostgreSQL container through Singularity +- Load database dumps by executing the `load_dumps.sh` script + +### 2. Container Management +Once the database is ready: +- Stop the container: `make down` +- Start it again: `make up` + +### 3. Dependencies Installation +Install all required dependencies using conda: + +```bash +conda env create -f ./scripts/environment.yml +conda activate pids +``` + +## Running the Framework + +Once both the database and conda environment are ready, run the framework with: + +```bash +python pidsmaker/main.py SYSTEM DATASET +``` + +For more details, see the [introduction](introduction.md). \ No newline at end of file diff --git a/pidsmaker/config/config.py b/pidsmaker/config/config.py index 923cbeff..15c27b49 100644 --- a/pidsmaker/config/config.py +++ b/pidsmaker/config/config.py @@ -541,17 +541,13 @@ def __init__(self, type, vals: list = None, desc: str = None): ), "file": Arg( str, - vals=AND( - ["type", "path"], - desc="Which features use for file nodes. Features will be concatenated.", - ), + vals=AND(["type", "path"]), + desc="Which features use for file nodes. Features will be concatenated.", ), "netflow": Arg( str, - vals=AND( - ["type", "remote_ip", "remote_port"], - desc="Which features use for netflow nodes. Features will be concatenated.", - ), + vals=AND(["type", "remote_ip", "remote_port"]), + desc="Which features use for netflow nodes. Features will be concatenated.", ), }, "multi_dataset": Arg( @@ -711,7 +707,9 @@ def __init__(self, type, vals: list = None, desc: str = None): }, "gnn_training": { "use_seed": Arg(bool), - "deterministic": Arg(bool, desc="Whether to force PyTorch to use deterministic algorithms."), + "deterministic": Arg( + bool, desc="Whether to force PyTorch to use deterministic algorithms." + ), "num_epochs": Arg(int), "patience": Arg(int), "lr": Arg(float), @@ -724,9 +722,7 @@ def __init__(self, type, vals: list = None, desc: str = None): "inference_device": Arg( str, vals=OR(["cpu", "cuda"]), desc="Device used during testing." ), - "used_method": Arg( - str, vals=OR(["default"]), desc="Which training pipeline use." - ), + "used_method": Arg(str, vals=OR(["default"]), desc="Which training pipeline use."), "encoder": { "dropout": Arg(float), "used_methods": Arg( @@ -734,6 +730,7 @@ def __init__(self, type, vals: list = None, desc: str = None): vals=AND(list(ENCODERS_CFG.keys())), desc="First part of the neural network. Usually GNN encoders to capture complex patterns.", ), + "x_is_tuple": Arg(bool, desc="Whether to consider nodes differently when being source or destination."), **ENCODERS_CFG, }, "decoder": { diff --git a/pidsmaker/config/pipeline.py b/pidsmaker/config/pipeline.py index 7112751e..c17785ab 100644 --- a/pidsmaker/config/pipeline.py +++ b/pidsmaker/config/pipeline.py @@ -24,17 +24,9 @@ Arg, ) -DEFAULT_ROOT_ARTIFACT_DIR = "/home/artifacts/" # Destination folder (in the container) for generated files. Will be created if doesn't exist. ROOT_PROJECT_PATH = pathlib.Path(__file__).parent.parent.parent.resolve() ROOT_GROUND_TRUTH_DIR = os.path.join(ROOT_PROJECT_PATH, "Ground_Truth/") - -DATABASE_DEFAULT_CONFIG = { - "host": "postgres", # Host machine where the db is located - "user": "postgres", # Database user - "password": "postgres", # The password to the database user - "port": "5432", # The port number for Postgres -} # ================================================================================ @@ -43,7 +35,7 @@ def get_default_cfg(args): Inits the shared cfg object with default configurations. """ cfg = CN() - cfg._artifact_dir = args.artifact_dir_in_container or DEFAULT_ROOT_ARTIFACT_DIR + cfg._artifact_dir = args.artifact_dir cfg._test_mode = args.test_mode cfg._debug = not args.wandb @@ -64,8 +56,10 @@ def get_default_cfg(args): # Database: we simply create variables for all configurations described in the dict cfg.database = CN() - for attr, value in DATABASE_DEFAULT_CONFIG.items(): - setattr(cfg.database, attr, value) + cfg.database.host = args.database_host + cfg.database.user = args.database_user + cfg.database.password = args.database_password + cfg.database.port = args.database_port # Dataset: we simply create variables for all configurations described in the dict set_dataset_cfg(cfg, args.dataset) @@ -112,7 +106,7 @@ def get_runtime_required_args(return_unknown_args=False, args=None): ) parser.add_argument("--wandb", action="store_true", help="Whether to submit logs to wandb") parser.add_argument( - "--project", type=str, default="", help="Name of the wandb project (optional)" + "--project", type=str, default="PIDSMaker", help="Name of the wandb project" ) parser.add_argument("--exp", type=str, default="", help="Name of the experiment") parser.add_argument( @@ -139,9 +133,21 @@ def get_runtime_required_args(return_unknown_args=False, args=None): parser.add_argument( "--tuning_file_path", default="", help="If set, use the given YML path for tuning" ) + parser.add_argument( + "--database_host", default="postgres", help="Host machine where the db is located" + ) + parser.add_argument( + "--database_user", default="postgres", help="Database user to connect to the database" + ) + parser.add_argument( + "--database_password", default="postgres", help="The password to the database user" + ) + parser.add_argument( + "--database_port", default="5432", help="The port number for Postgres (default: 5432)" + ) parser.add_argument("--sweep_id", default="", help="ID of a wandb sweep for multi-agent runs") parser.add_argument( - "--artifact_dir_in_container", default="", help="ID of a wandb sweep for multi-agent runs" + "--artifact_dir", default="/home/artifacts/", help="Destination folder for generated files" ) parser.add_argument( "--test_mode", @@ -450,7 +456,7 @@ def get_yml_cfg(args): # Inits with default configurations cfg = get_default_cfg(args) - + # Checks that all configurations are valid and merge yml file to cfg yml_file = get_yml_file(args.model) merge_cfg_and_check_syntax(cfg, yml_file) diff --git a/pidsmaker/detection/evaluation_methods/evaluation_utils.py b/pidsmaker/detection/evaluation_methods/evaluation_utils.py index af055268..a8e7cc86 100644 --- a/pidsmaker/detection/evaluation_methods/evaluation_utils.py +++ b/pidsmaker/detection/evaluation_methods/evaluation_utils.py @@ -40,6 +40,7 @@ def classifier_evaluation(y_test, y_test_pred, scores): if labels_exist: tn, fp, fn, tp = confusion_matrix(y_test, y_test_pred).ravel() else: + log("WARNING: Computing confusion matrix failed.") tn, fp, fn, tp = 1, 1, 1, 1 # only to not break tests eps = 1e-12 @@ -51,15 +52,18 @@ def classifier_evaluation(y_test, y_test_pred, scores): try: auc_val = roc_auc_score(y_test, scores) - except: + except ValueError as e: + log(f"WARNING: AUC calculation failed: {e}") auc_val = float("nan") try: ap = ap_score(y_test, scores) - except: + except ValueError as e: + log(f"WARNING: AP calculation failed: {e}") ap = float("nan") try: balanced_acc = balanced_accuracy_score(y_test, y_test_pred) - except: + except ValueError as e: + log(f"WARNING: Balanced ACC calculation failed: {e}") balanced_acc = float("nan") sensitivity = tp / (tp + fn + eps) @@ -659,13 +663,12 @@ def plot_detected_attacks_vs_precision(scores, nodes, node2attacks, labels, out_ # Update tp and fp based on label if sorted_labels[i] == 1: tp += 1 + # Update detected attacks set if node has associated attacks + if node in node2attacks: + detected_attacks.update(node2attacks[node]) else: fp += 1 - # Update detected attacks set if node has associated attacks - if node in node2attacks: - detected_attacks.update(node2attacks[node]) - # Calculate precision and detected attacks percentage precision = tp / (tp + fp) detected_attacks_percentage = (len(detected_attacks) / total_attacks) * 100 diff --git a/pidsmaker/detection/evaluation_methods/node_evaluation.py b/pidsmaker/detection/evaluation_methods/node_evaluation.py index 665d5e03..42e21ab0 100644 --- a/pidsmaker/detection/evaluation_methods/node_evaluation.py +++ b/pidsmaker/detection/evaluation_methods/node_evaluation.py @@ -37,7 +37,7 @@ def get_node_predictions(val_tw_path, test_tw_path, cfg, **kwargs): log(f"Loading data from {test_tw_path}...") threshold_method = cfg.detection.evaluation.node_evaluation.threshold_method - if threshold_method == "magic": + if threshold_method == "magic": # data leaking by using test data thr = get_threshold(test_tw_path, threshold_method) else: thr = get_threshold(val_tw_path, threshold_method) @@ -98,7 +98,8 @@ def get_node_predictions(val_tw_path, test_tw_path, cfg, **kwargs): if use_kmeans: results = compute_kmeans_labels( - results, topk_K=cfg.detection.evaluation.node_evaluation.kmeans_top_K + results, + topk_K=cfg.detection.evaluation.node_evaluation.kmeans_top_K, ) return results, thr @@ -193,7 +194,8 @@ def get_node_predictions_node_level(val_tw_path, test_tw_path, cfg, **kwargs): else: pred_score = reduce_losses_to_score( - losses["loss"], cfg.detection.evaluation.node_evaluation.threshold_method + losses["loss"], + cfg.detection.evaluation.node_evaluation.threshold_method, ) results[node_id]["score"] = pred_score @@ -222,7 +224,8 @@ def get_node_predictions_node_level(val_tw_path, test_tw_path, cfg, **kwargs): if use_kmeans: results = compute_kmeans_labels( - results, topk_K=cfg.detection.evaluation.node_evaluation.kmeans_top_K + results, + topk_K=cfg.detection.evaluation.node_evaluation.kmeans_top_K, ) return results, thr @@ -244,7 +247,14 @@ def analyze_false_positives( return fp_in_malicious_tw_ratio -def main(val_tw_path, test_tw_path, model_epoch_dir, cfg, tw_to_malicious_nodes, **kwargs): +def main( + val_tw_path, + test_tw_path, + model_epoch_dir, + cfg, + tw_to_malicious_nodes, + **kwargs, +): if cfg._is_node_level: get_preds_fn = get_node_predictions_node_level else: @@ -334,7 +344,12 @@ def main(val_tw_path, test_tw_path, model_epoch_dir, cfg, tw_to_malicious_nodes, stats = classifier_evaluation(y_truth, y_preds, pred_scores) fp_in_malicious_tw_ratio = analyze_false_positives( - y_truth, y_preds, pred_scores, max_val_loss_tw, nodes, tw_to_malicious_nodes + y_truth, + y_preds, + pred_scores, + max_val_loss_tw, + nodes, + tw_to_malicious_nodes, ) stats["fp_in_malicious_tw_ratio"] = round(fp_in_malicious_tw_ratio, 3) @@ -345,7 +360,7 @@ def main(val_tw_path, test_tw_path, model_epoch_dir, cfg, tw_to_malicious_nodes, tps_in_atts.append((att, tps)) stats["percent_detected_attacks"] = ( - round(len(attack_to_GPs) / len(attack_to_TPs), 2) if len(attack_to_TPs) > 0 else 0 + round(len(attack_to_TPs) / len(attack_to_GPs), 2) if len(attack_to_GPs) > 0 else 0 ) fps, tps, precision, recall = get_metrics_if_all_attacks_detected( diff --git a/pidsmaker/detection/training_methods/inference_loop.py b/pidsmaker/detection/training_methods/inference_loop.py index 0847a97e..a84328ee 100644 --- a/pidsmaker/detection/training_methods/inference_loop.py +++ b/pidsmaker/detection/training_methods/inference_loop.py @@ -30,7 +30,6 @@ def test_edge_level( ): model.eval() - edge_list = None start_time = data.t[0] all_losses = [] @@ -63,21 +62,17 @@ def test_edge_level( "edge_type": edge_types.astype(int), } ) - if edge_list is None: - edge_list = edge_df - else: - edge_list = pd.concat([edge_list, edge_df]) # Here is a checkpoint, which records all edge losses in the current time window time_interval = ( - ns_time_to_datetime_US(start_time) + "~" + ns_time_to_datetime_US(edge_list["time"].max()) + ns_time_to_datetime_US(start_time) + "~" + ns_time_to_datetime_US(edge_df["time"].max()) ) logs_dir = os.path.join(cfg.detection.gnn_training._edge_losses_dir, split, model_epoch_file) os.makedirs(logs_dir, exist_ok=True) csv_file = os.path.join(logs_dir, time_interval + ".csv") - edge_list.to_csv(csv_file, sep=",", header=True, index=False, encoding="utf-8") + edge_df.to_csv(csv_file, sep=",", header=True, index=False, encoding="utf-8") return all_losses diff --git a/pidsmaker/encoders/linear_encoder.py b/pidsmaker/encoders/linear_encoder.py index 5714993e..2f5dbe10 100644 --- a/pidsmaker/encoders/linear_encoder.py +++ b/pidsmaker/encoders/linear_encoder.py @@ -8,7 +8,8 @@ def __init__(self, in_dim, out_dim, dropout=0.0): self.dropout = nn.Dropout(dropout) def forward(self, x, *args, **kwargs): - if isinstance(x, tuple): + # Handle both tuples and lists (PyG batching may convert tuples to lists) + if isinstance(x, (tuple, list)): h = self.dropout(self.lin1(x[0])), self.dropout(self.lin1(x[1])) else: h = self.dropout(self.lin1(x)) diff --git a/pidsmaker/factory.py b/pidsmaker/factory.py index d1536d23..cc52ec35 100644 --- a/pidsmaker/factory.py +++ b/pidsmaker/factory.py @@ -81,8 +81,6 @@ def encoder_factory(cfg, msg_dim, in_dim, device, max_node_num, graph_reindexer) if use_tgn: in_dim = tgn_memory_dim - original_edge_dim = edge_dim - for method in map( lambda x: x.strip(), cfg.detection.gnn_training.encoder.used_methods.replace("-", ",").split(","), @@ -467,6 +465,7 @@ def objective_factory(cfg, in_dim, graph_reindexer, device, objective_cfg=None): raise ValueError(f"Invalid objective {objective}") # We wrap objectives into this class to calculate some metrics on validation set easily + # This is useful only if use_few_shot is True is_edge_type_prediction = objective_cfg.used_methods.strip() == "predict_edge_type" objectives = [ ValidationWrapper( diff --git a/pidsmaker/main.py b/pidsmaker/main.py index 9fcf694e..1ebcf4e5 100644 --- a/pidsmaker/main.py +++ b/pidsmaker/main.py @@ -20,7 +20,10 @@ gnn_training, graph_preprocessing, ) -from pidsmaker.experiments.tuning import fuse_cfg_with_sweep_cfg, get_tuning_sweep_cfg +from pidsmaker.experiments.tuning import ( + fuse_cfg_with_sweep_cfg, + get_tuning_sweep_cfg, +) from pidsmaker.experiments.uncertainty import ( avg_std_metrics, fuse_hyperparameter_metrics, @@ -159,7 +162,10 @@ def run_pipeline_with_experiments(cfg): hyper_to_metrics = defaultdict(list) for hyper in hyperparameters: - log(f"[@hyperparameter {hyper}] - Started", pre_return_line=True) + log( + f"[@hyperparameter {hyper}] - Started", + pre_return_line=True, + ) for i in range(iterations): log(f"[@iteration {i}]", pre_return_line=True) @@ -184,7 +190,11 @@ def run_pipeline_with_experiments(cfg): for i in range(iterations): log(f"[@iteration {i}]", pre_return_line=True) cfg = update_cfg_for_uncertainty_exp( - method, i, iterations, copy.deepcopy(original_cfg), hyperparameter=None + method, + i, + iterations, + copy.deepcopy(original_cfg), + hyperparameter=None, ) metrics, times = run_pipeline(cfg, method=method, iteration=i) method_to_metrics[method].append({**metrics, **times}) @@ -255,14 +265,9 @@ def run_pipeline_from_sweep(cfg): ) tags = args.tags.split(",") if args.tags != "" else [args.model] - if args.project != "": - project = args.project - else: - project = "PIDSMaker" - wandb.init( - mode="online" if (args.wandb and args.tuning_mode == "none") else "disabled", - project=project, + mode=("online" if (args.wandb and args.tuning_mode == "none") else "disabled"), + project=args.project, name=exp_name, tags=tags, ) @@ -273,7 +278,7 @@ def run_pipeline_from_sweep(cfg): cfg = get_yml_cfg(args) wandb.config.update(clean_cfg_for_log(cfg)) - main(cfg, project=project, exp=exp_name, sweep_id=args.sweep_id) + main(cfg, project=args.project, exp=exp_name, sweep_id=args.sweep_id) wandb.finish() diff --git a/pidsmaker/model.py b/pidsmaker/model.py index 0a09eb5f..9eb7a54a 100644 --- a/pidsmaker/model.py +++ b/pidsmaker/model.py @@ -129,11 +129,14 @@ def gather_h(self, batch, res): h_dst = res.get("h_dst", None) if None in [h_src, h_dst]: - h_src, h_dst = ( - (h[batch.edge_index[0]], h[batch.edge_index[1]]) - if isinstance(h, torch.Tensor) - else h - ) + if isinstance(h, torch.Tensor): + # h is a single tensor with node embeddings - index by edge_index + h_src, h_dst = h[batch.edge_index[0]], h[batch.edge_index[1]] + elif isinstance(h, (tuple, list)): + # h is (h_src_nodes, h_dst_nodes) with separate node embeddings - index each + h_src, h_dst = h[0][batch.edge_index[0]], h[1][batch.edge_index[1]] + else: + h_src, h_dst = h return h, h_src, h_dst diff --git a/pidsmaker/preprocessing/build_graph_methods/build_default_graphs.py b/pidsmaker/preprocessing/build_graph_methods/build_default_graphs.py index 06dae428..784d1c42 100644 --- a/pidsmaker/preprocessing/build_graph_methods/build_default_graphs.py +++ b/pidsmaker/preprocessing/build_graph_methods/build_default_graphs.py @@ -262,7 +262,7 @@ def get_batches(arr, batch_size): start_time = events_list[0][-2] temp_list = [] BATCH = 1024 - window_size_in_sec = cfg.preprocessing.build_graphs.time_window_size * 60_000_000_000 + window_size_in_ns = cfg.preprocessing.build_graphs.time_window_size * 60_000_000_000 last_batch = False for batch_edges in get_batches(events_list, BATCH): @@ -272,7 +272,7 @@ def get_batches(arr, batch_size): if (len(batch_edges) < BATCH) or (temp_list[-1] == events_list[-1]): last_batch = True - if (batch_edges[-1][-2] > start_time + window_size_in_sec) or last_batch: + if (batch_edges[-1][-2] > start_time + window_size_in_ns) or last_batch: time_interval = ( ns_time_to_datetime_US(start_time) + "~" diff --git a/pidsmaker/tgn.py b/pidsmaker/tgn.py index 1436dac8..80bdf811 100644 --- a/pidsmaker/tgn.py +++ b/pidsmaker/tgn.py @@ -392,7 +392,9 @@ def insert(self, src: Tensor, dst: Tensor): # Compute cumulative start indices cum_edge_counts = torch.cat([torch.tensor([0], device=nodes.device), edge_counts.cumsum(0)]) - local_slots = torch.arange(nodes.size(0), device=nodes.device) - cum_edge_counts[self._assoc[nodes]] + local_slots = ( + torch.arange(nodes.size(0), device=nodes.device) - cum_edge_counts[self._assoc[nodes]] + ) dense_id = local_slots + (self._assoc[nodes] * temp_size) # Initialize dense tensors with temporary size diff --git a/pidsmaker/utils/data_utils.py b/pidsmaker/utils/data_utils.py index aefa25a0..16dfa835 100644 --- a/pidsmaker/utils/data_utils.py +++ b/pidsmaker/utils/data_utils.py @@ -522,7 +522,13 @@ def run_reindexing_preprocessing(datasets, graph_reindexer, device, cfg): log_dataset_stats(datasets) # By default we only have x_src and x_dst of shape (E, d), here we create x of shape (N, d) use_tgn = "tgn" in cfg.detection.gnn_training.encoder.used_methods - reindex_graphs(datasets, graph_reindexer, device, use_tgn) + reindex_graphs( + datasets, + graph_reindexer, + device, + use_tgn, + x_is_tuple=cfg.detection.gnn_training.encoder.x_is_tuple, + ) return datasets @@ -704,7 +710,7 @@ def inter_batching(dataset, method): ): batch = data_list[i : i + bs] data = collate(CollatableTemporalData, data_list=batch)[0] - + use_tgn = "tgn" in cfg.detection.gnn_training.encoder.used_methods if cfg._debug and use_tgn: debug_test_batching(batch, data, cfg) @@ -782,7 +788,7 @@ def node_features_reshape(self, edge_index, x_src, x_dst, max_num_node=None, x_i scatter(x_dst, edge_index[1], out=output, dim=0, reduce="mean") x_dst_result = output.clone() - return x_src_result[:max_num_node], x_dst_result[:max_num_node] + return (x_src_result[:max_num_node], x_dst_result[:max_num_node]) else: if self.fix_buggy_graph_reindexer: output = output.clone() @@ -811,15 +817,11 @@ def reindex_graph(self, data, x_is_tuple=False, use_tgn=False): data.edge_index, data.x_src, data.x_dst, x_is_tuple=x_is_tuple ) data.original_n_id = n_id + data.x = x if not use_tgn: data.src, data.dst = edge_index[0], edge_index[1] - if x_is_tuple: - data.x_src, data.x_dst = x - else: - data.x = x - data.node_type, *_ = self._reindex_graph( data.edge_index, data.node_type_src, data.node_type_dst, x_is_tuple=False ) @@ -912,10 +914,10 @@ def load_model(model, path: str, cfg, map_location=None): return model -def reindex_graphs(datasets, graph_reindexer, device, use_tgn): +def reindex_graphs(datasets, graph_reindexer, device, use_tgn, x_is_tuple=False): for dataset in datasets: for data_list in dataset: for batch in log_tqdm(data_list, desc="Reindexing graphs"): batch.to(device) - graph_reindexer.reindex_graph(batch, use_tgn=use_tgn) + graph_reindexer.reindex_graph(batch, use_tgn=use_tgn, x_is_tuple=x_is_tuple) batch.to("cpu") diff --git a/pidsmaker/utils/utils.py b/pidsmaker/utils/utils.py index 731b90ce..529fb546 100644 --- a/pidsmaker/utils/utils.py +++ b/pidsmaker/utils/utils.py @@ -661,7 +661,7 @@ def set_seed(cfg): torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False - + if cfg.detection.gnn_training.deterministic: torch.use_deterministic_algorithms(True, warn_only=True) diff --git a/scripts/Makefile b/scripts/Makefile new file mode 100644 index 00000000..387a3948 --- /dev/null +++ b/scripts/Makefile @@ -0,0 +1,72 @@ +# Makefile for Singularity/Apptainer PostgreSQL management + +.PHONY: up down status load-dumps full-setup logs clean help + +# PostgreSQL management +up: + @./postgres-start.sh + +down: + @./postgres-stop.sh + +status: + @./postgres-status.sh + +logs: + @echo "PostgreSQL logs:" + @tail -f postgres_log/postgresql*.log 2>/dev/null || echo "No logs found" + +clean: down + @echo "Cleaning up PostgreSQL data..." + @rm -rf postgres_data postgres_run postgres_log + @echo "PostgreSQL data cleaned" + +reset: clean up + +app-build: + @echo "Building PIDSMaker container..." + @if command -v apptainer &> /dev/null; then \ + apptainer build pidsmaker.sif pidsmaker.def || echo "Build failed - check if you have fakeroot access"; \ + elif command -v singularity &> /dev/null; then \ + singularity build pidsmaker.sif pidsmaker.def || echo "Build failed - check if you have fakeroot access"; \ + else \ + echo "ERROR: Neither apptainer nor singularity found"; exit 1; \ + fi + +app-run: up + @echo "Running PIDSMaker application..." + @CONTAINER_CMD=$$(command -v apptainer &> /dev/null && echo "apptainer" || echo "singularity"); \ + $$CONTAINER_CMD run --nv \ + --env DB_HOST=localhost \ + --env DOCKER_PORT=5432 \ + --env DB_USER=postgres \ + --env DB_PASSWORD=postgres \ + --bind ${PWD}:/workspace \ + pidsmaker.sif + +load-dumps: up + @echo "Loading database dumps from inside container..." + @CONTAINER_CMD=$$(command -v apptainer &> /dev/null && echo "apptainer" || echo "singularity"); \ + if [ -f "./load_dumps.sh" ]; then \ + echo "Found load_dumps.sh, executing inside container..."; \ + $$CONTAINER_CMD exec instance://postgres_instance /scripts/load_dumps.sh; \ + else \ + echo "Error: ./load_dumps.sh not found"; \ + exit 1; \ + fi + +full-setup: up load-dumps + @echo "PostgreSQL setup complete with dumps loaded" + +help: + @echo "Available commands:" + @echo " up - Start PostgreSQL" + @echo " down - Stop PostgreSQL" + @echo " status - Check PostgreSQL status" + @echo " logs - Show PostgreSQL logs" + @echo " load-dumps - Load database dumps" + @echo " full-setup - Start PostgreSQL and load dumps" + @echo " clean - Stop and remove all data" + @echo " reset - Clean and restart PostgreSQL" + @echo " app-build - Build PIDSMaker container" + @echo " app-run - Run PIDSMaker with PostgreSQL" diff --git a/settings/scripts/create_database.sh b/scripts/create_database.sh similarity index 100% rename from settings/scripts/create_database.sh rename to scripts/create_database.sh diff --git a/settings/scripts/download_cadets_e3.sh b/scripts/download_cadets_e3.sh similarity index 100% rename from settings/scripts/download_cadets_e3.sh rename to scripts/download_cadets_e3.sh diff --git a/settings/scripts/download_cadets_e5.sh b/scripts/download_cadets_e5.sh similarity index 100% rename from settings/scripts/download_cadets_e5.sh rename to scripts/download_cadets_e5.sh diff --git a/settings/scripts/download_clearscope_e3.sh b/scripts/download_clearscope_e3.sh similarity index 100% rename from settings/scripts/download_clearscope_e3.sh rename to scripts/download_clearscope_e3.sh diff --git a/settings/scripts/download_clearscope_e5.sh b/scripts/download_clearscope_e5.sh similarity index 100% rename from settings/scripts/download_clearscope_e5.sh rename to scripts/download_clearscope_e5.sh diff --git a/settings/scripts/download_theia_e3.sh b/scripts/download_theia_e3.sh similarity index 100% rename from settings/scripts/download_theia_e3.sh rename to scripts/download_theia_e3.sh diff --git a/settings/scripts/download_theia_e5.sh b/scripts/download_theia_e5.sh similarity index 100% rename from settings/scripts/download_theia_e5.sh rename to scripts/download_theia_e5.sh diff --git a/settings/scripts/e3_tools.sh b/scripts/e3_tools.sh similarity index 100% rename from settings/scripts/e3_tools.sh rename to scripts/e3_tools.sh diff --git a/scripts/environment.yaml b/scripts/environment.yaml new file mode 100644 index 00000000..83acfd12 --- /dev/null +++ b/scripts/environment.yaml @@ -0,0 +1,46 @@ +name: pids +channels: +- conda-forge +dependencies: +- pip=25.1.1 +- python=3.9.23 +- python_abi=3.9 +- psycopg2 +- tqdm +- pip: + - --extra-index-url https://download.pytorch.org/whl/cu117 + - -f https://data.pyg.org/whl/torch-1.13.0+cu117.html + - torch==1.13.1+cu117 + - torchvision==0.14.1+cu117 + - torchaudio==0.13.1 + - scikit-learn==1.2.0 + - networkx==2.8.7 + - xxhash==3.2.0 + - graphviz==0.20.1 + - psutil + - scipy==1.10.1 + - matplotlib==3.8.4 + - wandb==0.16.6 + - chardet==5.2.0 + - nltk==3.8.1 + - igraph==0.11.5 + - cairocffi==1.7.0 + - wget==3.2 + - torch_geometric==2.5.3 + - pyg_lib==0.2.0 + - torch_scatter==2.1.1 + - torch_sparse==0.6.17 + - torch_cluster==1.6.1 + - torch_spline_conv==1.2.2 + - gensim==4.3.1 + - pytz==2024.1 + - pandas==2.2.2 + - yacs==0.1.8 + - numpy==1.26.4 + - gdown==5.2.0 + - pytest==8.3.5 + - pytest-cov==6.1.1 + - pre-commit==4.2.0 + - setuptools==61.0 + - mkdocs-material==9.6.12 + - mkdocs-glightbox==0.4.0 diff --git a/scripts/load_dumps.sh b/scripts/load_dumps.sh new file mode 100755 index 00000000..da642ee7 --- /dev/null +++ b/scripts/load_dumps.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -e + +echo "Starting database dump restoration..." + +for dump_file in /data/*.dump; do + if [ ! -f "$dump_file" ]; then + echo "No .dump files found in /data/ directory" + break + fi + + db_name=$(basename "$dump_file" .dump) + + echo "Processing $dump_file -> database '$db_name'" + + if psql -U postgres -h localhost -p 5432 -lqt | cut -d \| -f 1 | grep -qw "$db_name"; then + echo "Database '$db_name' already exists. Checking if it has data..." + + table_count=$(psql -U postgres -h localhost -p 5432 -d "$db_name" -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null || echo "0") + + if [ "$table_count" -gt 0 ]; then + echo "Database '$db_name' already has $table_count tables. Skipping restoration." + continue + else + echo "Database '$db_name' exists but is empty. Proceeding with restoration..." + fi + else + echo "Creating database '$db_name'..." + psql -U postgres -h localhost -p 5432 -c "CREATE DATABASE \"$db_name\";" 2>/dev/null || { + echo "Warning: Could not create database '$db_name' (may already exist)" + } + fi + + echo "Restoring $dump_file into database '$db_name'..." + + # Use --clean --if-exists to handle existing objects gracefully + if pg_restore -U postgres -h localhost -p 5432 --clean --if-exists --no-owner --no-privileges -d "$db_name" "$dump_file" 2>/dev/null; then + echo "Successfully restored $dump_file" + + final_table_count=$(psql -U postgres -h localhost -p 5432 -d "$db_name" -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null || echo "0") + echo " Database '$db_name' now has $final_table_count tables" + else + echo "Warning: pg_restore reported errors for $dump_file (this may be normal for some dump formats)" + fi + + echo "" +done + +echo "Database dump restoration completed" + +echo "Summary of available databases:" +psql -U postgres -h localhost -p 5432 -c "\l" | grep -E "^\s+[a-zA-Z]" | head -20 diff --git a/scripts/postgres-start.sh b/scripts/postgres-start.sh new file mode 100755 index 00000000..03d75cbf --- /dev/null +++ b/scripts/postgres-start.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +# PostgreSQL startup script for Singularity/Apptainer + +set -e + +# Detect which container runtime is available +if command -v apptainer &> /dev/null; then + CONTAINER_CMD="apptainer" + export APPTAINER_TMPDIR="${TMPDIR:-/tmp}/apptainer-${USER}" + export APPTAINER_CACHEDIR="${HOME}/.apptainer/cache" + export APPTAINER_SESSIONDIR="${TMPDIR:-/tmp}/apptainer-sessions-${USER}" + mkdir -p "$APPTAINER_TMPDIR" "$APPTAINER_CACHEDIR" "$APPTAINER_SESSIONDIR" +elif command -v singularity &> /dev/null; then + CONTAINER_CMD="singularity" + export SINGULARITY_TMPDIR="${TMPDIR:-/tmp}/singularity-${USER}" + export SINGULARITY_CACHEDIR="${HOME}/.singularity/cache" + mkdir -p "$SINGULARITY_TMPDIR" "$SINGULARITY_CACHEDIR" +else + echo "ERROR: Neither apptainer nor singularity found in PATH" + exit 1 +fi + +# Configuration +POSTGRES_IMAGE="postgres.sif" +POSTGRES_INSTANCE="postgres_instance" +DATA_DIR="postgres_data" +RUN_DIR="postgres_run" +LOG_DIR="postgres_log" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${YELLOW}Starting PostgreSQL with ${CONTAINER_CMD}...${NC}" + +# Check if postgres.sif exists +if [ ! -f "$POSTGRES_IMAGE" ]; then + echo -e "${YELLOW}PostgreSQL image not found. Pulling from Docker Hub...${NC}" + $CONTAINER_CMD pull $POSTGRES_IMAGE docker://postgres:17 +fi + +# Create necessary directories +echo -e "${YELLOW}Creating directories...${NC}" +mkdir -p $DATA_DIR $RUN_DIR $LOG_DIR + +# Create INPUT_DIR if it doesn't exist +INPUT_DIR=${INPUT_DIR:-$(pwd)/data} +if [ ! -d "$INPUT_DIR" ]; then + echo -e "${YELLOW}Creating INPUT_DIR: $INPUT_DIR${NC}" + mkdir -p "$INPUT_DIR" +fi + +# Check if instance already exists +if $CONTAINER_CMD instance list | grep -q "$POSTGRES_INSTANCE"; then + echo -e "${YELLOW}PostgreSQL instance $POSTGRES_INSTANCE already exists${NC}" + # Check if it's responsive + if $CONTAINER_CMD exec instance://$POSTGRES_INSTANCE pg_isready -h localhost -U postgres > /dev/null 2>&1; then + echo -e "${GREEN}PostgreSQL instance is already running and responsive${NC}" + exit 0 + else + echo -e "${YELLOW}Instance exists but not responsive, stopping it...${NC}" + $CONTAINER_CMD instance stop $POSTGRES_INSTANCE + sleep 2 + fi +fi + +# Check if any other postgres processes are running +if pgrep -f "${CONTAINER_CMD}.*postgres" > /dev/null; then + echo -e "${YELLOW}Other PostgreSQL processes detected, cleaning up...${NC}" + pkill -f "${CONTAINER_CMD}.*postgres" || true + sleep 2 +fi + +# Set environment variables (works for both singularity and apptainer) +if [ "$CONTAINER_CMD" = "apptainer" ]; then + export APPTAINERENV_POSTGRES_PASSWORD=postgres + export APPTAINERENV_POSTGRES_USER=postgres + export APPTAINERENV_POSTGRES_DB=postgres +else + export SINGULARITYENV_POSTGRES_PASSWORD=postgres + export SINGULARITYENV_POSTGRES_USER=postgres + export SINGULARITYENV_POSTGRES_DB=postgres +fi + +# Prepare bind mounts - only bind if files/directories exist +BIND_MOUNTS="--bind $DATA_DIR:/var/lib/postgresql/data" +BIND_MOUNTS="$BIND_MOUNTS --bind $RUN_DIR:/var/run/postgresql" +BIND_MOUNTS="$BIND_MOUNTS --bind $LOG_DIR:/var/log" +BIND_MOUNTS="$BIND_MOUNTS --bind ./:/scripts" + +# Always bind INPUT_DIR +BIND_MOUNTS="$BIND_MOUNTS --bind $INPUT_DIR:/data" + +if [ -f "./postgres_config/postgresql.conf" ]; then + BIND_MOUNTS="$BIND_MOUNTS --bind ./postgres_config/postgresql.conf:/etc/postgresql/postgresql.conf" +fi + +# Start PostgreSQL instance +echo -e "${YELLOW}Starting PostgreSQL instance...${NC}" +echo -e "${YELLOW}Using INPUT_DIR: $INPUT_DIR${NC}" + +$CONTAINER_CMD instance start $BIND_MOUNTS $POSTGRES_IMAGE $POSTGRES_INSTANCE + +# Start PostgreSQL inside the instance +echo -e "${YELLOW}Starting PostgreSQL server inside instance...${NC}" +$CONTAINER_CMD exec instance://$POSTGRES_INSTANCE bash -c "docker-entrypoint.sh postgres &" + +# Get the PID of the instance (optional, for compatibility) +INSTANCE_PID=$(pgrep -f "${CONTAINER_CMD}.*$POSTGRES_INSTANCE" | head -1) +if [ -n "$INSTANCE_PID" ]; then + echo $INSTANCE_PID > postgres.pid +fi + +# Wait for PostgreSQL to be ready +echo -e "${YELLOW}Waiting for PostgreSQL to start...${NC}" +for i in {1..30}; do + if $CONTAINER_CMD exec instance://$POSTGRES_INSTANCE pg_isready -h localhost -U postgres > /dev/null 2>&1; then + echo -e "${GREEN}PostgreSQL is ready!${NC}" + echo -e "${GREEN}Connection: $CONTAINER_CMD exec instance://$POSTGRES_INSTANCE psql -h localhost -U postgres${NC}" + echo -e "${GREEN}Instance: $POSTGRES_INSTANCE${NC}" + exit 0 + fi + echo -n "." + sleep 2 +done + +echo -e "${RED}PostgreSQL failed to start within 60 seconds${NC}" +$CONTAINER_CMD instance stop $POSTGRES_INSTANCE 2>/dev/null || true +exit 1 diff --git a/scripts/postgres-status.sh b/scripts/postgres-status.sh new file mode 100755 index 00000000..8be6b04e --- /dev/null +++ b/scripts/postgres-status.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# PostgreSQL status script for Singularity/Apptainer + +# Detect which container runtime is available +if command -v apptainer &> /dev/null; then + CONTAINER_CMD="apptainer" +elif command -v singularity &> /dev/null; then + CONTAINER_CMD="singularity" +else + echo "ERROR: Neither apptainer nor singularity found in PATH" + exit 1 +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${YELLOW}PostgreSQL Status:${NC}" + +# Check multiple ways to detect if PostgreSQL is running +POSTGRES_RUNNING=false + +# Method 1: Check PID file +if [ -f postgres.pid ]; then + PID=$(cat postgres.pid) + if kill -0 $PID 2>/dev/null; then + echo -e "${GREEN}✓ PostgreSQL process is running (PID: $PID)${NC}" + POSTGRES_RUNNING=true + else + echo -e "${YELLOW}! PID file exists but process is not running${NC}" + rm postgres.pid + fi +fi + +# Method 2: Check if we can connect (most reliable test) +if [ ! -f postgres.sif ]; then + echo -e "${RED}✗ postgres.sif not found${NC}" + exit 1 +fi + +if $CONTAINER_CMD exec postgres.sif pg_isready -h localhost -U postgres > /dev/null 2>&1; then + echo -e "${GREEN}✓ PostgreSQL is accepting connections${NC}" + echo -e "${GREEN} Connection: ${CONTAINER_CMD} exec postgres.sif psql -h localhost -U postgres${NC}" + POSTGRES_RUNNING=true + + # Show database list + echo -e "${YELLOW}Databases:${NC}" + $CONTAINER_CMD exec postgres.sif psql -h localhost -U postgres -c "\l" 2>/dev/null | \ + grep -v template | grep -v "^-" | grep -v "^(" | grep -v "Name.*Owner" | \ + grep -v "^\s*$" | head -10 + + # Show PostgreSQL version + echo -e "${YELLOW}Version:${NC}" + $CONTAINER_CMD exec postgres.sif psql -h localhost -U postgres -c "SELECT version();" -t 2>/dev/null | head -1 + +else + echo -e "${RED}✗ PostgreSQL is not accepting connections${NC}" +fi + +# Method 3: Check process list as fallback +if [ "$POSTGRES_RUNNING" = false ]; then + # Check for any postgres-related processes with more flexible patterns + if pgrep -f "postgres" > /dev/null || pgrep -f "${CONTAINER_CMD}.*postgres" > /dev/null; then + echo -e "${YELLOW}! Found postgres-related process but cannot connect${NC}" + echo -e "${YELLOW} Process list:${NC}" + ps aux | grep -E "(postgres|${CONTAINER_CMD})" | grep -v grep | head -5 + else + echo -e "${RED}✗ No PostgreSQL processes found${NC}" + fi +fi + +# Method 4: Check if port 5432 is listening +if ss -tlnp 2>/dev/null | grep -q ":5432 " || netstat -tlnp 2>/dev/null | grep -q ":5432 "; then + echo -e "${GREEN}✓ Port 5432 is listening${NC}" + POSTGRES_RUNNING=true +else + echo -e "${RED}✗ Port 5432 is not listening${NC}" +fi + +# Final status +if [ "$POSTGRES_RUNNING" = true ]; then + echo -e "${GREEN}Overall Status: PostgreSQL is running and accessible${NC}" + exit 0 +else + echo -e "${RED}Overall Status: PostgreSQL is not running or not accessible${NC}" + exit 1 +fi \ No newline at end of file diff --git a/scripts/postgres-stop.sh b/scripts/postgres-stop.sh new file mode 100755 index 00000000..6455df4e --- /dev/null +++ b/scripts/postgres-stop.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# PostgreSQL shutdown script for Singularity/Apptainer + +# Detect which container runtime is available +if command -v apptainer &> /dev/null; then + CONTAINER_CMD="apptainer" +elif command -v singularity &> /dev/null; then + CONTAINER_CMD="singularity" +else + echo "ERROR: Neither apptainer nor singularity found in PATH" + exit 1 +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${YELLOW}Stopping PostgreSQL...${NC}" + +STOPPED=false + +# Method 1: Stop instance if it exists +if $CONTAINER_CMD instance list | grep -q "postgres_instance"; then + echo -e "${YELLOW}Stopping ${CONTAINER_CMD} instance: postgres_instance${NC}" + $CONTAINER_CMD instance stop postgres_instance + STOPPED=true +fi + +# Method 2: Stop using PID file if it exists +if [ -f postgres.pid ]; then + PID=$(cat postgres.pid) + if kill -0 $PID 2>/dev/null; then + echo -e "${YELLOW}Stopping PostgreSQL process (PID: $PID)${NC}" + kill $PID + # Wait for graceful shutdown + for i in {1..10}; do + if ! kill -0 $PID 2>/dev/null; then + echo -e "${GREEN}PostgreSQL stopped gracefully${NC}" + STOPPED=true + break + fi + sleep 1 + done + # Force kill if still running + if kill -0 $PID 2>/dev/null; then + echo -e "${YELLOW}Force killing PostgreSQL${NC}" + kill -9 $PID + STOPPED=true + fi + fi + rm postgres.pid +fi + +# Method 3: Fallback - kill any container postgres processes +if pkill -f "${CONTAINER_CMD}.*postgres"; then + echo -e "${YELLOW}Killed remaining PostgreSQL processes${NC}" + STOPPED=true +fi + +if [ "$STOPPED" = true ]; then + echo -e "${GREEN}PostgreSQL stopped${NC}" +else + echo -e "${YELLOW}No PostgreSQL instances were found running${NC}" +fi \ No newline at end of file diff --git a/settings/scripts/uncompress_darpa_files.sh b/scripts/uncompress_darpa_files.sh similarity index 100% rename from settings/scripts/uncompress_darpa_files.sh rename to scripts/uncompress_darpa_files.sh diff --git a/settings/scripts/load_dumps.sh b/settings/scripts/load_dumps.sh deleted file mode 100755 index 7c56e036..00000000 --- a/settings/scripts/load_dumps.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -for dump_file in /data/*.dump; do - db_name=$(basename "$dump_file" .dump) - - echo "Restoring $dump_file into database '$db_name'..." - - pg_restore -U postgres -h localhost -p 5432 -d "$db_name" "$dump_file" -done diff --git a/tests/test_framework.py b/tests/test_framework.py index 9a9e8a78..e81645a1 100644 --- a/tests/test_framework.py +++ b/tests/test_framework.py @@ -7,13 +7,12 @@ from pidsmaker import main from pidsmaker.config import ( - DEFAULT_ROOT_ARTIFACT_DIR, ENCODERS_CFG, get_runtime_required_args, get_yml_cfg, ) -TESTS_ARTIFACT_DIR = os.path.join(DEFAULT_ROOT_ARTIFACT_DIR, "tests/") +TESTS_ARTIFACT_DIR = os.path.join("/home/artifacts/", "tests/") def prepare_cfg(