feat: Add Singularity support and improve code organization

.gitignore

@@ -169,3 +169,14 @@ nohup.out
 artifacts/
 data/
 docs/site/
+postgres_data/
+postgres_lock/
+
+# Singularity files
+*.sif
+*.def
+
+# Postgres directories
+postgres_config/
+postgres_run/
+postgres_log/

compose-postgres.yml

@@ -12,7 +12,7 @@ services:
     volumes:
       - postgres_data:/var/lib/postgresql/data
       - ./postgres/init-create-empty-databases.sh:/docker-entrypoint-initdb.d/init-create-empty-databases.sh
-      - ./settings/scripts:/scripts
+      - ./scripts:/scripts
       - ${INPUT_DIR:-/data}:/data
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]

config/flash.yml

@@ -20,7 +20,6 @@ featurization:
     epochs: 10 # 300
     training_split: train
     used_method: flash
-    use_seed: True
     flash:
       min_count: 1
       workers: 15
@@ -41,7 +40,6 @@ detection:
       used_method: none
   gnn_training:
     used_method: default
-    use_seed: True
     deterministic: False
     num_epochs: 12
     patience: 3

config/kairos.yml

@@ -45,7 +45,6 @@ detection:
       used_method: none
   gnn_training:
     used_method: default
-    use_seed: True
     deterministic: False
     num_epochs: 12
     patience: 3

config/magic.yml

@@ -34,7 +34,6 @@ detection:
       used_method: none
   gnn_training:
     used_method: default
-    use_seed: True
     deterministic: False
     num_epochs: 12
     patience: 3

config/nodlink.yml

@@ -18,7 +18,6 @@ featurization:
   feat_training:
     epochs: 100
     emb_dim: 256
-    use_seed: True
     training_split: train
     used_method: fasttext
     fasttext:
@@ -45,7 +44,6 @@ detection:
       used_method: none
   gnn_training:
     used_method: default
-    use_seed: True
     deterministic: False
     num_epochs: 12
     patience: 3

config/orthrus.yml

@@ -21,7 +21,6 @@ featurization:
     emb_dim: 128
     epochs: 50
     training_split: all
-    use_seed: True
     used_method: word2vec
     word2vec:
       alpha: 0.025
@@ -58,7 +57,6 @@ detection:
       used_method: none
   gnn_training:
     used_method: default
-    use_seed: True
     deterministic: False
     num_epochs: 12
     patience: 3

config/rcaid.yml

@@ -20,7 +20,6 @@ featurization:
   feat_training:
     epochs: 5
     emb_dim: 128
-    use_seed: True
     training_split: all
     used_method: doc2vec
     doc2vec:
@@ -43,7 +42,6 @@ detection:
       used_method: none
   gnn_training:
     used_method: default
-    use_seed: True
     deterministic: False
     num_epochs: 12
     patience: 3

config/threatrace.yml

@@ -34,7 +34,6 @@ detection:
       used_method: none
   gnn_training:
     used_method: default
-    use_seed: True
     deterministic: False
     num_epochs: 12
     patience: 3

config/tuned_components/tuned_alacarte.yml

@@ -2,7 +2,6 @@ featurization:
   feat_training:
     epochs: 10
     emb_dim: 128
-    use_seed: True
     training_split: train
     used_method: alacarte
     alacarte:

config/tuned_components/tuned_doc2vec.yml

@@ -2,7 +2,6 @@ featurization:
   feat_training:
     epochs: 10
     emb_dim: 128
-    use_seed: True
     training_split: train
     used_method: doc2vec
     doc2vec:

config/tuned_components/tuned_fasttext.yml

@@ -2,7 +2,6 @@ featurization:
   feat_training:
     epochs: 10
     emb_dim: 256
-    use_seed: True
     training_split: train
     used_method: fasttext
     fasttext:

config/tuned_components/tuned_flash.yml

@@ -4,7 +4,6 @@ featurization:
     epochs: 10
     training_split: train
     used_method: flash
-    use_seed: True
     flash:
       min_count: 1
       workers: 10

config/tuned_components/tuned_word2vec.yml

@@ -3,7 +3,6 @@ featurization:
     epochs: 10
     emb_dim: 128
     training_split: train
-    use_seed: True
     used_method: word2vec
     word2vec:
       alpha: 0.025

docs/docs/create-db-from-scratch.md

@@ -7,7 +7,7 @@ You can download all required files directly by running:
 pip install gdown
 ```
 ```shell
-./settings/scripts/download_{dataset}.sh {data_folder}
+./scripts/download_{dataset}.sh {data_folder}
 ```
 where `{dataset}` can be either `clearscope_e3`, `cadets_e3`, `theia_e3`, `clearscope_e5`, `cadets_e5` or `theia_e5` and `{data_folder}` is the absolute path to the output folder where all raw files will be downloaded.
 
@@ -26,14 +26,18 @@ sudo docker compose exec pids bash
 
 4. Convert the DARPA files 
 ```shell
-./settings/scripts/uncompress_darpa_files.sh /data/
+./scripts/uncompress_darpa_files.sh /data/
 ```
 
 > [!NOTE]  
 > This may take multiple hours depending on the dataset.
 
 ### Optional configurations
-- optionally, if using a specific postgres database instead of the postgres docker, update the connection config by setting `DATABASE_DEFAULT_CONFIG` within `pidsmaker/config.py`.
+- optionally, if using a specific postgres database instead of the postgres docker, pass the details as command line arguments to the python scripts
+  - `--database_host`: the host machine where the database is located (default: `postgres`)
+  - `--database_user`: the database user to connect to the database (default: `postgres`)
+  - `--database_password`: the password for the database user (default: `postgres`)
+  - `--database_port`: the port number for Postgres (default: `5432`)
 
 - optionaly, if you want to change the output folder where generated files are stored, update accordingly the volume by uncommenting `./artifacts:/home/artifacts` in `compose.yml`.
 

docs/docs/singularity_install.md

@@ -0,0 +1,39 @@
+# Install Framework using Singularity
+
+For quick installation on environments where Docker is not available (such as HPC clusters), you can use Singularity. This guide assumes Singularity is already installed on your system.
+
+## Setup Process
+
+### 1. Database Setup
+The Makefile in `./scripts/Makefile` provides easy environment setup:
+
+```bash
+make full-setup
+```
+
+This command will:
+- Download and run a PostgreSQL container through Singularity
+- Load database dumps by executing the `load_dumps.sh` script
+
+### 2. Container Management
+Once the database is ready:
+- Stop the container: `make down`
+- Start it again: `make up`
+
+### 3. Dependencies Installation
+Install all required dependencies using conda:
+
+```bash
+conda env create -f ./scripts/environment.yml
+conda activate pids
+```
+
+## Running the Framework
+
+Once both the database and conda environment are ready, run the framework with:
+
+```bash
+python pidsmaker/main.py SYSTEM DATASET
+```
+
+For more details, see the [introduction](introduction.md).

docs/scripts/args/args_detection.md

@@ -45,7 +45,6 @@
     </li>
     <li class='bullet'><span class="key">gnn_training</span>
     <ul>
-        <li class='no-bullet'><span class="key-leaf">use_seed</span>: <span class="value">bool</span></li>
         <li class='no-bullet'><span class="key-leaf">deterministic</span>: <span class="value">bool (19)</span></li>
         <li class='no-bullet'><span class="key-leaf">num_epochs</span>: <span class="value">int</span></li>
         <li class='no-bullet'><span class="key-leaf">patience</span>: <span class="value">int</span></li>

docs/scripts/args/args_featurization.md

@@ -5,7 +5,6 @@
     <ul>
         <li class='no-bullet'><span class="key-leaf">emb_dim</span>: <span class="value">int (1)</span></li>
         <li class='no-bullet'><span class="key-leaf">epochs</span>: <span class="value">int (2)</span></li>
-        <li class='no-bullet'><span class="key-leaf">use_seed</span>: <span class="value">bool</span></li>
         <li class='no-bullet'><span class="key-leaf">training_split</span>: <span class="value">str (3)</span></li>
         <li class='no-bullet'><span class="key-leaf">multi_dataset_training</span>: <span class="value">bool (4)</span></li>
         <li class='no-bullet'><span class="key-leaf">used_method</span>: <span class="value">str (5)</span></li>