From 5e7b258ed3ed59152a543dda2184b1bab866d9e7 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 3 Mar 2025 15:44:42 +0100 Subject: [PATCH 1/3] Implementing IEP008 - contituency field The change has been proposed and discussed over a year ago --- CHANGELOG.md | 5 +++++ intelmq/etc/harmonization.conf | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fefa2f65fa..c87338997e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,11 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ### Data Format +- Implementing [IEP008](https://github.com/certtools/ieps/tree/main/008) introducing the `constituency` field for easier identification in + multi-constituency setups. (PR# by Kamil Mańkowski) + To use in current PostgreSQL installations, a schema update may be + necessary: `ALTER TABLE events ADD "constituency" text;`. + ### Bots #### Collectors diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index 027643ac9c..d0c2dc29d0 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -362,6 +362,10 @@ "tlp": { "description": "Traffic Light Protocol level of the event.", "type": "TLP" + }, + "constituency": { + "description": "Internal identifier for multi-constituency setup", + "type": "String" } }, "report": { From dca0f558935af9d5f6782e39b8ce9ac6fb5cfc11 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 3 Mar 2025 17:21:20 +0100 Subject: [PATCH 2/3] Fix order --- intelmq/etc/harmonization.conf | 8 ++++---- intelmq/tests/bin/initdb.sql | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index d0c2dc29d0..7d3c882190 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -17,6 +17,10 @@ "description": "Free text commentary about the abuse event inserted by an analyst.", "type": "String" }, + "constituency": { + "description": "Internal identifier for multi-constituency setup", + "type": "String" + }, "destination.abuse_contact": { "description": "Abuse contact for destination address. A comma separated list.", "type": "LowercaseString" @@ -362,10 +366,6 @@ "tlp": { "description": "Traffic Light Protocol level of the event.", "type": "TLP" - }, - "constituency": { - "description": "Internal identifier for multi-constituency setup", - "type": "String" } }, "report": { diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index 5a5f839f58..b7f3ffbf23 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -4,6 +4,7 @@ CREATE TABLE events ( "classification.taxonomy" varchar(100), "classification.type" text, "comment" text, + "constituency" text, "destination.abuse_contact" text, "destination.account" text, "destination.allocated" timestamp with time zone, @@ -93,4 +94,4 @@ CREATE INDEX "idx_events_source.asn" ON events USING btree ("source.asn"); CREATE INDEX "idx_events_source.ip" ON events USING btree ("source.ip"); CREATE INDEX "idx_events_source.fqdn" ON events USING btree ("source.fqdn"); CREATE INDEX "idx_events_time.observation" ON events USING btree ("time.observation"); -CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); +CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); \ No newline at end of file From d62540ba3c3a06e89f9ddb662047a450847c93b5 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Wed, 9 Apr 2025 16:42:11 +0200 Subject: [PATCH 3/3] Tests, upgrade function, news --- NEWS.md | 5 +++++ intelmq/lib/upgrades.py | 28 ++++++++++++++++++++++++++-- intelmq/tests/bin/initdb.sql | 2 +- intelmq/tests/lib/test_upgrades.py | 6 ++++++ 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 12c225784f..442e228d5a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,6 +18,11 @@ Please refer to the change log for a full list of changes. ### Tools ### Data Format +To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema +update is necessary: +```sql +ALTER TABLE events ADD "constituency" text; +``` ### Configuration diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index ee22b60a69..057b71c513 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -41,7 +41,8 @@ 'v320_update_turris_greylist_url', 'v322_url_replacement', 'v322_removed_feeds_and_bots', - 'v340_deprecations' + 'v340_deprecations', + 'v341_new_fields' ] @@ -974,6 +975,29 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization +def v341_new_fields(configuration, harmonization, dry_run, **kwargs): + """ + Add new fields to IntelMQ Data Format + """ + changed = None + if "event" not in harmonization: + return changed, configuration, harmonization + + builtin_harmonisation = load_configuration( + resource_filename("intelmq", "etc/harmonization.conf") + ) + for field in [ + "constituency", + ]: + if field not in harmonization["event"]: + if field not in builtin_harmonisation["event"]: + # ensure forward-compatibility if we ever remove something from harmonisation + continue + harmonization["event"][field] = builtin_harmonisation["event"][field] + changed = True + return changed, configuration, harmonization + + UPGRADES = OrderedDict([ ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)), ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)), @@ -1004,7 +1028,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): ((3, 3, 0), ()), ((3, 3, 1), ()), ((3, 4, 0), (v340_deprecations, )), - ((3, 4, 1), ()), + ((3, 4, 1), (v341_new_fields, )), ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index b7f3ffbf23..c5c9b77650 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -94,4 +94,4 @@ CREATE INDEX "idx_events_source.asn" ON events USING btree ("source.asn"); CREATE INDEX "idx_events_source.ip" ON events USING btree ("source.ip"); CREATE INDEX "idx_events_source.fqdn" ON events USING btree ("source.fqdn"); CREATE INDEX "idx_events_time.observation" ON events USING btree ("time.observation"); -CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); \ No newline at end of file +CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index a30800b9cb..96ec2e02f7 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -856,6 +856,12 @@ def test_v340_twitter_collector(self): self.assertIn('twitter-collector', result[0]) self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1]) + def test_v341_new_fields(self): + """ Test adding new harmonisation fields """ + result = upgrades.v341_new_fields({}, {"event": {"old-field": "must stay"}}, False) + self.assertTrue(result[0]) + self.assertIn("old-field", result[2]["event"]) + self.assertIn("constituency", result[2]["event"]) for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name,