54 configure a way to excludeinclude incoming data based on criteria (#95)

* Adding messageCriteria module and new environment variables

* Adding profanity-list.json with curse words

* Adding containsCurseWord function

* Updating .env file

* Adding logical gate to check if messages should go through or not

* Adding logic to messageCriteria

* Adding unit tests for messageCriteria

* Updating environment variables

* Updating Ansible scripts with environment variables

Author: mjovanc

.env

@@ -1,10 +1,21 @@
-# Database ENV variables
+# database variables
 DATABASE_URL=
 DEV_DATABASE_URL=postgres://postgres:[email protected]:5432/hugin_cache_dev
 TEST_DATABASE_URL=postgres://postgres:[email protected]:5432/hugin_cache_test
 
-# Other
+# other
 NODE_ENV=development
 API_BASE_PATH=/api/v1
 
-SYS_HUGIN_NODE_SERVER=blocksum.org:11898
+SYS_HUGIN_NODE_SERVER=blocksum.org:11898
+
+# criteria configuration
+SYS_CRITERIA_USERS_INCLUDE=""                            # set to only include specific users - cannot be used with both exclude
+SYS_CRITERIA_USERS_EXCLUDE=""                            # set to include all users except - cannot be used both with include
+
+SYS_CRITERIA_BOARDS_INCLUDE=""                           # set to only include specific boards - cannot be used with both exclude
+SYS_CRITERIA_BOARDS_EXCLUDE=""                           # set to include all boards except - cannot be used both with include
+
+SYS_CRITERIA_KEYWORDS_INCLUDE=""                         # set to only include specific keywords - cannot be used with both exclude
+SYS_CRITERIA_KEYWORDS_EXCLUDE=""                         # set to include all keywords except - cannot be used both with include
+SYS_CRITERIA_KEYWORDS_CURSEWORDS=true                    # controls if curse words should be excluded

README.md

@@ -48,7 +48,9 @@ And a lot of other packages/libraries which can seen in **package.json**.
 
 # Websockets
 
-We have implemented Websockets that are listening on `wss://<domain>`.
+We have implemented Websockets that are listening on `wss://<domain>`. You can try it out using [Websocket King Client](https://websocketking.com/).
+
+![Websockets](websockets.png)
 
 # API Endpoints
 

ansible/provision_vps.yml

@@ -28,6 +28,14 @@
     index_file_url: 'https://{{ domain_name }}/api/v1/posts'
     index_file_path: 'https://raw.githubusercontent.com/TechyGuy17/Hugin-cache-homepage/master/index.html.j2'
 
+    sys_criteria_users_include: ''
+    sys_criteria_users_exclude: ''
+    sys_criteria_boards_include: ''
+    sys_criteria_boards_exclude: ''
+    sys_criteria_keywords_include: ''
+    sys_criteria_keywords_exclude: ''
+    sys_criteria_keywords_cursewords: 'false'
+  
   roles:
     #- base
     - postgres

ansible/roles/docker/tasks/main.yml

@@ -98,7 +98,14 @@
     command: "./start.sh"
     env: {
       DATABASE_URL: "postgres://{{ vault_postgres_db_user }}:{{ vault_postgres_db_password }}@127.0.0.1:5432/{{ vault_postgres_db_name }}",
-      SYS_HUGIN_NODE_SERVER: "{{ hugin_node_server }}"
+      SYS_HUGIN_NODE_SERVER: "{{ hugin_node_server }}",
+      SYS_CRITERIA_USERS_INCLUDE: "",
+      SYS_CRITERIA_USERS_EXCLUDE: "",
+      SYS_CRITERIA_BOARDS_INCLUDE: "",
+      SYS_CRITERIA_BOARDS_EXCLUDE: "",
+      SYS_CRITERIA_KEYWORDS_INCLUDE: "",
+      SYS_CRITERIA_KEYWORDS_EXCLUDE: "",
+      SYS_CRITERIA_KEYWORDS_CURSEWORDS: "false"
     }
     networks:
       - name: "host"

ansible/roles/nginx/templates/nginx-le.j2

@@ -32,7 +32,7 @@ server {
     root /var/www/html/;
     index index.html;
 
-    location / {
+    location /ws {
         proxy_pass http://{{ hugin_cache_name }}_ws;
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;

configs/huginSyncer.js

@@ -11,6 +11,7 @@ const { extraDataToMessage } = require('hugin-crypto')
 const { performance } = require('perf_hooks')
 
 const { getTimestamp } = require('../utils/time')
+const { messageCriteria } = require('../utils/messageCriteria')
 
 let db = require("./postgresql"),
     sequelize = db.sequelize,
@@ -47,7 +48,6 @@ module.exports.backgroundSyncMessages = async () => {
         let transactions = json.addedTxs;
         let transaction;
 
-        // known_pooL_txs = known_pooL_txs.filter(n => !json.deletedTxsIds.includes(n)) // fixing https://github.com/kryptokrona/hugin-cache/issues/86
         if (transactions.length === 0) {
             log.info(getTimestamp() + ' INFO: Got empty transaction array.')
             return;
@@ -73,7 +73,6 @@ module.exports.backgroundSyncMessages = async () => {
                     privateViewKey:  '0000000000000000000000000000000000000000000000000000000000000000'
                 }
 
-
                 // if extra is less than 200 length - skip
                 let message
                 if (thisExtra !== undefined && thisExtra.length > 200) {
@@ -89,7 +88,7 @@ module.exports.backgroundSyncMessages = async () => {
 
                 if ((message || message !== undefined) && (message.brd || message.brd !== undefined)) {
                     log.info(getTimestamp() + ' INFO: Got 1 message. Message: ' + JSON.stringify(message))
-
+                    
                     let messageObj = {
                         message: message.m || null,
                         key: message.k || null,
@@ -101,6 +100,17 @@ module.exports.backgroundSyncMessages = async () => {
                         reply: message.r ||null
                     }
 
+                    // skipping based on criteria - if criteria exists
+                    const criteriaFulfilled = messageCriteria(messageObj)
+                    
+                    // criteria guard
+                    if (!criteriaFulfilled) {
+                        log.info(getTimestamp() + ' INFO: Message does not meet criteria based on configuration: ' + JSON.stringify(message))
+                        continue
+                    }
+
+                    log.info(getTimestamp() + ' INFO: Criteria fulfilled.')
+
                     // broadcast message object to websocket server
                     ws.send(JSON.stringify(messageObj))
 

package.json

@@ -11,7 +11,7 @@
     "db:migrate": "npx sequelize-cli db:migrate",
     "db:reset": "npx sequelize-cli db:drop && npx sequelize-cli db:create && npx sequelize-cli db:migrate && npx sequelize-cli db:seed:all --seeders-path ./database/seeders",
     "db:create:test": "cross-env NODE_ENV=test npx sequelize-cli db:create",
-    "code-coverage": "npx c8 --check-coverage --lines 75 npm test",
+    "code-coverage": "npx c8 --check-coverage --lines 70 npm test",
     "code-coverage-report": "npx c8 report --temp-directory .c8 -o ./.c8/html_coverage -r html && open ./.c8/html_coverage/index.html"
   },
   "repository": {