chore: release v0.0.14

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,90 @@
+name: Bug Report
+description: Report a bug to help us improve
+title: "[Bug]: "
+labels: [bug]
+assignees: []
+
+body:
+  - type: dropdown
+    id: environment
+    attributes:
+      label: Where are you using the app?
+      options:
+        - Cloud (Hosted by Us)
+        - Self-Hosted (OSS) with Docker
+        - Self-Hosted (OSS) without Docker
+    validations:
+      required: true
+
+  - type: input
+    id: app_version
+    attributes:
+      label: App Version
+      description: Enter the version number you are using (if known).
+      placeholder: "e.g., v1.2.3"
+    validations:
+      required: false
+
+  - type: input
+    id: browser
+    attributes:
+      label: Browser
+      description: Which browser are you using?
+      placeholder: "e.g., Chrome 124, Firefox 115, Safari 17"
+    validations:
+      required: true
+
+  - type: input
+    id: operating_system
+    attributes:
+      label: Operating System
+      description: Your operating system and version.
+      placeholder: "e.g., Windows 11, macOS Sonoma, Ubuntu 22.04"
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps_to_reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: How can we reproduce the problem?
+      placeholder: |
+        1. Go to '...'
+        2. Click on '...'
+        3. Scroll down to '...'
+        4. See error
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected_behavior
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen instead?
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual_behavior
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant Logs or Screenshots
+      description: Please paste any logs, screenshots, or console errors if available.
+      placeholder: "Paste logs or upload screenshots."
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional_context
+    attributes:
+      label: Additional Context
+      description: Anything else we should know?
+    validations:
+      required: false

ENVEXAMPLE

@@ -7,6 +7,8 @@ DB_PASSWORD=postgres                    # PostgreSQL password
 DB_HOST=postgres                        # Host for PostgreSQL in Docker
 DB_PORT=5432                            # Port for PostgreSQL (default: 5432)
 ENCRYPTION_KEY=f4d5e6a7b8c9d0e1f23456789abcdef01234567890abcdef123456789abcdef0      # Key for encrypting sensitive data (passwords and proxies)
+SESSION_SECRET=maxun_session # A strong, random string used to sign session cookies. Recommended to define your own session secret to avoid session hijacking.
+
 MINIO_ENDPOINT=minio                    # MinIO endpoint in Docker
 MINIO_PORT=9000                         # Port for MinIO (default: 9000)
 MINIO_CONSOLE_PORT=9001                 # Web UI Port for MinIO (default: 9001)

README.md

@@ -98,6 +98,7 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca
 | `DB_HOST`             | Yes       | Host address where the Postgres database server is running.                                  | Database connection will fail.                               |
 | `DB_PORT`             | Yes       | Port number used to connect to the Postgres database server.                                 | Database connection will fail.                               |
 | `ENCRYPTION_KEY`      | Yes       | Key used for encrypting sensitive data (proxies, passwords).                                 | Encryption functionality will not work.                      |
+| `SESSION_SECRET`      | No       | A strong, random string used to sign session cookies                                          | Uses default secret. Recommended to define your own session secret to avoid session hijacking.  |
 | `MINIO_ENDPOINT`      | Yes       | Endpoint URL for MinIO, to store Robot Run Screenshots.                                      | Connection to MinIO storage will fail.                       |
 | `MINIO_PORT`          | Yes       | Port number for MinIO service.                                                               | Connection to MinIO storage will fail.                       |
 | `MINIO_CONSOLE_PORT`          | No       | Port number for MinIO WebUI service. Needed for Docker setup.                         | Cannot access MinIO Web UI. |

index.html

@@ -6,7 +6,7 @@
     <meta name="theme-color" content="#000000" />
     <meta
       name="description"
-      content="Web site created using Vite"
+      content="Maxun is an open-source no-code web data extraction platform. Train a robot in 2 minutes to extract data on auto-pilot!"
     />
     <link rel="icon" type="image/png" href="src/assets/maxunlogo.png">
     <title>Maxun | Open Source No Code Web Data Extraction Platform</title>

maxun-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",

maxun-core/src/interpret.ts

@@ -37,14 +37,16 @@ declare global {
  * Defines optional intepreter options (passed in constructor)
  */
 interface InterpreterOptions {
+  mode?: string;
   maxRepeats: number;
   maxConcurrency: number;
   serializableCallback: (output: any) => (void | Promise<void>);
   binaryCallback: (output: any, mimeType: string) => (void | Promise<void>);
   debug: boolean;
   debugChannel: Partial<{
-    activeId: Function,
-    debugMessage: Function,
+    activeId: (id: number) => void,
+    debugMessage: (msg: string) => void,
+    setActionType: (type: string) => void,
   }>
 }
 
@@ -377,12 +379,20 @@ export default class Interpreter extends EventEmitter {
      */
     const wawActions: Record<CustomFunctions, (...args: any[]) => void> = {
       screenshot: async (params: PageScreenshotOptions) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('screenshot');
+        }
+
         const screenshotBuffer = await page.screenshot({
           ...params, path: undefined,
         });
         await this.options.binaryCallback(screenshotBuffer, 'image/png');
       },
       enqueueLinks: async (selector: string) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('enqueueLinks');
+        }
+
         const links: string[] = await page.locator(selector)
           .evaluateAll(
             // @ts-ignore
@@ -409,55 +419,61 @@ export default class Interpreter extends EventEmitter {
         await page.close();
       },
       scrape: async (selector?: string) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scrape');
+        }
+
         await this.ensureScriptsLoaded(page);
 
         const scrapeResults: Record<string, string>[] = await page.evaluate((s) => window.scrape(s ?? null), selector);
         await this.options.serializableCallback(scrapeResults);
       },
 
       scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; shadow: string}>) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scrapeSchema');
+        }
+
+        if (this.options.mode && this.options.mode === 'editor') {
+          await this.options.serializableCallback({});
+          return;
+        }
+
         await this.ensureScriptsLoaded(page);
 
         const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
 
-        const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult];
-        newResults.forEach((result) => {
-          Object.entries(result).forEach(([key, value]) => {
-              const keyExists = this.cumulativeResults.some(
-                  (item) => key in item && item[key] !== undefined
-              );
-
-              if (!keyExists) {
-                  this.cumulativeResults.push({ [key]: value });
-              }
-          });
+        if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
+          this.cumulativeResults = [];
+        }
+
+        if (this.cumulativeResults.length === 0) {
+          this.cumulativeResults.push({});
+        }
+
+        const mergedResult = this.cumulativeResults[0];
+        const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
+
+        Object.entries(resultToProcess).forEach(([key, value]) => {
+          if (value !== undefined) {
+            mergedResult[key] = value;
+          }
         });
-
-        const mergedResult: Record<string, string>[] = [
-          Object.fromEntries( 
-            Object.entries(
-              this.cumulativeResults.reduce((acc, curr) => {
-                Object.entries(curr).forEach(([key, value]) => {
-                  // If the key doesn't exist or the current value is not undefined, add/update it
-                  if (value !== undefined) {
-                    acc[key] = value;
-                  }
-                });
-                return acc;
-              }, {})
-            )
-          )
-        ];
-
-        // Log cumulative results after each action
-        console.log("CUMULATIVE results:", this.cumulativeResults);
-        console.log("MERGED results:", mergedResult);
-
-        await this.options.serializableCallback(mergedResult);
-        // await this.options.serializableCallback(scrapeResult);
+
+        console.log("Updated merged result:", mergedResult);
+        await this.options.serializableCallback([mergedResult]);
       },
 
       scrapeList: async (config: { listSelector: string, fields: any, limit?: number, pagination: any }) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scrapeList');
+        }
+
+        if (this.options.mode && this.options.mode === 'editor') {
+          await this.options.serializableCallback({});
+          return;
+        }
+
         await this.ensureScriptsLoaded(page);
         if (!config.pagination) {
           const scrapeResults: Record<string, any>[] = await page.evaluate((cfg) => window.scrapeList(cfg), config);
@@ -469,6 +485,10 @@ export default class Interpreter extends EventEmitter {
       },
 
       scrapeListAuto: async (config: { listSelector: string }) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scrapeListAuto');
+        }
+
         await this.ensureScriptsLoaded(page);
 
         const scrapeResults: { selector: string, innerText: string }[] = await page.evaluate((listSelector) => {
@@ -479,6 +499,10 @@ export default class Interpreter extends EventEmitter {
       },
 
       scroll: async (pages?: number) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scroll');
+        }
+
         await page.evaluate(async (pagesInternal) => {
           for (let i = 1; i <= (pagesInternal ?? 1); i += 1) {
             // @ts-ignore
@@ -488,6 +512,10 @@ export default class Interpreter extends EventEmitter {
       },
 
       script: async (code: string) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('script');
+        }
+
         const AsyncFunction: FunctionConstructor = Object.getPrototypeOf(
           async () => { },
         ).constructor;
@@ -496,6 +524,10 @@ export default class Interpreter extends EventEmitter {
       },
 
       flag: async () => new Promise((res) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('flag');
+        }
+
         this.emit('flag', page, res);
       }),
     };
@@ -526,6 +558,10 @@ export default class Interpreter extends EventEmitter {
         const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
         await wawActions[step.action as CustomFunctions](...(params ?? []));
       } else {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType(String(step.action));
+        }
+
         // Implements the dot notation for the "method name" in the workflow
         const levels = String(step.action).split('.');
         const methodName = levels[levels.length - 1];

nginx.conf

@@ -1,45 +1,24 @@
 server {
     listen 80;
+    server_name _;
 
+    root /var/www/maxun;
+    index index.html;
+
+    # Serve the frontend
     location / {
-        root /usr/share/nginx/html;
         try_files $uri $uri/ /index.html;
     }
-
-    location /api {
-        proxy_pass http://localhost:8080;
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection 'upgrade';
-        proxy_set_header Host $host;
-        proxy_cache_bypass $http_upgrade;
-
-        # Add timeout configurations
-        proxy_connect_timeout 60s;
-        proxy_send_timeout 60s;
-        proxy_read_timeout 60s;
-
-        # Add error handling
-        proxy_intercept_errors on;
-        error_page 502 503 504 /50x.html;
-    }
-
-    location ~ ^/(record|workflow|storage|auth|integration|proxy|api-docs) {
-        proxy_pass http://localhost:8080;
+
+    # Proxy for backend 
+    location  ^/(auth|storage|record|workflow|robot|proxy|api-docs|api)(/|$) {
+        proxy_pass http://localhost:8080; # change as per your setup
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection 'keep-alive';  # Ensure connections remain open
         proxy_set_header Connection 'upgrade';
         proxy_set_header Host $host;
-        proxy_cache_bypass $http_upgrade;
-
-        # Timeout configurations
-        proxy_connect_timeout 60s;
-        proxy_send_timeout 60s;
-        proxy_read_timeout 60s;
-
-        # Error handling for these routes
-        proxy_intercept_errors on;
-        error_page 502 503 504 /50x.html;
+        proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    	proxy_set_header X-Forwarded-Proto $scheme;
     }
-}
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "author": "Maxun",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
@@ -50,7 +50,7 @@
     "lodash": "^4.17.21",
     "loglevel": "^1.8.0",
     "loglevel-plugin-remote": "^0.6.8",
-    "maxun-core": "^0.0.15",
+    "maxun-core": "^0.0.16",
     "minio": "^8.0.1",
     "moment-timezone": "^0.5.45",
     "node-cron": "^3.0.3",