Feat: Add image extraction for webpages (media parsing)

mediaParser.js

@@ -0,0 +1,44 @@
+const axios = require('axios');
+const cheerio = require('cheerio');
+const { URL } = require('url');
+
+async function extractImages(url) {
+    try {
+        // 1. Fetch HTML
+        const { data: html } = await axios.get(url);
+
+        // 2. Load HTML into cheerio
+        const $ = cheerio.load(html);
+
+        const images = [];
+        const seen = new Set(); // to track duplicates
+
+        // 3. Loop through each <img> tag
+        $('img').each((index, element) => {
+            let src = $(element).attr('src');
+            const alt = $(element).attr('alt') || '';
+
+            if (src) {
+                // 4. Convert relative URLs to absolute URLs
+                try {
+                    src = new URL(src, url).href;
+                } catch {
+                    // skip invalid URLs
+                    return;
+                }
+
+                // 5. Skip duplicates
+                if (!seen.has(src)) {
+                    seen.add(src);
+                    images.push({ url: src, altText: alt });
+                }
+            }
+        });
+
+        return images;
+
+    } catch (error) {
+        console.log('Oops! Something went wrong while fetching images:', error.message);
+        return [];
+    }
+}

package.json

@@ -24,10 +24,11 @@
     "@types/react-dom": "^18.0.1",
     "@types/uuid": "^8.3.4",
     "airtable": "^0.12.2",
-    "axios": "^1.9.0",
+    "axios": "^1.12.2",
     "bcrypt": "^5.1.1",
     "body-parser": "^1.20.3",
     "buffer": "^6.0.3",
+    "cheerio": "^1.1.2",
     "connect-pg-simple": "^10.0.0",
     "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
@@ -80,6 +81,7 @@
     "styled-components": "^5.3.3",
     "swagger-jsdoc": "^6.2.8",
     "swagger-ui-express": "^5.0.1",
+    "tesseract.js": "^6.0.1",
     "typedoc": "^0.23.8",
     "typescript": "^4.6.3",
     "uuid": "^8.3.2",