Merge pull request #94 from extractus/6.2.4

v6.2.4

Author: ndaidong

.github/workflows/ci-test.yml

@@ -23,6 +23,8 @@ jobs:
         node-version: ${{ matrix.node_version }}
 
     - name: run npm scripts
+      env:
+        PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
       run: |
         npm install
         npm run lint

README.md

@@ -214,6 +214,26 @@ await extract(url, null, {
 Passing requests to proxy is useful while running `@extractus/feed-extractor` on browser.
 View `examples/browser-feed-reader` as reference example.
 
+Another way to work with proxy is use `agent` option instead of `proxy` as below:
+
+```js
+import { extract } from '@extractus/feed-extractor'
+
+import { HttpsProxyAgent } from 'https-proxy-agent'
+
+const proxy = 'http://abc:[email protected]:31113'
+
+const url = 'https://news.google.com/rss'
+
+const feed = await extract(url, null, {
+  agent: new HttpsProxyAgent(proxy),
+})
+console.log('Run feed-extractor with proxy:', proxy)
+console.log(feed)
+```
+
+For more info about [https-proxy-agent](https://www.npmjs.com/package/https-proxy-agent), check [its repo](https://github.com/TooTallNate/proxy-agents).
+
 
 ### `extractFromJson()`
 

dist/cjs/feed-extractor.js

@@ -1,5 +1,5 @@
 {
   "name": "@extractus/feed-extractor",
-  "version": "6.2.3",
+  "version": "6.2.4",
   "main": "./feed-extractor.js"
 }

dist/cjs/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.2.3",
+  "version": "6.2.4",
   "name": "@extractus/feed-extractor",
   "description": "To read and normalize RSS/ATOM/JSON feed data",
   "homepage": "https://extractor-demos.pages.dev",
@@ -36,14 +36,15 @@
   },
   "dependencies": {
     "bellajs": "^11.1.2",
-    "cross-fetch": "^3.1.6",
-    "fast-xml-parser": "^4.2.4",
-    "html-entities": "^2.3.6"
+    "cross-fetch": "^4.0.0",
+    "fast-xml-parser": "^4.2.5",
+    "html-entities": "^2.4.0"
   },
   "devDependencies": {
-    "esbuild": "^0.18.2",
-    "eslint": "^8.42.0",
-    "jest": "^29.5.0",
+    "esbuild": "^0.18.11",
+    "eslint": "^8.44.0",
+    "https-proxy-agent": "^7.0.0",
+    "jest": "^29.6.0",
     "nock": "^13.3.1"
   },
   "keywords": [

dist/feed-extractor.esm.js

@@ -4,12 +4,16 @@
 import { readFileSync } from 'fs'
 
 import nock from 'nock'
+import { HttpsProxyAgent } from 'https-proxy-agent'
 
 import { hasProperty, isString } from 'bellajs'
 
 import { extract, extractFromXml, extractFromJson, read } from './main.js'
 import { isValid as isValidUrl } from './utils/linker.js'
 
+const env = process.env || {}
+const PROXY_SERVER = env.PROXY_SERVER || ''
+
 const feedAttrs = 'title link description generator language published entries'.split(' ')
 const entryAttrs = 'title link description published id'.split(' ')
 
@@ -395,6 +399,19 @@ describe('test extract with `baseUrl` option', () => {
   })
 })
 
+if (PROXY_SERVER !== '') {
+  describe('test extract live RSS via proxy server', () => {
+    test('check if extract method works with proxy server', async () => {
+      const url = 'https://news.google.com/rss'
+      const result = await extract(url, {}, {
+        agent: new HttpsProxyAgent(PROXY_SERVER),
+      })
+      expect(result.title).toContain('Google News')
+      expect(result.entries.length).toBeGreaterThan(0)
+    }, 10000)
+  })
+}
+
 describe('check old method read()', () => {
   test('ensure that depricated method read() still works', async () => {
     const url = 'https://realworld-standard-feed.tld/rss'

package.json

@@ -19,9 +19,10 @@ export default async (url, options = {}) => {
       'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0',
     },
     proxy = null,
+    agent = null,
   } = options
 
-  const res = proxy ? await profetch(url, proxy) : await fetch(url, { headers })
+  const res = proxy ? await profetch(url, proxy) : await fetch(url, { headers, agent })
 
   const status = res.status
   if (status >= 400) {