Merge pull request #40 from eviltik/main

return axios errors, trim() response body, skipContentTypeCheck

Author: ndaidong

src/config.js

@@ -9,7 +9,8 @@ const requestOptions = {
   responseType: 'text',
   responseEncoding: 'utf8',
   timeout: 6e4, // 1 minute
-  maxRedirects: 3
+  maxRedirects: 3,
+  skipContentTypeCheck: false
 }
 
 export const getRequestOptions = () => {

src/main.js

@@ -22,13 +22,16 @@ export {
 
 export const read = async (url) => {
   const xmldata = await getXML(url)
-  if (!xmldata) {
-    throw new Error(`Could not fetch XML content from "${url}"`)
+
+  const { xml, error } = xmldata
+  if (error) {
+    throw error
   }
-  const { xml } = xmldata
+
   if (!validate(xml)) {
     throw new Error(`Failed while validating XML format from "${url}"`)
   }
+
   info('Parsing XML data...')
   const jsonObj = xml2obj(xml)
   return isRSS(jsonObj) ? parseRSS(jsonObj) : isAtom(jsonObj) ? parseAtom(jsonObj) : null

src/utils/retrieve.js

@@ -6,22 +6,49 @@ import logger from './logger.js'
 
 import { getRequestOptions } from '../config.js'
 
+const acceptedContentTypes = [
+  'text/xml',
+  'application/xml',
+  'application/atom+xml',
+  'application/rss+xml',
+  'application/x-rss+xml'
+]
+
 export default async (url) => {
+  if (typeof url !== 'string') {
+    const error = new Error('url should be a string')
+    return { error }
+  }
+
+  const skipContentTypeCheck = getRequestOptions().skipContentTypeCheck || false
+
   try {
     const res = await axios.get(url, getRequestOptions())
 
-    const contentType = res.headers['content-type'] || ''
-    if (!contentType || !contentType.includes('xml')) {
-      logger.error(`Got invalid content-type (${contentType}) from "${url}"`)
-      return null
+    let contentType = res.headers['content-type'] || ''
+    // text/xml;charset= ..
+    contentType = contentType.split(';')[0]
+
+    if (!skipContentTypeCheck) {
+      if (!contentType || acceptedContentTypes.indexOf(contentType) === -1) {
+        logger.error(`Got invalid content-type (${contentType}) from "${url}"`)
+        const error = new Error(`invalid content-type (${contentType}) from "${url}"`)
+        return { error }
+      }
+    }
+
+    if (res.data === undefined) {
+      const error = new Error('something wrong append with axios')
+      return { error }
     }
+
     const result = {
       url,
-      xml: res.data
+      xml: res.data.trim()
     }
     return result
-  } catch (err) {
-    logger.error(err.message || err)
-    return null
+  } catch (error) {
+    logger.error(error.message || error)
+    return { error }
   }
 }

src/utils/retrieve.test.js

@@ -24,24 +24,39 @@ test('test retrieve from good source', async () => {
   expect(result.xml).toBe('<div>this is content</div>')
 })
 
-test('test retrieve with invalid content type', async () => {
+test('test retrieve from good source, but having \\r\\n before/after root xml', async () => {
+  const url = 'https://some.where/good/page'
+  const { baseUrl, path } = parseUrl(url)
+  nock(baseUrl).head(path).reply(200)
+  nock(baseUrl).get(path).reply(200, '\n\r\r\n\n<div>this is content</div>\n\r\r\n\n', {
+    'Content-Type': 'application/xml'
+  })
+  const result = await retrieve(url)
+  expect(result.xml).toBe('<div>this is content</div>')
+})
+
+test('test retrieve with invalid response content type', async () => {
   const url = 'https://some.where/bad/page'
   const { baseUrl, path } = parseUrl(url)
   nock(baseUrl).head(path).reply(200)
   nock(baseUrl).get(path).reply(200, '', {
     'Content-Type': 'something/strange'
   })
   const result = await retrieve(url)
-  expect(result).toBe(null)
+  expect(typeof result).toBe('object')
+  expect(typeof result.error).toBe('object')
+  expect(typeof result.error.message).toBe('string')
 })
 
-test('test retrieve with invalid status', async () => {
+test('test retrieve with invalid response status code', async () => {
   const url = 'https://some.where/bad/page'
   const { baseUrl, path } = parseUrl(url)
   nock(baseUrl).head(path).reply(500)
   nock(baseUrl).get(path).reply(500, '<xml><message>Error 500</message></xml>', {
     'Content-Type': 'application/xml'
   })
   const result = await retrieve(url)
-  expect(result).toBe(null)
+  expect(typeof result).toBe('object')
+  expect(typeof result.error).toBe('object', result.error)
+  expect(typeof result.error.message).toBe('string')
 })