Development

lib/constant.js

@@ -0,0 +1,37 @@
+const BulletPointsUnicode = [
+  '\u006F',
+  '\u0308',
+  '\u2013',
+  '\u2014',
+  '\u2022',
+  '\u2023',
+  '\u2043',
+  '\u204C',
+  '\u204D',
+  '\u2217',
+  '\u2219',
+  '\u25A0',
+  '\u25A1',
+  '\u25AA',
+  '\u25AB',
+  '\u25B8',
+  '\u25BA',
+  '\u25C6',
+  '\u25CB',
+  '\u25CF',
+  '\u25D8',
+  '\u25E6',
+  '\u2610',
+  '\u2605',
+  '\u26AB',
+  '\u26AC',
+  '\u2726',
+  '\u2736',
+  '\u27A2',
+  '\u29BE',
+  '\u2713',
+  '\u2729',
+  '\u29BF'
+];
+
+module.exports = BulletPointsUnicode;

lib/extract.js

@@ -1,3 +1,5 @@
+const BulletPointsUnicode = require( './constant' );
+
 var fs = require( 'fs' )
   , path = require( 'path' )
   , XmlEntities = require( 'html-entities' ).XmlEntities
@@ -11,8 +13,8 @@ var fs = require( 'fs' )
   , satisfiedExtractors = 0
   , hasInitialized = false
   , STRIP_ONLY_SINGLE_LINEBREAKS = /(^|[^\n])\n(?!\n)/g
-  , WHITELIST_PRESERVE_LINEBREAKS = /[^A-Za-z\x80-\xFF\x24\u20AC\xA3\xA5 0-9 \u2015\u2116\u2018\u2019\u201C|\u201D\u2026 \uFF0C \u2013 \u2014 \u00C0-\u1FFF \u2C00-\uD7FF \uFB50–\uFDFF \uFE70–\uFEFF \uFF01-\uFFE6 \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~'-\w\n\r]*/g  // eslint-disable-line max-len
-  , WHITELIST_STRIP_LINEBREAKS = /[^A-Za-z\x80-\xFF\x24\u20AC\xA3\xA5 0-9 \u2015\u2116\u2018\u2019\u201C|\u201D\u2026 \uFF0C \u2013 \u2014 \u00C0-\u1FFF \u2C00-\uD7FF \uFB50–\uFDFF \uFE70–\uFEFF \uFF01-\uFFE6 \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~'-\w]*/g  // eslint-disable-line max-len
+  , WHITELIST_PRESERVE_LINEBREAKS = 'A-Za-z\x80-\xFF\x24\u20AC\xA3\xA5 0-9 \u2015\u2116\u2018\u2019\u201C|\u201D\u2026 \uFF0C \u2013 \u2014 \u00C0-\u1FFF \u2C00-\uD7FF \uFB50–\uFDFF \uFE70–\uFEFF \uFF01-\uFFE6 .,?""!@#$%^&*()-_=+;:<>/\\\|}{\\[\\]`~\'-\\w\\n\\r'  // eslint-disable-line max-len
+  , WHITELIST_STRIP_LINEBREAKS = 'A-Za-z\x80-\xFF\x24\u20AC\xA3\xA5 0-9 \u2015\u2116\u2018\u2019\u201C|\u201D\u2026 \uFF0C \u2013 \u2014 \u00C0-\u1FFF \u2C00-\uD7FF \uFB50–\uFDFF \uFE70–\uFEFF \uFF01-\uFFE6 .,?""!@#$%^&*()-_=+;:<>/\\\|}{\\[\\]`~\'-\\w'  // eslint-disable-line max-len
   ;
 
 function registerExtractor( extractor ) {
@@ -51,31 +53,40 @@ function testExtractor( extractor, options ) {
 
 // global, all file type, content cleansing
 function cleanseText( options, cb ) {
-  return function( error, text ) {
+  return function( error, result ) {
     if ( !error ) {
       // clean up text
-      text = util.replaceBadCharacters( text );
+      let texts = util.replaceBadCharacters( result.texts );
+      const bulletPattern = BulletPointsUnicode.join( '' )
+        , preserveLineBreaksReg = util.generatePreserveRegex(
+          WHITELIST_PRESERVE_LINEBREAKS, bulletPattern
+        )
+        , stripLineBreaksReg = util.generatePreserveRegex(
+          WHITELIST_STRIP_LINEBREAKS, bulletPattern
+        );
 
       if ( options.preserveLineBreaks || options.preserveOnlyMultipleLineBreaks ) {
         if ( options.preserveOnlyMultipleLineBreaks ) {
-          text = text.replace( STRIP_ONLY_SINGLE_LINEBREAKS, '$1 ' ).trim();
+          texts = texts.replace( STRIP_ONLY_SINGLE_LINEBREAKS, '$1 ' ).trim();
         }
-        text = text.replace( WHITELIST_PRESERVE_LINEBREAKS, ' ' );
+        texts = texts.replace( preserveLineBreaksReg, ' ' );
       } else {
-        text = text.replace( WHITELIST_STRIP_LINEBREAKS, ' ' );
+        texts = texts.replace( stripLineBreaksReg, ' ' );
       }
 
       // multiple spaces, tabs, vertical tabs, non-breaking space]
-      text = text.replace( / (?! )/g, '' )
+      texts = texts.replace( /\s(?=\s)/g, '' )
         .replace( /[ \t\v\u00A0]{2,}/g, ' ' );
 
-      text = entities.decode( text );
+      texts = entities.decode( texts );
+      result.texts = texts;
     }
-    cb( error, text );
+    cb( error, result );
   };
 }
 
 function initializeExtractors( options ) {
+  console.log({ options });
   var extractors;
 
   hasInitialized = true;

lib/extractors/doc-osx.js

@@ -30,7 +30,7 @@ function extractText( filePath, options, cb ) {
       cb( error, null );
       return;
     }
-    cb( null, result.trim() );
+    cb( null, { texts: result.trim() });
   });
 }
 

lib/extractors/doc.js

@@ -22,7 +22,7 @@ function extractText( filePath, options, cb ) {
         }
         cb( err, null );
       } else {
-        cb( null, stdout.trim().replace( /\[pic\]/g, '' ) );
+        cb( null, { texts: stdout.trim().replace( /\[pic\]/g, '' ) });
       }
     }
   );

lib/extractors/docx.js

@@ -59,7 +59,7 @@ function extractText( filePath, options, cb ) {
       if ( zipfile.entryCount === ++processedEntries ) {
         if ( result.length ) {
           text = _calculateExtractedText( result, options.preserveLineBreaks );
-          cb( null, text );
+          cb( null, { texts: text });
         } else {
           cb( new Error(
             'Extraction could not find content in file, are you' +

lib/extractors/dxf.js

@@ -18,7 +18,7 @@ function extractText( filePath, options, cb ) {
         return;
       }
 
-      cb( null, stdout );
+      cb( null, { texts: stdout });
     }
   );
 }

lib/extractors/epub.js

@@ -29,7 +29,7 @@ function extractText( filePath, options, cb ) {
                 allText += outText;
                 chapterCount++;
                 if ( chapterCount === epub.flow.length ) {
-                  cb( null, allText );
+                  cb( null, { texts: allText });
                 }
               }
             });

lib/extractors/html.js

@@ -73,7 +73,7 @@ function extractFromText( data, options, cb ) {
     return;
   }
 
-  cb( null, text );
+  cb( null, { texts: text });
 }
 
 function extractText( filePath, options, cb ) {

lib/extractors/odt.js

@@ -45,7 +45,7 @@ function extractText( filePath, options, cb ) {
             nodeTexts.push( $( nodes[i] ).text() );
           }
 
-          cb( null, nodeTexts.join( '\n' ) );
+          cb( null, { texts: nodeTexts.join( '\n' ) });
         });
       }
     });

lib/extractors/pdf.js

@@ -1,23 +1,45 @@
+/* eslint-disable no-unused-expressions */
 var path = require( 'path' )
   , exec = require( 'child_process' ).exec
-  , extract = require( 'pdf-text-extract' )
-  ;
+  , PDFExtract = require( 'pdf.js-extract' ).PDFExtract
+  , fs = require( 'fs' );
 
 function extractText( filePath, options, cb ) {
   // See https://github.com/dbashford/textract/issues/75 for description of
-  // what is happening here
-  var pdftotextOptions = options.pdftotextOptions || { layout: 'raw' };
-
-  extract( filePath, pdftotextOptions, function( error, pages ) {
-    var fullText;
-    if ( error ) {
-      error = new Error( 'Error extracting PDF text for file at [[ ' +
-        path.basename( filePath ) + ' ]], error: ' + error.message );
+  // what is happening here;
+  const pdfExtract = new PDFExtract()
+  , buffer = fs.readFileSync( filePath )
+  , option = { disableCombineTextItems: true }; /* see below */
+  pdfExtract.extractBuffer( buffer, option, ( err, data ) => {
+    if ( err ) {
+      const error = new Error( 'Error extracting PDF text for file at [[ ' +
+        path.basename( filePath ) + ' ]], error: ' + err.message );
       cb( error, null );
       return;
     }
-    fullText = pages.join( ' ' ).trim();
-    cb( null, fullText );
+
+    if ( data ) {
+      const fonts = {}
+      , contents = data.pages.map( ({ content }) => content
+        .sort( ({ y: aY }, { y: bY }) => aY - bY )
+        .reduce( ( prev, { y, str, x, height, width, fontName }) => {
+          const prevLen = prev.length;
+          if ( prev[prevLen - 1] && prev[prevLen - 1].y === y ) {
+            prev[prevLen - 1].str += str;
+            prev[prevLen - 1].metas.push({ str, fontName, x, y, height, width });
+          } else {
+            prev.push({ str, metas: [{ str, fontName, x, y, height, width }], y });
+          }
+          fonts[fontName] ? fonts[fontName]++ : fonts[fontName] = 1;
+
+          return prev;
+        }, [] )
+      )
+      , texts = contents.map( ( content ) =>
+          content.map( ({ str }) => str.trim() ? str.trim() : str ).join( `\n` )
+        );
+      cb( null, { texts: texts.join( '\n' ).trim(), contents, fonts });
+    }
   });
 }
 

lib/extractors/pptx.js

@@ -62,7 +62,7 @@ function extractText( filePath, options, cb ) {
           return slide.text;
         }).join( '\n' );
         text = _calculateExtractedText( slidesText );
-        cb( null, text );
+        cb( null, { texts: text });
       } else {
         cb(
           new Error( 'Extraction could not find slides in file, are you' +

lib/extractors/text.js

@@ -26,7 +26,7 @@ function extractText( filePath, options, cb ) {
       cb( e );
       return;
     }
-    cb( null, decoded );
+    cb( null, { texts: decoded });
   });
 }
 

lib/extractors/xls.js

@@ -19,7 +19,7 @@ function extractText( filePath, options, cb ) {
     result += CSVs[key];
   });
 
-  cb( null, result );
+  cb( null, { texts: result });
 }
 
 module.exports = {

lib/util.js

@@ -153,11 +153,22 @@ function runExecIntoFile( label, filePath, options, execOptions, genCommand, cb
   );
 }
 
+/**
+ * Generates regex pattern to preserve define characters and remove the rest.
+ *
+ * @param  {...string} args Regex pattern.
+ * @returns {string}
+ */
+function generatePreserveRegex( ...args ) {
+  return `[^${args.map( arg => arg )}]*`;
+}
+
 module.exports = {
   createExecOptions: createExecOptions,
   unzipCheck: unzipCheck,
   getTextFromZipFile: getTextFromZipFile,
   yauzlError: yauzlError,
   runExecIntoFile: runExecIntoFile,
-  replaceBadCharacters: replaceBadCharacters
+  replaceBadCharacters: replaceBadCharacters,
+  generatePreserveRegex: generatePreserveRegex
 };

package.json

@@ -47,32 +47,32 @@
     "epub"
   ],
   "dependencies": {
-    "mime": "2.2.0",
-    "pdf-text-extract": "1.3.1",
-    "xpath": "0.0.23",
-    "xmldom": "0.1.27",
-    "j": "0.4.3",
     "cheerio": "1.0.0-rc.2",
-    "marked": "0.6.2",
-    "meow": "3.7.0",
+    "epub2": "1.3.4",
     "got": "5.7.1",
     "html-entities": "1.2.0",
     "iconv-lite": "0.4.15",
+    "j": "0.4.3",
     "jschardet": "1.4.1",
-    "yauzl": "2.7.0",
-    "epub2": "1.3.4"
+    "marked": "0.6.2",
+    "meow": "3.7.0",
+    "mime": "2.2.0",
+    "pdf.js-extract": "^0.2.1",
+    "xmldom": "0.1.27",
+    "xpath": "0.0.23",
+    "yauzl": "2.7.0"
   },
   "devDependencies": {
     "chai": "1.5.0",
     "eslint": "2.11.1",
     "eslint-config-airbnb": "^9.0.1",
-    "eslint-plugin-react": "^5.1.1",
-    "eslint-plugin-jsx-a11y": "^1.2.0",
     "eslint-plugin-import": "^1.7.0 ",
+    "eslint-plugin-jsx-a11y": "^1.2.0",
+    "eslint-plugin-react": "^5.1.1",
     "mocha": "1.9.0"
   },
   "scripts": {
-    "test": "node_modules/.bin/mocha",
+    "test": "node ./test/index.js",
     "lint": "node_modules/.bin/eslint -c .eslintrc.json lib"
   },
   "license": "MIT",

test/index.js

@@ -0,0 +1,5 @@
+let textract = require( '../lib' );
+
+textract.fromFileWithPath( '/Users/xplicit/Documents/work/personal/ai-cv-generator-documents/cvs/Oluwakemi Akinola-Adedeji_Resume.pdf', { preserveLineBreaks: true }, ( error, text ) => {
+  console.log({ text: text.texts });
+});

yarn.lock

@@ -358,6 +358,11 @@ domhandler@^2.3.0:
   dependencies:
     domelementtype "1"
 
+[email protected]:
+  version "0.0.24"
+  resolved "https://registry.yarnpkg.com/dommatrix/-/dommatrix-0.0.24.tgz#0b793da372992878b74c31f64ac85e7a13cb382a"
+  integrity sha512-PatEhAW5pIHr28MvFQGV5iiHNloqvecQZlxs7/8s/eulLqZI3uVqPkrO7YDuqsebovr/9mmcWDSWzVG4amEZgQ==
+
 [email protected]:
   version "1.5.1"
   resolved "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf"
@@ -993,9 +998,10 @@ map-obj@^1.0.0, map-obj@^1.0.1:
   version "1.0.1"
   resolved "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"
 
-[email protected]:
-  version "0.3.17"
-  resolved "https://registry.npmjs.org/marked/-/marked-0.3.17.tgz#607f06668b3c6b1246b28f13da76116ac1aa2d2b"
+[email protected]:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/marked/-/marked-0.6.2.tgz#c574be8b545a8b48641456ca1dbe0e37b6dccc1a"
+  integrity sha512-LqxwVH3P/rqKX4EKGz7+c2G9r98WeM/SW34ybhgNGhUQNKtf1GmmSkJ6cDGJ/t6tiyae49qRkpyTw2B9HOrgUA==
 
 [email protected]:
   version "3.7.0"
@@ -1166,11 +1172,13 @@ path-type@^1.0.0:
     pify "^2.0.0"
     pinkie-promise "^2.0.0"
 
-[email protected]:
-  version "1.3.1"
-  resolved "https://registry.npmjs.org/pdf-text-extract/-/pdf-text-extract-1.3.1.tgz#a5b232ad9850949d77c435e5f44fe0d105435840"
+pdf.js-extract@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/pdf.js-extract/-/pdf.js-extract-0.2.1.tgz#9f574932bb442c79a54efd9eb5f93ec6454f6a5d"
+  integrity sha512-oUs5KaTVCelIyiBajCx3zAZKurkN9oVwRdqbSeDqeofddxNuwJRur86fCETvKZ/tX5nZJUSZWq3ie76PsArz7A==
   dependencies:
-    yargs "^1.2.5"
+    dommatrix "0.0.24"
+    web-streams-polyfill "3.2.0"
 
 pend@~1.2.0:
   version "1.2.0"
@@ -1500,6 +1508,11 @@ voc@:
   version "1.1.0"
   resolved "https://registry.npmjs.org/voc/-/voc-1.1.0.tgz#d1a08aeff66646bf17cdba2e47c935a7a9b0218b"
 
+[email protected]:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.0.tgz#a6b74026b38e4885869fb5c589e90b95ccfc7965"
+  integrity sha512-EqPmREeOzttaLRm5HS7io98goBgZ7IVz79aDvqjD0kYXLtFZTc0T/U6wHTPKyIjb+MdN7DFIIX6hgdBEpWmfPA==
+
 wordwrap@~1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
@@ -1559,10 +1572,6 @@ xtend@^4.0.0:
   version "4.0.1"
   resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"
 
-yargs@^1.2.5:
-  version "1.3.3"
-  resolved "https://registry.npmjs.org/yargs/-/yargs-1.3.3.tgz#054de8b61f22eefdb7207059eaef9d6b83fb931a"
-
 [email protected]:
   version "2.7.0"
   resolved "https://registry.npmjs.org/yauzl/-/yauzl-2.7.0.tgz#e21d847868b496fc29eaec23ee87fdd33e9b2bce"