More Data From the PDF

server/api/lib/mavat/index.js

@@ -317,11 +317,21 @@ const getByPlan = plan =>
 				jurisdiction: getJurisdictionString(cheerioPage),
 				files: planFiles,
 				planStatusList: getPlanStatusList(cheerioPage),
-				planExplanation: pageInstructions ? pageInstructions.planExplanation : undefined,
-				chartsOneEight: pageInstructions ? pageInstructions.chartsOneEight : undefined,
-				chartFour: pageInstructions ? pageInstructions.chartFour : undefined,
-				chartFive: pageInstructions ? pageInstructions.chartFive : undefined,
-				chartSix: pageInstructions ? pageInstructions.chartSix : undefined
+				kindOfPlan: pageInstructions?.textOneFour?.kindOfPlan,
+				isContainsDetailed: pageInstructions?.textOneFour?.isContainsDetailed,
+				laws: pageInstructions?.textOneFour?.laws,
+				permit: pageInstructions?.textOneFour?.permit,
+				unionAndDivision: pageInstructions?.textOneFour?.unionAndDivision,
+				planExplanation: pageInstructions?.planExplanation,
+				chartOneSix: pageInstructions?.chartOneSix,
+				chartOneSeven: pageInstructions?.chartOneSeven,
+				chartsOneEight: pageInstructions?.chartsOneEight,
+				chartsThreeOne: pageInstructions?.chartsThreeOne,
+				chartsThreeTwo: pageInstructions?.chartsThreeTwo,
+				chartFour: pageInstructions?.chartFour,
+				chartFive: pageInstructions?.chartFive,
+				chartSix: pageInstructions?.chartSix,
+				chartSevenOne: pageInstructions?.chartSevenOne
 			});
 		});
 

server/api/lib/mavat/planInstructions/index.js

@@ -1,3 +1,6 @@
+const HummusRecipe = require('hummus-recipe');
+const hummus = require('hummus');
+
 const fs = require('fs');
 const util = require('util');
 const path = require('path');
@@ -10,18 +13,71 @@ const unlink = util.promisify(fs.unlink);
 const mkdir = util.promisify(fs.mkdir);
 
 const processPlanInstructionsFile = async (fileDir) => {
-	const files = await readdir(fileDir);
 
-	const pdfs = files.filter(fileName => fileName.endsWith('.pdf'));
+	try {
+		const files = await readdir(fileDir);
 
-	if (pdfs.length > 0) {
-		// we just want the one file, at least for now
-		const filePath = path.join(fileDir, pdfs[0]);
-		const data = await extractPdfData(filePath);
-		return data;
-	}
+		const pdfs = files.filter(fileName => fileName.endsWith('.pdf'));
+
+		if (pdfs.length > 0) {
+			const pdfFileName = pdfs[0];
+			let filePath = path.join(fileDir, pdfFileName);
+			let fileSizeInMB = fs.statSync(filePath).size / (1024 * 1024);
+
+
+			// the pdf extension that we are using to read table 3.2 is very problematic in memory usage.
+			// we know that the end of plans can contain heavy drawings.
+			// if the file is larger than 3 MB, the plan contains these drawings.
+			// We remove those drawings by looking at the pages size.
+			// Pages that contains text have a unique size of around 595 on 842 (or filpped)
+			// If the page is not at that size, it contains a drawing
+			if (fileSizeInMB > 3) {
+
+				const instream = new hummus.PDFRStreamForFile(filePath);
+				const pdfReader = hummus.createReader(instream);
+				const pageCount = pdfReader.getPagesCount();
+
+				let lastFinePage = 0;
+				for (let i = 0; i < pageCount; i++) {
+					const boxForPage = pdfReader.parsePage(i).getMediaBox();
+
+					if ((Math.abs(boxForPage[2] - 595) < 3 && Math.abs(boxForPage[3] - 842) < 3) ||
+						(Math.abs(boxForPage[3] - 595) < 3 && Math.abs(boxForPage[2] - 842) < 3)) {  // it can be flipped (table 5 page for example)
+						continue;
+					}
+
+					// if we got here, this is a page that is not fine
+					// it's not i - 1 because Hummus Recipe page index is starting at 1 (and this is starting at 0)
+					lastFinePage = i;
+					break;
+				}
 
-	return undefined;
+				instream.close();
+
+				if (lastFinePage !== 0) {
+
+					// everything but the last page
+					const pagesToAdd = [...Array(lastFinePage).keys()];
+
+					const strippedPdfPath = path.join(fileDir, 'strippedTemp.pdf');
+					const pdfDoc = new HummusRecipe('new', strippedPdfPath);
+					pdfDoc
+						.appendPage(filePath, pagesToAdd)
+						.endPDF();
+
+					filePath = strippedPdfPath;
+				}
+			}
+
+			const data = await extractPdfData(filePath);
+			return data;
+		}
+
+	}
+
+	catch(e) {
+		log.error('processPlanInstructionsFile error:\n' + e);
+	}
 };
 
 const clearOldPlanFiles = async (fileDir) => {

server/api/lib/mavat/planInstructions/parser/chartOneSeven.js

@@ -0,0 +1,62 @@
+const { chartToArrayBuilder } = require('./chartToArrayBuilder');
+const { getFromArr } = require('./parsersUtils');
+const log = require('../../../log');
+
+// this function look for the correct columns for the given headers, and returns a factory embedded with these findings
+const rowAbstractFactoryChart1_7 = (firstPageOfTable, headersStartIndex) => {
+	// clean the headers
+	firstPageOfTable = firstPageOfTable.map(row => row.map(cell => cell.replace(/\n/g, ' ')
+		.replace(/ {2}/g, ' ')
+		.trim()));
+
+	if (headersStartIndex === -1) {
+		// it's an error
+		log.error('didn\'t find headers for some char 1.6');
+		return () => {};
+	}
+
+	const header = firstPageOfTable[headersStartIndex];
+
+	const kindIdx = header.findIndex(title => title.includes('סוג המסמך'));
+	const containsIdx = header.findIndex(title => title.includes('תחולה'));
+	const scaleIdx = header.findIndex(title => title.includes('קנה מידה'));
+	const numberOfPagesIdx = header.findIndex(title => title.includes('מספר עמודים'));
+	const editDateIdx = header.findIndex(title => title.includes('תאריך עריכה'));
+	const editorIdx = header.findIndex(title => title.includes('עורך המסמך'));
+	const creationDateIdx = header.findIndex(title => title.includes('תאריך יצירה'));
+	const descriptionIdx = header.findIndex(title => title.includes('תיאור המסמך'));
+	const includedIdx = header.findIndex(title => title.includes('נכלל בהוראות התכנית'));
+
+
+
+	return (row) => {
+		return {
+			kind: getFromArr(row, kindIdx),
+			contains: getFromArr(row, containsIdx),
+			scale: getFromArr(row, scaleIdx),
+			number_of_pages: getFromArr(row, numberOfPagesIdx),
+			edit_date: getFromArr(row, editDateIdx),
+			editor: getFromArr(row, editorIdx),
+			creation_date: getFromArr(row, creationDateIdx),
+			description: getFromArr(row, descriptionIdx),
+			included: getFromArr(row, includedIdx),
+		};
+	};
+};
+
+const extractChartOneSeven = (pageTables) => {
+	return chartToArrayBuilder({
+		pageTables,
+		rowAbstractFactory: rowAbstractFactoryChart1_7,
+		startOfChartPred: (cell) => cell.includes('מסמכי התכנית') && cell.includes('1.7'),
+		offsetOfRowWithDataInChart: 1,
+		chartDonePredicate: (row) => row.some(cell => cell.includes('כל מסמכי התכנית מהווים חלק בלתי נפרד ממנה')),
+		getHeaderRowIndex: (page, searchFrom) => page.slice(searchFrom).findIndex(row => row.some(cell => cell.includes('סוג המסמך'))) + searchFrom,
+		rowTrimmer: (row) => row.map((cell) => cell.replace(/\n/g, ' ').replace(/ {2}/g, ' ').trim()),
+		identifier: '1.7'
+	});
+};
+
+module.exports = {
+	extractChartOneSeven
+};

server/api/lib/mavat/planInstructions/parser/chartOneSix.js

@@ -0,0 +1,55 @@
+const { chartToArrayBuilder } = require('./chartToArrayBuilder');
+const { getFromArr } = require('./parsersUtils');
+const log = require('../../../log');
+
+// this function look for the correct columns for the given headers, and returns a factory embedded with these findings
+const rowAbstractFactoryChart1_6 = (firstPageOfTable, headersStartIndex) => {
+	// clean the headers
+	firstPageOfTable = firstPageOfTable.map(row => row.map(cell => cell.replace(/\n/g, ' ')
+		.replace(/ {2}/g, ' ')
+		.trim()));
+
+	if (headersStartIndex === -1) {
+		// it's an error
+		log.error('didn\'t find headers for some char 1.6');
+		return () => {};
+	}
+
+	const header = firstPageOfTable[headersStartIndex];
+
+	const planNumberIdx = header.findIndex(title => title.includes('מספר תכנית מאושרת'));
+	const kindIdx = header.findIndex(title => title.includes('סוג יחס'));
+	const commentIdx = header.findIndex(title => title.includes('הערה ליחס'));
+	const yalkootNumberIdx = header.findIndex(title => title.includes('מספר ילקוט פרסומים'));
+	const yalkootPageNumberIdx = header.findIndex(title => title.includes('עמוד בילקוט פרסומים'));
+	const dateIdx = header.findIndex(title => title.includes('תאריך'));
+
+
+	return (row) => {
+		return {
+			prev_plan_number: getFromArr(row, planNumberIdx),
+			kind: getFromArr(row, kindIdx),
+			comment: getFromArr(row, commentIdx),
+			yalkoot_number: getFromArr(row, yalkootNumberIdx),
+			yalkoot_page_number: getFromArr(row, yalkootPageNumberIdx),
+			date: getFromArr(row, dateIdx),
+		};
+	};
+};
+
+const extractChartOneSix = (pageTables) => {
+	return chartToArrayBuilder({
+		pageTables,
+		rowAbstractFactory: rowAbstractFactoryChart1_6,
+		startOfChartPred: (cell) => cell.includes('יחס בין התכנית לבין תכניות מאושרות קודמות') && cell.includes('1.6'),
+		offsetOfRowWithDataInChart: 1,
+		chartDonePredicate: (row) => row.some(cell => cell.includes('מסמכי התכנית') && cell.includes('1.7')),
+		getHeaderRowIndex: (page, searchFrom) => page.slice(searchFrom).findIndex(row => row.some(cell => cell.includes('סוג יחס'))) + searchFrom,
+		rowTrimmer: (row) => row.map((cell) => cell.replace(/\n/g, ' ').replace(/ {2}/g, ' ').trim()),
+		identifier: '1.6'
+	});
+};
+
+module.exports = {
+	extractChartOneSix
+};

server/api/lib/mavat/planInstructions/parser/chartSevenOne.js

@@ -0,0 +1,72 @@
+const { chartToArrayBuilder } = require('./chartToArrayBuilder');
+const { getFromArr } = require('./parsersUtils');
+const log = require('../../../log');
+
+// this function look for the correct columns for the given headers, and returns a factory embedded with these findings
+const rowAbstractFactoryChart7_1 = (firstPageOfTable, headersStartIndex) => {
+	// clean the headers
+	firstPageOfTable = firstPageOfTable.map(row => row.map(cell => cell.replace(/\n/g, ' ')
+		.replace(/ {2}/g, ' ')
+		.trim()));
+
+	if (headersStartIndex === -1) {
+		// it's an error
+		log.error('didn\'t find headers for some chart 3.1_without_change');
+		return () => {};
+	}
+
+	const header = firstPageOfTable[headersStartIndex];
+
+	const phaseIndex = header.findIndex(title => title.includes('מספר שלב'));
+	const phaseDescription = header.findIndex(title => title.includes('תאור שלב'));
+	const conditioning = header.findIndex(title => title.includes('התנייה'));
+
+	return (row) => {
+		return {
+			phase: getFromArr(row, phaseIndex),
+			phase_description: getFromArr(row, phaseDescription),
+			conditioning: getFromArr(row, conditioning),
+		};
+	};
+};
+
+
+const extractChartSevenOne = (pageTables) => {
+	const rowsFromBuilder = chartToArrayBuilder({
+		pageTables,
+		rowAbstractFactory: rowAbstractFactoryChart7_1,
+		startOfChartPred: (cell) => cell.includes('7.1') && cell.includes('שלבי ביצוע'),
+		offsetOfRowWithDataInChart: 1,
+		chartDonePredicate: (row) => row.some(cell => cell.includes('7.2') && cell.includes('מימוש התכנית')),
+		getHeaderRowIndex: (page, searchFrom) => page.slice(searchFrom).findIndex(row => row.some(cell => cell.includes('מספר שלב')) &&
+                row.some(cell => cell.includes('תאור שלב')) &&
+                row.some(cell => cell.includes('התנייה'))) + searchFrom,
+		rowTrimmer: (row) => row.map((cell) => cell.replace(/\n/g, ' ').replace(/ {2}/g, ' ').trim()),
+		identifier: '7.1'
+	});
+
+	const newRows = [];
+
+	for (let i = 0; i < rowsFromBuilder.length; i++) {
+		const currRow = rowsFromBuilder[i];
+
+		if (currRow.phase === '' && newRows.length > 0) {
+			const lastFromNewRows = newRows[newRows.length - 1];
+
+			lastFromNewRows.phase_description += ' ' + currRow.phase_description;
+			lastFromNewRows.conditioning += ' ' + currRow.conditioning;
+		}
+
+		else {
+			newRows.push(currRow);
+		}
+
+	}
+
+	return newRows;
+
+};
+
+module.exports = {
+	extractChartSevenOne
+};