Merge remote-tracking branch 'origin/main' into mark/enable-streaming

Author: annapo23

README.md

@@ -154,7 +154,7 @@ Request #3 => page_2_markdown + page_3_image
 
 ### Installation
 
-- Install **poppler-utils** on the system, it should be available in path variable
+- Install **poppler** on the system, it should be available in path variable. See the [pdf2image documentation](https://pdf2image.readthedocs.io/en/latest/installation.html) for instructions by platform.
 - Install py-zerox:
 
 ```sh

node-zerox/src/handleWarnings.ts

@@ -0,0 +1,15 @@
+// Tesseract relies on node-fetch v2, which has a deprecated version of punycode
+// Suppress the warning for now. Check in when teseract updates to node-fetch v3
+// https://github.com/naptha/tesseract.js/issues/876
+if (process.stderr.write === process.stderr.constructor.prototype.write) {
+  const stdErrWrite = process.stderr.write;
+  process.stderr.write = function (chunk: any, ...args: any[]) {
+    const str = Buffer.isBuffer(chunk) ? chunk.toString() : chunk;
+
+    // Filter out the punycode deprecation warning
+    if (str.includes("punycode")) {
+      return true;
+    }
+    return stdErrWrite.apply(process.stderr, [chunk]);
+  };
+}

node-zerox/src/index.ts

@@ -4,6 +4,7 @@ import path from "path";
 import pLimit, { Limit } from "p-limit";
 import Tesseract from "tesseract.js";
 
+import "./handleWarnings";
 import {
   addWorkersToTesseractScheduler,
   cleanupImage,

node-zerox/src/utils/model.ts

@@ -2,7 +2,7 @@ import { LLMParams } from "../types";
 
 const defaultLLMParams: LLMParams = {
   frequencyPenalty: 0, // OpenAI defaults to 0
-  maxTokens: 2000,
+  maxTokens: 4000,
   presencePenalty: 0, // OpenAI defaults to 0
   temperature: 0,
   topP: 1, // OpenAI defaults to 1

py_zerox/pyzerox/core/zerox.py

@@ -7,6 +7,7 @@
 import aiofiles
 import aiofiles.os as async_os
 import asyncio
+from ..constants import PDFConversionDefaultOptions
 
 # Package Imports
 from ..processor import (
@@ -26,6 +27,8 @@ async def zerox(
     cleanup: bool = True,
     concurrency: int = 10,
     file_path: Optional[str] = "",
+    image_density: int = PDFConversionDefaultOptions.DPI,
+    image_height: tuple[Optional[int], int] = PDFConversionDefaultOptions.SIZE,
     maintain_format: bool = False,
     model: str = "gpt-4o-mini",
     output_dir: Optional[str] = None,
@@ -130,7 +133,7 @@ async def zerox(
                                                  **subset_pdf_create_kwargs)
 
         # Convert the file to a series of images, below function returns a list of image paths in page order
-        images = await convert_pdf_to_images(local_path=local_path, temp_dir=temp_directory)
+        images = await convert_pdf_to_images(image_density=image_density, image_height=image_height, local_path=local_path, temp_dir=temp_directory)
 
         if maintain_format:
             for image in images:

py_zerox/pyzerox/processor/pdf.py

@@ -11,14 +11,14 @@
 from ..models import litellmmodel
 
 
-async def convert_pdf_to_images(local_path: str, temp_dir: str) -> List[str]:
+async def convert_pdf_to_images(image_density: int, image_height: tuple[Optional[int], int], local_path: str, temp_dir: str) -> List[str]:
     """Converts a PDF file to a series of images in the temp_dir. Returns a list of image paths in page order."""
     options = {
         "pdf_path": local_path,
         "output_folder": temp_dir,
-        "dpi": PDFConversionDefaultOptions.DPI,
+        "dpi": image_density,
         "fmt": PDFConversionDefaultOptions.FORMAT,
-        "size": PDFConversionDefaultOptions.SIZE,
+        "size": image_height,
         "thread_count": PDFConversionDefaultOptions.THREAD_COUNT,
         "use_pdftocairo": PDFConversionDefaultOptions.USE_PDFTOCAIRO,
         "paths_only": True,