From 2f3e6e4f6e5dc41560d4fe4cd44bb919c0c0dcbf Mon Sep 17 00:00:00 2001 From: Jon Parise Date: Mon, 30 Mar 2026 16:09:20 -0400 Subject: [PATCH] Replace requests with urllib Python's urllib package provides all of the HTTP things that our client needs other than multipart encoding, which we can implement ourselves. This change also rewrites the client tests to made more idiomatic use of pytest. --- instaparser/client.py | 270 +++++--- pyproject.toml | 5 +- tests/README.md | 143 ----- tests/conftest.py | 164 ----- tests/test_client.py | 1217 +++++++++++++------------------------ tests/test_integration.py | 327 ---------- 6 files changed, 608 insertions(+), 1518 deletions(-) delete mode 100644 tests/README.md delete mode 100644 tests/conftest.py delete mode 100644 tests/test_integration.py diff --git a/instaparser/client.py b/instaparser/client.py index b356cc4..9d6dbcf 100644 --- a/instaparser/client.py +++ b/instaparser/client.py @@ -3,11 +3,13 @@ """ import json +import uuid from collections.abc import Callable -from typing import Any, BinaryIO -from urllib.parse import urljoin - -import requests +from http.client import HTTPResponse +from typing import Any, BinaryIO, NoReturn +from urllib.error import HTTPError +from urllib.parse import urlencode, urljoin +from urllib.request import Request, urlopen from .article import Article from .exceptions import ( @@ -20,6 +22,79 @@ from .summary import Summary +def _encode_multipart_formdata( + fields: dict[str, str], + files: dict[str, BinaryIO | bytes], +) -> tuple[bytes, str]: + """ + Encode fields and files as multipart/form-data. + + Args: + fields: Dictionary of form field name/value pairs + files: Dictionary of file field name to file-like object or bytes + + Returns: + Tuple of (encoded body bytes, content-type header value) + """ + boundary = uuid.uuid4().hex + lines: list[bytes] = [] + + for name, value in fields.items(): + lines.append(f"--{boundary}\r\n".encode()) + lines.append(f'Content-Disposition: form-data; name="{name}"\r\n'.encode()) + lines.append(b"\r\n") + lines.append(f"{value}\r\n".encode()) + + for name, file_data in files.items(): + if hasattr(file_data, "read"): + data = file_data.read() + else: + data = file_data + lines.append(f"--{boundary}\r\n".encode()) + lines.append(f'Content-Disposition: form-data; name="{name}"; filename="upload"\r\n'.encode()) + lines.append(b"Content-Type: application/octet-stream\r\n") + lines.append(b"\r\n") + lines.append(data) + lines.append(b"\r\n") + + lines.append(f"--{boundary}--\r\n".encode()) + + body = b"".join(lines) + content_type = f"multipart/form-data; boundary={boundary}" + return body, content_type + + +def _map_http_error(e: HTTPError) -> NoReturn: + """ + Translate an HTTPError into the appropriate Instaparser domain exception. + + Reads the response body from the HTTPError, extracts the ``reason`` + field from the JSON payload (if present), and raises the matching + Instaparser exception. + """ + status_code = e.code + + body = e.read().decode("utf-8") + error_message = f"API request failed with status {status_code}" + try: + error_data = json.loads(body) + if isinstance(error_data, dict) and "reason" in error_data: + error_message = error_data["reason"] + except (ValueError, json.JSONDecodeError): + error_message = body or error_message + + errors: dict[int, tuple[type[InstaparserAPIError], str]] = { + 400: (InstaparserValidationError, "Invalid request"), + 401: (InstaparserAuthenticationError, "Invalid API key"), + 403: (InstaparserAPIError, "Account suspended"), + 409: (InstaparserAPIError, "Exceeded monthly API calls"), + 429: (InstaparserRateLimitError, "Rate limit exceeded"), + } + + exc_cls, default_msg = errors.get(status_code, (InstaparserAPIError, "")) + raise exc_cls(error_message or default_msg, status_code=status_code, response=e) + + class InstaparserClient: """ Client for interacting with the Instaparser API. @@ -42,69 +117,74 @@ def __init__(self, api_key: str, base_url: str | None = None): """ self.api_key = api_key self.base_url = base_url or self.BASE_URL - self.session = requests.Session() - self.session.headers.update( - { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - } - ) + self.headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } - def _handle_response(self, response: requests.Response) -> dict[str, Any]: + def __repr__(self) -> str: + return f"" + + def _request( + self, + method: str, + path: str, + *, + json_data: dict | None = None, + params: dict | None = None, + multipart_fields: dict[str, str] | None = None, + multipart_files: dict[str, BinaryIO | bytes] | None = None, + ) -> HTTPResponse: """ - Handle API response and raise appropriate exceptions. + Make an HTTP request using urllib. Args: - response: The HTTP response object + method: HTTP method (GET, POST) + path: API endpoint path (e.g. "/api/1/article") + json_data: JSON body payload + params: Query parameters + multipart_fields: Form fields for multipart upload + multipart_files: Files for multipart upload Returns: - Parsed JSON response data + HTTPResponse on success Raises: - InstaparserAuthenticationError: For 401 errors - InstaparserRateLimitError: For 429 errors - InstaparserValidationError: For 400 errors - InstaparserAPIError: For other API errors + HTTPError: On non-2xx status codes (callers convert via _map_http_error) """ - status_code = response.status_code + url = urljoin(self.base_url, path) + if params: + url = f"{url}?{urlencode(params)}" - if status_code == 200: - try: - parsed = response.json() - if isinstance(parsed, dict): - return parsed - except ValueError: - # Some endpoints might return non-JSON - return {"raw": response.text} - - error_message = f"API request failed with status {status_code}" + data = None + headers = self.headers + + if multipart_fields or multipart_files: + data, content_type = _encode_multipart_formdata(multipart_fields or {}, multipart_files or {}) + headers = headers.copy() + headers["Content-Type"] = content_type + elif json_data is not None: + data = json.dumps(json_data).encode("utf-8") + + req = Request(url, data=data, headers=headers, method=method) + response: HTTPResponse = urlopen(req) + return response + + def _read_json(self, response: HTTPResponse) -> dict[str, Any]: + """ + Read and parse a successful JSON response. + + Returns: + Parsed JSON dict, or ``{"raw": text}`` if the body isn't valid JSON. + """ + body = response.read().decode("utf-8") try: - error_data = response.json() - if isinstance(error_data, dict) and "reason" in error_data: - error_message = error_data["reason"] - except ValueError: - error_message = response.text or error_message - - if status_code == 401: - raise InstaparserAuthenticationError( - error_message or "Invalid API key", status_code=status_code, response=response - ) - elif status_code == 403: - raise InstaparserAPIError(error_message or "Account suspended", status_code=status_code, response=response) - elif status_code == 409: - raise InstaparserAPIError( - error_message or "Exceeded monthly API calls", status_code=status_code, response=response - ) - elif status_code == 429: - raise InstaparserRateLimitError( - error_message or "Rate limit exceeded", status_code=status_code, response=response - ) - elif status_code == 400: - raise InstaparserValidationError( - error_message or "Invalid request", status_code=status_code, response=response - ) - else: - raise InstaparserAPIError(error_message, status_code=status_code, response=response) + parsed = json.loads(body) + if isinstance(parsed, dict): + return parsed + except (ValueError, json.JSONDecodeError): + pass + return {"raw": body} def Article(self, url: str, content: str | None = None, output: str = "html", use_cache: bool = True) -> Article: """ @@ -127,20 +207,21 @@ def Article(self, url: str, content: str | None = None, output: str = "html", us if output not in ("html", "text", "markdown"): raise InstaparserValidationError("output must be 'html', 'text', or 'markdown'") - endpoint = urljoin(self.base_url, "/api/1/article") - payload = { + payload: dict[str, Any] = { "url": url, "output": output, } - # API expects string 'false' to disable cache, not boolean False if not use_cache: payload["use_cache"] = "false" - if content is not None: payload["content"] = content - response = self.session.post(endpoint, json=payload) - data = self._handle_response(response) + try: + response = self._request("POST", "/api/1/article", json_data=payload) + except HTTPError as e: + _map_http_error(e) + + data = self._read_json(response) return Article( url=data.get("url"), title=data.get("title"), @@ -188,30 +269,27 @@ def Summary( ... print(f"Received: {line}") >>> summary = client.Summary(url="https://example.com/article", stream_callback=on_line) """ - endpoint = urljoin(self.base_url, "/api/1/summary") - payload = { + payload: dict[str, Any] = { "url": url, "stream": stream_callback is not None, } - # API expects string 'false' to disable cache, not boolean False if not use_cache: payload["use_cache"] = "false" - if content is not None: payload["content"] = content - if stream_callback is not None: - # Handle streaming response - response = self.session.post(endpoint, json=payload, stream=True) - if response.status_code != 200: - self._handle_response(response) + try: + response = self._request("POST", "/api/1/summary", json_data=payload) + except HTTPError as e: + _map_http_error(e) - key_sentences = [] + if stream_callback is not None: + key_sentences: list[str] = [] overview = "" - for line in response.iter_lines(): + for raw_line in response: + line = raw_line.strip(b"\r\n") if line: line_str = line.decode("utf-8") - # Call the callback for each line stream_callback(line_str) if line_str.startswith("key_sentences:"): @@ -225,12 +303,15 @@ def Summary( return Summary(key_sentences=key_sentences, overview=overview) else: - response = self.session.post(endpoint, json=payload) - data = self._handle_response(response) + data = self._read_json(response) return Summary(key_sentences=data.get("key_sentences", []), overview=data.get("overview", "")) def PDF( - self, url: str | None = None, file: BinaryIO | bytes | None = None, output: str = "html", use_cache: bool = True + self, + url: str | None = None, + file: BinaryIO | bytes | None = None, + output: str = "html", + use_cache: bool = True, ) -> PDF: """ Parse a PDF from a URL or file. @@ -255,37 +336,38 @@ def PDF( if output not in ("html", "text", "markdown"): raise InstaparserValidationError("output must be 'html', 'text', or 'markdown'") - endpoint = urljoin(self.base_url, "/api/1/pdf") - if file is not None: - # POST request with file upload - files = {"file": file} - data = { - "output": output, - } - # API expects string 'false' to disable cache, not boolean False + fields = {"output": output} if not use_cache: - data["use_cache"] = "false" + fields["use_cache"] = "false" if url: - data["url"] = url + fields["url"] = url - # Remove Content-Type header for multipart/form-data - headers = {k: v for k, v in self.session.headers.items() if k != "Content-Type"} - response = self.session.post(endpoint, files=files, data=data, headers=headers) + try: + response = self._request( + "POST", + "/api/1/pdf", + multipart_fields=fields, + multipart_files={"file": file}, + ) + except HTTPError as e: + _map_http_error(e) elif url: - # GET request with URL params = { "url": url, "output": output, } - # API expects string 'false' to disable cache, not boolean False if not use_cache: params["use_cache"] = "false" - response = self.session.get(endpoint, params=params) + + try: + response = self._request("GET", "/api/1/pdf", params=params) + except HTTPError as e: + _map_http_error(e) else: raise InstaparserValidationError("Either 'url' or 'file' must be provided") - result = self._handle_response(response) + result = self._read_json(response) return PDF( url=result.get("url"), title=result.get("title"), diff --git a/pyproject.toml b/pyproject.toml index 7883f82..1773397 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,9 +28,7 @@ classifiers = [ "Topic :: Text Processing :: Markup :: HTML", "Typing :: Typed", ] -dependencies = [ - "requests>=2.25.0", -] +dependencies = [] [dependency-groups] dev = [ @@ -38,7 +36,6 @@ dev = [ "pytest-cov>=4.0", "ruff>=0.1.0", "mypy>=1.0", - "types-requests", ] build = [ "build>=1.4.0", diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 68c95cc..0000000 --- a/tests/README.md +++ /dev/null @@ -1,143 +0,0 @@ -# Instaparser Python Testing - -This directory contains comprehensive tests for the Instaparser Python Library. - -## Test Files - -### `test_client.py` (67 tests) -Tests for the `InstaparserClient` class: -- Client initialization (with/without custom base URL) -- Response handling (success, errors, edge cases) -- Article API method (URL, content, output formats, caching) -- Summary API method (URL, content, streaming, caching) -- PDF API method (URL, file uploads, output formats, caching) -- Network errors and timeouts -- Edge cases (malformed responses, empty responses, large content) -- URL encoding and special characters -- Session header management - -### `test_article.py` (25 tests) -Tests for the `Article` class: -- Initialization with various data configurations -- HTML vs text output handling -- Property access (title, author, words, images, videos, etc.) -- Edge cases (None values, empty strings, Unicode, special characters) -- Very long content handling -- Date handling (string, integer, None) -- Complex HTML structures -- Body fallback logic (HTML → text → None) - -### `test_summary.py` (15 tests) -Tests for the `Summary` class: -- Initialization with key_sentences and overview -- Empty and whitespace-only content -- Very long content -- Unicode and special characters -- String representation methods (`__str__`, `__repr__`) - -### `test_pdf.py` (20 tests) -Tests for the `PDF` class: -- Inheritance from Article -- PDF-specific behavior (is_rtl=False, videos=[]) -- Override behavior (ensures is_rtl and videos are always correct) -- Unicode and special characters -- Very long content -- Text output format -- All Article properties and methods - -### `test_exceptions.py` (13 tests) -Tests for exception classes: -- Exception inheritance hierarchy -- Base `InstaparserError` -- `InstaparserAPIError` with status codes and responses -- Specific exceptions (Authentication, RateLimit, Validation) -- Exception raising and catching - -### `test_init.py` (10 tests) -Tests for package-level exports: -- All classes and exceptions are exported -- `__all__` list matches exports -- Version information -- Import functionality -- Exception inheritance chain verification - -### `test_integration.py` (12 tests) -Integration-style tests: -- Complete workflows (Article, Summary, PDF) -- Error handling across the library -- Multiple client instances -- Client reuse for multiple requests -- Output format variations - -### `conftest.py` -Shared pytest fixtures: -- `api_key`: Sample API key for testing -- `base_url`: Base URL for testing -- `client`: InstaparserClient instance -- `mock_article_response`: Mock article API response -- `mock_article_text_response`: Mock article API response (text format) -- `mock_summary_response`: Mock summary API response -- `mock_pdf_response`: Mock PDF API response -- `mock_response`: Factory for creating mock HTTP responses -- `mock_session`: Mock requests.Session - -## Running Tests - -### Run all tests: -```bash -pytest tests/ -``` - -### Run specific test file: -```bash -pytest tests/test_client.py -``` - -### Run with coverage: -```bash -pytest tests/ --cov=instaparser --cov-report=html -``` - -### Run with verbose output: -```bash -pytest tests/ -v -``` - -### Run specific test: -```bash -pytest tests/test_client.py::TestInstaparserClientArticle::test_article_from_url -``` - -## Test Coverage - -The test suite covers: -- ✅ All public API methods -- ✅ All classes and their properties -- ✅ Error handling and exceptions -- ✅ Edge cases (empty data, None values, large content) -- ✅ Unicode and special characters -- ✅ Network errors and timeouts -- ✅ Malformed responses -- ✅ Streaming functionality -- ✅ File uploads -- ✅ URL encoding -- ✅ Package exports -- ✅ Integration workflows - -## Test Statistics - -- **Total test files**: 8 -- **Total test functions**: ~145 -- **Test coverage**: Comprehensive across all components - -## Dependencies - -Tests require: -- `pytest>=6.0` -- `pytest-cov>=2.0` (optional, for coverage) -- `requests>=2.25.0` (for mocking) - -Install test dependencies: -```bash -pip install -e ".[dev]" -``` diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 37d1508..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -Shared fixtures and utilities for tests. -""" - -import json -from unittest.mock import MagicMock, Mock - -import pytest -import requests - -from instaparser import InstaparserClient - - -@pytest.fixture -def api_key(): - """Sample API key for testing.""" - return "test-api-key-12345" - - -@pytest.fixture -def base_url(): - """Base URL for testing.""" - return "https://api.test.instaparser.com" - - -@pytest.fixture -def client(api_key, base_url): - """Create an InstaparserClient instance for testing.""" - return InstaparserClient(api_key=api_key, base_url=base_url) - - -@pytest.fixture -def mock_article_response(): - """Mock article API response.""" - return { - "url": "https://example.com/article", - "title": "Test Article Title", - "site_name": "Example Site", - "author": "John Doe", - "date": 1609459200, - "description": "This is a test article description", - "thumbnail": "https://example.com/thumb.jpg", - "words": 500, - "is_rtl": False, - "images": ["https://example.com/image1.jpg"], - "videos": [], - "html": "

This is the article body in HTML format.

", - } - - -@pytest.fixture -def mock_article_text_response(): - """Mock article API response with text output.""" - return { - "url": "https://example.com/article", - "title": "Test Article Title", - "site_name": "Example Site", - "author": "John Doe", - "date": 1609459200, - "description": "This is a test article description", - "thumbnail": "https://example.com/thumb.jpg", - "words": 500, - "is_rtl": False, - "images": ["https://example.com/image1.jpg"], - "videos": [], - "text": "This is the article body in plain text format.", - } - - -@pytest.fixture -def mock_article_markdown_response(): - """Mock article API response with markdown output.""" - return { - "url": "https://example.com/article", - "title": "Test Article Title", - "site_name": "Example Site", - "author": "John Doe", - "date": 1609459200, - "description": "This is a test article description", - "thumbnail": "https://example.com/thumb.jpg", - "words": 500, - "is_rtl": False, - "images": ["https://example.com/image1.jpg"], - "videos": [], - "markdown": "# Test Article Title\n\nThis is the article body in **markdown** format.", - } - - -@pytest.fixture -def mock_summary_response(): - """Mock summary API response.""" - return { - "key_sentences": [ - "This is the first key sentence.", - "This is the second key sentence.", - "This is the third key sentence.", - ], - "overview": "This is a comprehensive overview of the article content.", - } - - -@pytest.fixture -def mock_pdf_response(): - """Mock PDF API response.""" - return { - "url": "https://example.com/document.pdf", - "title": "Test PDF Document", - "site_name": "Example Site", - "author": "Jane Smith", - "date": 1609459200, - "description": "This is a test PDF document", - "thumbnail": None, - "words": 1000, - "is_rtl": False, - "images": [], - "videos": [], - "html": "

This is the PDF content in HTML format.

", - } - - -@pytest.fixture -def mock_response(): - """Create a mock requests.Response object.""" - - def _create_response(status_code=200, json_data=None, text=None, headers=None): - response = Mock(spec=requests.Response) - response.status_code = status_code - response.headers = headers or {"Content-Type": "application/json"} - - if json_data is not None: - response.json.return_value = json_data - response.text = json.dumps(json_data) - elif text is not None: - response.text = text - response.json.side_effect = ValueError("Not JSON") - else: - response.json.return_value = {} - response.text = "" - - return response - - return _create_response - - -@pytest.fixture -def mock_session(mocker): - """Mock the requests.Session used by InstaparserClient.""" - session = MagicMock(spec=requests.Session) - session.headers = {} - session.post = MagicMock() - session.get = MagicMock() - - # Default successful response - default_response = Mock(spec=requests.Response) - default_response.status_code = 200 - default_response.json.return_value = {} - default_response.text = "{}" - default_response.headers = {} - session.post.return_value = default_response - session.get.return_value = default_response - - # Patch requests.Session to return our mock - mocker.patch("requests.Session", return_value=session) - return session diff --git a/tests/test_client.py b/tests/test_client.py index 1cf1a94..6dd1307 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,13 +1,13 @@ -""" -Tests for InstaparserClient class. -""" - -from unittest.mock import Mock, patch +import io +import json +from unittest.mock import Mock +from urllib.error import HTTPError, URLError import pytest -import requests from instaparser import InstaparserClient +from instaparser.article import Article +from instaparser.client import _encode_multipart_formdata from instaparser.exceptions import ( InstaparserAPIError, InstaparserAuthenticationError, @@ -15,803 +15,448 @@ InstaparserValidationError, ) - -class TestInstaparserClientInitialization: - """Tests for InstaparserClient initialization.""" - - def test_client_initialization_with_api_key(self, api_key): - """Test client initialization with API key.""" - client = InstaparserClient(api_key=api_key) - - assert client.api_key == api_key - assert client.base_url == InstaparserClient.BASE_URL - assert client.session is not None - assert client.session.headers["Authorization"] == f"Bearer {api_key}" - assert client.session.headers["Content-Type"] == "application/json" - - def test_client_initialization_with_custom_base_url(self, api_key, base_url): - """Test client initialization with custom base URL.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - assert client.api_key == api_key - assert client.base_url == base_url - - def test_client_uses_default_base_url(self, api_key): - """Test that client uses default base URL when not provided.""" - client = InstaparserClient(api_key=api_key) - - assert client.base_url == "https://instaparser.com" - - -class TestInstaparserClientHandleResponse: - """Tests for _handle_response method.""" - - def test_handle_response_success(self, client, mock_response): - """Test handling successful response.""" - json_data = {"key": "value"} - response = mock_response(status_code=200, json_data=json_data) - - result = client._handle_response(response) - - assert result == json_data - - def test_handle_response_non_json_success(self, client, mock_response): - """Test handling successful non-JSON response.""" - response = mock_response(status_code=200, text="Plain text response") - - result = client._handle_response(response) - - assert result == {"raw": "Plain text response"} - - def test_handle_response_401_authentication_error(self, client, mock_response): - """Test handling 401 authentication error.""" - error_data = {"reason": "Invalid API key"} - response = mock_response(status_code=401, json_data=error_data) - - with pytest.raises(InstaparserAuthenticationError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 401 - assert "Invalid API key" in str(exc_info.value) - assert exc_info.value.response == response - - def test_handle_response_403_account_suspended(self, client, mock_response): - """Test handling 403 account suspended error.""" - error_data = {"reason": "Account suspended"} - response = mock_response(status_code=403, json_data=error_data) - - with pytest.raises(InstaparserAPIError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 403 - assert "Account suspended" in str(exc_info.value) - - def test_handle_response_409_monthly_limit(self, client, mock_response): - """Test handling 409 monthly limit exceeded error.""" - error_data = {"reason": "Monthly API calls exceeded"} - response = mock_response(status_code=409, json_data=error_data) - - with pytest.raises(InstaparserAPIError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 409 - assert "Monthly API calls exceeded" in str(exc_info.value) - - def test_handle_response_429_rate_limit_error(self, client, mock_response): - """Test handling 429 rate limit error.""" - error_data = {"reason": "Rate limit exceeded"} - response = mock_response(status_code=429, json_data=error_data) - - with pytest.raises(InstaparserRateLimitError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 429 - assert "Rate limit exceeded" in str(exc_info.value) - - def test_handle_response_400_validation_error(self, client, mock_response): - """Test handling 400 validation error.""" - error_data = {"reason": "Invalid request parameters"} - response = mock_response(status_code=400, json_data=error_data) - - with pytest.raises(InstaparserValidationError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 400 - assert "Invalid request parameters" in str(exc_info.value) - - def test_handle_response_500_generic_error(self, client, mock_response): - """Test handling 500 generic API error.""" - error_data = {"reason": "Internal server error"} - response = mock_response(status_code=500, json_data=error_data) - - with pytest.raises(InstaparserAPIError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 500 - assert "Internal server error" in str(exc_info.value) - - def test_handle_response_error_without_reason(self, client, mock_response): - """Test handling error response without reason field.""" - response = mock_response(status_code=500, json_data={}) - - with pytest.raises(InstaparserAPIError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 500 - assert "API request failed" in str(exc_info.value) - - def test_handle_response_error_with_text_only(self, client, mock_response): - """Test handling error response with text only (no JSON).""" - response = mock_response(status_code=500, text="Error message") - +API_KEY = "test-api-key-12345" +BASE_URL = "https://api.test.instaparser.com" + +ARTICLE_DATA = { + "url": "https://example.com/article", + "title": "Test Article Title", + "site_name": "Example Site", + "author": "John Doe", + "date": 1609459200, + "description": "This is a test article description", + "thumbnail": "https://example.com/thumb.jpg", + "words": 500, + "is_rtl": False, + "images": ["https://example.com/image1.jpg"], + "videos": [], + "html": "

This is the article body in HTML format.

", +} + +SUMMARY_DATA = { + "key_sentences": [ + "This is the first key sentence.", + "This is the second key sentence.", + "This is the third key sentence.", + ], + "overview": "This is a comprehensive overview of the article content.", +} + +PDF_DATA = { + "url": "https://example.com/document.pdf", + "title": "Test PDF Document", + "site_name": "Example Site", + "author": "Jane Smith", + "date": 1609459200, + "description": "This is a test PDF document", + "thumbnail": None, + "words": 1000, + "is_rtl": False, + "images": [], + "videos": [], + "html": "

This is the PDF content in HTML format.

", +} + +ERROR_CODES = [ + (400, InstaparserValidationError), + (401, InstaparserAuthenticationError), + (403, InstaparserAPIError), + (409, InstaparserAPIError), + (429, InstaparserRateLimitError), + (500, InstaparserAPIError), +] + + +@pytest.fixture +def client(): + return InstaparserClient(api_key=API_KEY, base_url=BASE_URL) + + +@pytest.fixture +def mock_request(client, monkeypatch): + mock = Mock() + monkeypatch.setattr(client, "_request", mock) + return mock + + +def make_response(*, json_data=None, text=None): + """Mock a successful HTTPResponse.""" + response = Mock() + response.status = 200 + if json_data is not None: + response.read.return_value = json.dumps(json_data).encode() + elif text is not None: + response.read.return_value = text.encode() + else: + response.read.return_value = b"{}" + return response + + +def make_streaming_response(lines): + """Mock a streaming HTTPResponse that yields *lines*.""" + response = Mock() + response.status = 200 + response.__iter__ = Mock(return_value=iter(lines)) + return response + + +def make_error(status_code, *, json_data=None, text=None): + """Create an HTTPError with a readable body.""" + if json_data is not None: + body = json.dumps(json_data).encode() + elif text is not None: + body = text.encode() + else: + body = b"{}" + return HTTPError("https://api.example.com", status_code, "", None, io.BytesIO(body)) + + +class TestClientInit: + def test_base_url(self): + client = InstaparserClient(api_key=API_KEY, base_url=BASE_URL) + assert client.base_url == BASE_URL + + def test_headers(self): + client = InstaparserClient(api_key=API_KEY) + assert client.headers["Authorization"] == f"Bearer {API_KEY}" + assert client.headers["Content-Type"] == "application/json" + + +class TestReadJson: + def test_valid_json(self, client): + assert client._read_json(make_response(json_data={"key": "value"})) == {"key": "value"} + + def test_non_json(self, client): + assert client._read_json(make_response(text="Plain text")) == {"raw": "Plain text"} + + +class TestArticle: + def test_basic_parse(self, client, mock_request): + mock_request.return_value = make_response(json_data=ARTICLE_DATA) + + article = client.Article(url="https://example.com/article") + + assert article.title == "Test Article Title" + assert article.url == "https://example.com/article" + assert article.body == "

This is the article body in HTML format.

" + + _method, _path = mock_request.call_args[0] + payload = mock_request.call_args[1]["json_data"] + assert _method == "POST" + assert _path == "/api/1/article" + assert payload["url"] == "https://example.com/article" + assert payload["output"] == "html" + assert "use_cache" not in payload + + def test_text_output(self, client, mock_request): + data = {**ARTICLE_DATA, "html": None, "text": "Plain text body."} + mock_request.return_value = make_response(json_data=data) + article = client.Article(url="u", output="text") + + assert article.body == "Plain text body." + assert mock_request.call_args[1]["json_data"]["output"] == "text" + + def test_markdown_output(self, client, mock_request): + mock_request.return_value = make_response(json_data={"url": "u", "markdown": "# MD"}) + article = client.Article(url="u", output="markdown") + + assert article.markdown == "# MD" + assert article.body == "# MD" + + def test_with_content(self, client, mock_request): + mock_request.return_value = make_response(json_data=ARTICLE_DATA) + client.Article(url="u", content="hi") + assert mock_request.call_args[1]["json_data"]["content"] == "hi" + + def test_use_cache_false(self, client, mock_request): + mock_request.return_value = make_response(json_data=ARTICLE_DATA) + client.Article(url="u", use_cache=False) + assert mock_request.call_args[1]["json_data"]["use_cache"] == "false" + + def test_invalid_output(self, client): + with pytest.raises(InstaparserValidationError, match="output must be"): + client.Article(url="u", output="invalid") + + def test_malformed_json_response(self, client, mock_request): + mock_request.return_value = make_response(text="Not valid JSON {") + article = client.Article(url="u") + assert isinstance(article, Article) + + +class TestSummary: + def test_basic_summary(self, client, mock_request): + mock_request.return_value = make_response(json_data=SUMMARY_DATA) + + summary = client.Summary(url="https://example.com/article") + + assert len(summary.key_sentences) == 3 + assert summary.overview == "This is a comprehensive overview of the article content." + assert mock_request.call_args[1]["json_data"]["stream"] is False + + def test_with_content(self, client, mock_request): + mock_request.return_value = make_response(json_data=SUMMARY_DATA) + client.Summary(url="u", content="hi") + assert mock_request.call_args[1]["json_data"]["content"] == "hi" + + def test_use_cache_false(self, client, mock_request): + mock_request.return_value = make_response(json_data=SUMMARY_DATA) + client.Summary(url="u", use_cache=False) + assert mock_request.call_args[1]["json_data"]["use_cache"] == "false" + + def test_empty_response(self, client, mock_request): + mock_request.return_value = make_response(json_data={}) + summary = client.Summary(url="u") + assert summary.key_sentences == [] + assert summary.overview == "" + + def test_streaming_callback(self, client, mock_request): + mock_request.return_value = make_streaming_response( + [ + b'key_sentences: ["Sentence 1", "Sentence 2"]\r\n', + b"delta: This is\r\n", + b"delta: a streaming\r\n", + b"delta: overview.\r\n", + ] + ) + + received = [] + summary = client.Summary(url="u", stream_callback=received.append) + + assert len(received) == 4 + assert received[0] == 'key_sentences: ["Sentence 1", "Sentence 2"]' + assert summary.key_sentences == ["Sentence 1", "Sentence 2"] + assert summary.overview == "This is a streaming overview." + assert mock_request.call_args[1]["json_data"]["stream"] is True + + def test_streaming_empty_lines_filtered(self, client, mock_request): + mock_request.return_value = make_streaming_response( + [ + b"key_sentences: []\r\n", + b"\r\n", + b"delta: Content\r\n", + b"\r\n", + ] + ) + summary = client.Summary(url="u", stream_callback=lambda _: None) + assert summary.overview == "Content" + + def test_streaming_malformed_key_sentences(self, client, mock_request): + mock_request.return_value = make_streaming_response( + [ + b"key_sentences: not valid json\r\n", + b"delta: Content\r\n", + ] + ) with pytest.raises(InstaparserAPIError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 500 - assert "Error message" in str(exc_info.value) - + client.Summary(url="u", stream_callback=lambda _: None) + assert exc_info.value.status_code == 412 + assert "Unable to generate key sentences" in str(exc_info.value) -class TestInstaparserClientArticle: - """Tests for Article method.""" + def test_streaming_error_response(self, client, mock_request): + mock_request.side_effect = make_error(401, json_data={"reason": "Invalid API key"}) + with pytest.raises(InstaparserAuthenticationError): + client.Summary(url="u", stream_callback=lambda _: None) - def test_article_from_url(self, client, mock_article_response, mock_response): - """Test parsing article from URL.""" - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_article_response) - mock_post.return_value = response - article = client.Article(url="https://example.com/article") +class TestPDF: + def test_from_url(self, client, mock_request): + mock_request.return_value = make_response(json_data=PDF_DATA) - assert article.title == "Test Article Title" - assert article.url == "https://example.com/article" - assert article.body == "

This is the article body in HTML format.

" + pdf = client.PDF(url="https://example.com/document.pdf") - # Verify API call - mock_post.assert_called_once() - call_args = mock_post.call_args - assert "/api/1/article" in call_args[0][0] - assert call_args[1]["json"]["url"] == "https://example.com/article" - assert call_args[1]["json"]["output"] == "html" - assert "use_cache" not in call_args[1]["json"] # Default is True, not sent + assert pdf.title == "Test PDF Document" + assert pdf.is_rtl is False + assert pdf.videos == [] + _method, _path = mock_request.call_args[0] + assert _method == "GET" + assert _path == "/api/1/pdf" + assert mock_request.call_args[1]["params"]["url"] == "https://example.com/document.pdf" - def test_article_with_text_output(self, client, mock_article_text_response, mock_response): - """Test parsing article with text output.""" - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_article_text_response) - mock_post.return_value = response - - article = client.Article(url="https://example.com/article", output="text") - - assert article.text == "This is the article body in plain text format." - assert article.body == "This is the article body in plain text format." - - call_args = mock_post.call_args - assert call_args[1]["json"]["output"] == "text" - - def test_article_with_content(self, client, mock_article_response, mock_response): - """Test parsing article with HTML content.""" - html_content = "Test content" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_article_response) - mock_post.return_value = response - - client.Article(url="https://example.com/article", content=html_content) - - call_args = mock_post.call_args - assert call_args[1]["json"]["content"] == html_content - - def test_article_with_use_cache_false(self, client, mock_article_response, mock_response): - """Test parsing article with use_cache=False.""" - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_article_response) - mock_post.return_value = response - - client.Article(url="https://example.com/article", use_cache=False) - - call_args = mock_post.call_args - assert call_args[1]["json"]["use_cache"] == "false" - - def test_article_with_markdown_output(self, client, mock_response): - """Test parsing article with markdown output.""" - markdown_response = { - "url": "https://example.com/article", - "title": "Test Article", - "markdown": "# Test Article\n\nMarkdown body content.", - } - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=markdown_response) - mock_post.return_value = response - - article = client.Article(url="https://example.com/article", output="markdown") - - assert article.markdown == "# Test Article\n\nMarkdown body content." - assert article.body == "# Test Article\n\nMarkdown body content." - - call_args = mock_post.call_args - assert call_args[1]["json"]["output"] == "markdown" - - def test_article_invalid_output(self, client): - """Test that invalid output format raises ValidationError.""" - with pytest.raises(InstaparserValidationError) as exc_info: - client.Article(url="https://example.com/article", output="invalid") - - assert "output must be 'html', 'text', or 'markdown'" in str(exc_info.value) - - -class TestInstaparserClientSummary: - """Tests for Summary method.""" - - def test_summary_from_url(self, client, mock_summary_response, mock_response): - """Test generating summary from URL.""" - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_summary_response) - mock_post.return_value = response - - summary = client.Summary(url="https://example.com/article") - - assert len(summary.key_sentences) == 3 - assert summary.overview == "This is a comprehensive overview of the article content." - - call_args = mock_post.call_args - assert "/api/1/summary" in call_args[0][0] - assert call_args[1]["json"]["url"] == "https://example.com/article" - assert call_args[1]["json"]["stream"] is False - - def test_summary_with_content(self, client, mock_summary_response, mock_response): - """Test generating summary with HTML content.""" - html_content = "Test content" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_summary_response) - mock_post.return_value = response - - client.Summary(url="https://example.com/article", content=html_content) - - call_args = mock_post.call_args - assert call_args[1]["json"]["content"] == html_content - - def test_summary_with_use_cache_false(self, client, mock_summary_response, mock_response): - """Test generating summary with use_cache=False.""" - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_summary_response) - mock_post.return_value = response - - client.Summary(url="https://example.com/article", use_cache=False) - - call_args = mock_post.call_args - assert call_args[1]["json"]["use_cache"] == "false" - - def test_summary_with_streaming_callback(self, client): - """Test generating summary with streaming callback.""" - # Mock streaming response - stream_lines = [ - b'key_sentences: ["Sentence 1", "Sentence 2"]', - b"delta: This is", - b"delta: a streaming", - b"delta: overview.", - ] - - with patch.object(client.session, "post") as mock_post: - streaming_response = Mock(spec=requests.Response) - streaming_response.status_code = 200 - streaming_response.iter_lines.return_value = stream_lines - streaming_response.headers = {} - mock_post.return_value = streaming_response - - received_lines = [] - - def stream_callback(line): - received_lines.append(line) - - summary = client.Summary(url="https://example.com/article", stream_callback=stream_callback) - - # Verify callback was called for each line - assert len(received_lines) == 4 - assert received_lines[0] == 'key_sentences: ["Sentence 1", "Sentence 2"]' - - # Verify summary was constructed from stream - assert len(summary.key_sentences) == 2 - assert summary.overview == "This is a streaming overview." - - # Verify stream=True was set - call_args = mock_post.call_args - assert call_args[1]["json"]["stream"] is True - assert call_args[1]["stream"] is True - - def test_summary_streaming_with_error(self, client, mock_response): - """Test that streaming errors are handled.""" - with patch.object(client.session, "post") as mock_post: - error_response = mock_response(status_code=401, json_data={"reason": "Invalid API key"}) - mock_post.return_value = error_response - - with pytest.raises(InstaparserAuthenticationError): - client.Summary(url="https://example.com/article", stream_callback=lambda x: None) - - -class TestInstaparserClientPDF: - """Tests for PDF method.""" - - def test_pdf_from_url_get(self, client, mock_pdf_response, mock_response): - """Test parsing PDF from URL using GET request.""" - with patch.object(client.session, "get") as mock_get: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_get.return_value = response - - pdf = client.PDF(url="https://example.com/document.pdf") - - assert pdf.title == "Test PDF Document" - assert pdf.url == "https://example.com/document.pdf" - assert pdf.is_rtl is False - assert pdf.videos == [] - - call_args = mock_get.call_args - assert "/api/1/pdf" in call_args[0][0] - assert call_args[1]["params"]["url"] == "https://example.com/document.pdf" - assert call_args[1]["params"]["output"] == "html" - - def test_pdf_from_file_post(self, client, mock_pdf_response, mock_response): - """Test parsing PDF from file using POST request.""" + def test_from_file(self, client, mock_request): pdf_file = Mock() - pdf_file.read.return_value = b"PDF content" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_post.return_value = response - - pdf = client.PDF(file=pdf_file) + mock_request.return_value = make_response(json_data=PDF_DATA) - assert pdf.title == "Test PDF Document" + pdf = client.PDF(file=pdf_file) - call_args = mock_post.call_args - assert "/api/1/pdf" in call_args[0][0] - assert "files" in call_args[1] - assert call_args[1]["files"]["file"] == pdf_file - # Content-Type should be removed for multipart/form-data - assert "Content-Type" not in call_args[1].get("headers", {}) + assert pdf.title == "Test PDF Document" + assert mock_request.call_args[0][0] == "POST" + assert mock_request.call_args[1]["multipart_files"]["file"] is pdf_file - def test_pdf_from_file_bytes(self, client, mock_pdf_response, mock_response): - """Test parsing PDF from bytes.""" - pdf_bytes = b"PDF content bytes" + def test_from_bytes(self, client, mock_request): + mock_request.return_value = make_response(json_data=PDF_DATA) + client.PDF(file=b"PDF content") + assert mock_request.call_args[1]["multipart_files"]["file"] == b"PDF content" - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_post.return_value = response - - pdf = client.PDF(file=pdf_bytes) - - assert pdf.title == "Test PDF Document" - call_args = mock_post.call_args - assert call_args[1]["files"]["file"] == pdf_bytes - - def test_pdf_with_text_output(self, client, mock_pdf_response, mock_response): - """Test parsing PDF with text output.""" - with patch.object(client.session, "get") as mock_get: - pdf_response = dict(mock_pdf_response) - pdf_response["text"] = "PDF content in text" - pdf_response.pop("html", None) - response = mock_response(status_code=200, json_data=pdf_response) - mock_get.return_value = response - - client.PDF(url="https://example.com/document.pdf", output="text") - - call_args = mock_get.call_args - assert call_args[1]["params"]["output"] == "text" - - def test_pdf_with_use_cache_false(self, client, mock_pdf_response, mock_response): - """Test parsing PDF with use_cache=False.""" - with patch.object(client.session, "get") as mock_get: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_get.return_value = response - - client.PDF(url="https://example.com/document.pdf", use_cache=False) - - call_args = mock_get.call_args - assert call_args[1]["params"]["use_cache"] == "false" - - def test_pdf_with_file_and_url(self, client, mock_pdf_response, mock_response): - """Test parsing PDF with both file and URL (should use POST).""" + def test_file_and_url(self, client, mock_request): + """When both file and url are given, POST with url in form fields.""" pdf_file = Mock() - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_post.return_value = response - - client.PDF(url="https://example.com/document.pdf", file=pdf_file) - - call_args = mock_post.call_args - assert call_args[1]["data"]["url"] == "https://example.com/document.pdf" - - def test_pdf_with_markdown_output(self, client, mock_pdf_response, mock_response): - """Test parsing PDF with markdown output.""" - with patch.object(client.session, "get") as mock_get: - pdf_response = dict(mock_pdf_response) - pdf_response["markdown"] = "# PDF Document\n\nPDF content in markdown" - pdf_response.pop("html", None) - response = mock_response(status_code=200, json_data=pdf_response) - mock_get.return_value = response - - client.PDF(url="https://example.com/document.pdf", output="markdown") - - call_args = mock_get.call_args - assert call_args[1]["params"]["output"] == "markdown" - - def test_pdf_invalid_output(self, client): - """Test that invalid output format raises ValidationError.""" - with pytest.raises(InstaparserValidationError) as exc_info: - client.PDF(url="https://example.com/document.pdf", output="invalid") - - assert "output must be 'html', 'text', or 'markdown'" in str(exc_info.value) - - def test_pdf_no_url_or_file(self, client): - """Test that missing both url and file raises ValidationError.""" - with pytest.raises(InstaparserValidationError) as exc_info: + mock_request.return_value = make_response(json_data=PDF_DATA) + client.PDF(url="https://example.com/document.pdf", file=pdf_file) + assert mock_request.call_args[1]["multipart_fields"]["url"] == "https://example.com/document.pdf" + + @pytest.mark.parametrize("output", ["text", "markdown"]) + def test_output_formats_url(self, client, mock_request, output): + data = {**PDF_DATA, "html": None, output: f"content in {output}"} + mock_request.return_value = make_response(json_data=data) + client.PDF(url="https://example.com/document.pdf", output=output) + assert mock_request.call_args[1]["params"]["output"] == output + + def test_use_cache_false_url(self, client, mock_request): + mock_request.return_value = make_response(json_data=PDF_DATA) + client.PDF(url="https://example.com/document.pdf", use_cache=False) + assert mock_request.call_args[1]["params"]["use_cache"] == "false" + + def test_use_cache_false_file(self, client, mock_request): + mock_request.return_value = make_response(json_data=PDF_DATA) + client.PDF(file=Mock(), use_cache=False) + assert mock_request.call_args[1]["multipart_fields"]["use_cache"] == "false" + + def test_invalid_output(self, client): + with pytest.raises(InstaparserValidationError, match="output must be"): + client.PDF(url="u", output="invalid") + + def test_no_url_or_file(self, client): + with pytest.raises(InstaparserValidationError, match="Either 'url' or 'file'"): client.PDF() - assert "Either 'url' or 'file' must be provided" in str(exc_info.value) - - -class TestInstaparserClientEdgeCases: - """Tests for edge cases and error scenarios.""" - - def test_article_network_error(self, client): - """Test handling network errors during Article request.""" - with patch.object(client.session, "post") as mock_post: - mock_post.side_effect = requests.exceptions.ConnectionError("Connection failed") - - with pytest.raises(requests.exceptions.ConnectionError): - client.Article(url="https://example.com/article") - - def test_article_timeout_error(self, client): - """Test handling timeout errors during Article request.""" - with patch.object(client.session, "post") as mock_post: - mock_post.side_effect = requests.exceptions.Timeout("Request timed out") - - with pytest.raises(requests.exceptions.Timeout): - client.Article(url="https://example.com/article") - - def test_article_malformed_json_response(self, client, mock_response): - """Test handling malformed JSON in successful response.""" - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, text="Not valid JSON {") - mock_post.return_value = response - - # When JSON parsing fails, _handle_response returns {'raw': text} - # This will cause Article initialization to fail or have None values - # The test verifies the client handles this gracefully - from instaparser.article import Article - - result = client.Article(url="https://example.com/article") - # Article should still be created, but may have None/empty values - assert isinstance(result, Article) - - def test_summary_empty_response(self, client, mock_response): - """Test handling empty summary response.""" - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data={}) - mock_post.return_value = response - - summary = client.Summary(url="https://example.com/article") - assert summary.key_sentences == [] - assert summary.overview == "" - - def test_summary_streaming_empty_lines(self, client): - """Test handling empty lines in streaming response.""" - stream_lines = [ - b"key_sentences: []", - b"", - b"delta: Content", - b"", - ] - - with patch.object(client.session, "post") as mock_post: - streaming_response = Mock(spec=requests.Response) - streaming_response.status_code = 200 - streaming_response.iter_lines.return_value = stream_lines - streaming_response.headers = {} - mock_post.return_value = streaming_response - - received_lines = [] - - def stream_callback(line): - received_lines.append(line) - - summary = client.Summary(url="https://example.com/article", stream_callback=stream_callback) - - # Empty lines should be filtered out by iter_lines check - assert summary.overview == "Content" - - def test_summary_streaming_malformed_key_sentences(self, client): - """Test handling malformed key_sentences in streaming response raises InstaparserAPIError.""" - stream_lines = [ - b"key_sentences: not valid json", - b"delta: Content", - ] - - with patch.object(client.session, "post") as mock_post: - streaming_response = Mock(spec=requests.Response) - streaming_response.status_code = 200 - streaming_response.iter_lines.return_value = stream_lines - streaming_response.headers = {} - mock_post.return_value = streaming_response - - def stream_callback(line): - pass - - # Should raise InstaparserAPIError when key_sentences JSON is malformed - with pytest.raises(InstaparserAPIError) as exc_info: - client.Summary(url="https://example.com/article", stream_callback=stream_callback) - - # Verify the exception has the correct message and status_code (from line 222) - assert exc_info.value.status_code == 412 - assert "Unable to generate key sentences" in str(exc_info.value) - - def test_pdf_file_with_url_parameter(self, client, mock_pdf_response, mock_response): - """Test PDF with both file and URL parameters.""" - pdf_file = Mock() - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_post.return_value = response - - client.PDF(url="https://example.com/document.pdf", file=pdf_file) - - call_args = mock_post.call_args - assert call_args[1]["data"]["url"] == "https://example.com/document.pdf" - assert call_args[1]["files"]["file"] == pdf_file - - def test_pdf_file_like_object(self, client, mock_pdf_response, mock_response): - """Test PDF with file-like object.""" - - class FileLike: - def read(self): - return b"PDF content" - - file_like = FileLike() - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_post.return_value = response - - client.PDF(file=file_like) - - call_args = mock_post.call_args - assert call_args[1]["files"]["file"] == file_like - - def test_article_url_encoding(self, client, mock_article_response, mock_response): - """Test Article with URL containing special characters.""" - special_url = "https://example.com/article?param=value&other=test" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_article_response) - mock_post.return_value = response - - client.Article(url=special_url) - - call_args = mock_post.call_args - assert call_args[1]["json"]["url"] == special_url - - def test_summary_url_encoding(self, client, mock_summary_response, mock_response): - """Test Summary with URL containing special characters.""" - special_url = "https://example.com/article?param=value&other=test" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_summary_response) - mock_post.return_value = response - - client.Summary(url=special_url) - - call_args = mock_post.call_args - assert call_args[1]["json"]["url"] == special_url - - def test_pdf_url_encoding(self, client, mock_pdf_response, mock_response): - """Test PDF with URL containing special characters.""" - special_url = "https://example.com/document.pdf?param=value" - - with patch.object(client.session, "get") as mock_get: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_get.return_value = response - - client.PDF(url=special_url) - - call_args = mock_get.call_args - assert call_args[1]["params"]["url"] == special_url - - def test_article_large_content(self, client, mock_article_response, mock_response): - """Test Article with large HTML content.""" - large_content = "" + "

Content

" * 10000 + "" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_article_response) - mock_post.return_value = response - - client.Article(url="https://example.com/article", content=large_content) - - call_args = mock_post.call_args - assert call_args[1]["json"]["content"] == large_content - - def test_summary_large_content(self, client, mock_summary_response, mock_response): - """Test Summary with large HTML content.""" - large_content = "" + "

Content

" * 10000 + "" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_summary_response) - mock_post.return_value = response - - client.Summary(url="https://example.com/article", content=large_content) - - call_args = mock_post.call_args - assert call_args[1]["json"]["content"] == large_content - - def test_pdf_large_file(self, client, mock_pdf_response, mock_response): - """Test PDF with large file.""" - large_file = b"PDF content " * 100000 - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_post.return_value = response - - client.PDF(file=large_file) - - call_args = mock_post.call_args - assert call_args[1]["files"]["file"] == large_file - - def test_handle_response_empty_error_text(self, client, mock_response): - """Test handling error response with empty text.""" - response = mock_response(status_code=500, text="") - - with pytest.raises(InstaparserAPIError) as exc_info: - client._handle_response(response) - - assert exc_info.value.status_code == 500 - - def test_handle_response_error_with_none_text(self, client): - """Test handling error response with None text.""" - response = Mock(spec=requests.Response) - response.status_code = 500 - response.text = None - response.json.side_effect = ValueError("Not JSON") - response.headers = {} +class TestTransportErrors: + def test_url_error(self, client, mock_request): + mock_request.side_effect = URLError("Connection failed") + with pytest.raises(URLError): + client.Article(url="u") + + def test_timeout_error(self, client, mock_request): + mock_request.side_effect = TimeoutError("timed out") + with pytest.raises(TimeoutError): + client.Article(url="u") + + +class TestURLConstruction: + @pytest.mark.parametrize("base", ["https://api.test.com", "https://api.test.com/"]) + def test_base_url_joining(self, base, monkeypatch): + client = InstaparserClient(api_key=API_KEY, base_url=base) + mock_urlopen = Mock(return_value=make_response(json_data={})) + monkeypatch.setattr("instaparser.client.urlopen", mock_urlopen) + client.Article(url="u") + req = mock_urlopen.call_args[0][0] + assert req.full_url.startswith("https://api.test.com/api/1/article") + + +class TestMultipleClients: + def test_independent_instances(self): + c1 = InstaparserClient(api_key="key1", base_url="https://api1.com") + c2 = InstaparserClient(api_key="key2", base_url="https://api2.com") + assert c1.api_key != c2.api_key + assert c1.base_url != c2.base_url + + def test_client_reuse(self, client, mock_request): + mock_request.return_value = make_response(json_data=ARTICLE_DATA) + a1 = client.Article(url="u1") + a2 = client.Article(url="u2") + assert mock_request.call_count == 2 + assert a1.title == a2.title == "Test Article Title" + + +class TestErrorPropagation: + @pytest.mark.parametrize("status, exc_cls", ERROR_CODES) + def test_article_errors(self, client, mock_request, status, exc_cls): + mock_request.side_effect = make_error(status, json_data={"reason": f"Error {status}"}) + with pytest.raises(exc_cls) as exc_info: + client.Article(url="u") + assert exc_info.value.status_code == status + + @pytest.mark.parametrize("status, exc_cls", ERROR_CODES) + def test_summary_errors(self, client, mock_request, status, exc_cls): + mock_request.side_effect = make_error(status, json_data={"reason": f"Error {status}"}) + with pytest.raises(exc_cls): + client.Summary(url="u") + + @pytest.mark.parametrize("status, exc_cls", ERROR_CODES) + def test_pdf_errors(self, client, mock_request, status, exc_cls): + mock_request.side_effect = make_error(status, json_data={"reason": f"Error {status}"}) + with pytest.raises(exc_cls): + client.PDF(url="u") + + def test_error_without_reason_field(self, client, mock_request): + mock_request.side_effect = make_error(500, json_data={}) + with pytest.raises(InstaparserAPIError, match="API request failed"): + client.Article(url="u") + + def test_error_plain_text_body(self, client, mock_request): + mock_request.side_effect = make_error(500, text="Error message") + with pytest.raises(InstaparserAPIError, match="Error message"): + client.Article(url="u") + + def test_error_empty_body(self, client, mock_request): + mock_request.side_effect = make_error(500, text="") with pytest.raises(InstaparserAPIError) as exc_info: - client._handle_response(response) - + client.Article(url="u") assert exc_info.value.status_code == 500 - def test_session_headers_preserved(self, api_key): - """Test that session headers are properly set and preserved.""" - client = InstaparserClient(api_key=api_key) - - # Headers should be set - assert "Authorization" in client.session.headers - assert "Content-Type" in client.session.headers - - # Headers should persist - assert client.session.headers["Authorization"] == f"Bearer {api_key}" - assert client.session.headers["Content-Type"] == "application/json" - - def test_base_url_joining(self, api_key, mock_response): - """Test that base URL is properly joined with endpoints.""" - custom_base = "https://api.test.com" - client = InstaparserClient(api_key=api_key, base_url=custom_base) - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data={}) - mock_post.return_value = response - - client.Article(url="https://example.com/article") - - call_args = mock_post.call_args - assert call_args[0][0].startswith(custom_base) - assert "/api/1/article" in call_args[0][0] - - def test_base_url_with_trailing_slash(self, api_key, mock_response): - """Test base URL with trailing slash is handled correctly.""" - custom_base = "https://api.test.com/" - client = InstaparserClient(api_key=api_key, base_url=custom_base) - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data={}) - mock_post.return_value = response - - client.Article(url="https://example.com/article") - - call_args = mock_post.call_args - # urljoin should handle trailing slash correctly - assert "/api/1/article" in call_args[0][0] - - def test_article_content_with_use_cache_false(self, client, mock_article_response, mock_response): - """Test Article with both content and use_cache=False.""" - html_content = "Test" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_article_response) - mock_post.return_value = response - - client.Article(url="https://example.com/article", content=html_content, use_cache=False) - - call_args = mock_post.call_args - assert call_args[1]["json"]["content"] == html_content - assert call_args[1]["json"]["use_cache"] == "false" - - def test_summary_content_with_use_cache_false(self, client, mock_summary_response, mock_response): - """Test Summary with both content and use_cache=False.""" - html_content = "Test" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_summary_response) - mock_post.return_value = response - - client.Summary(url="https://example.com/article", content=html_content, use_cache=False) - - call_args = mock_post.call_args - assert call_args[1]["json"]["content"] == html_content - assert call_args[1]["json"]["use_cache"] == "false" - - def test_pdf_file_with_use_cache_false(self, client, mock_pdf_response, mock_response): - """Test PDF with file and use_cache=False.""" - pdf_file = Mock() - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_post.return_value = response - - client.PDF(file=pdf_file, use_cache=False) - - call_args = mock_post.call_args - assert call_args[1]["data"]["use_cache"] == "false" - - def test_pdf_url_with_use_cache_false(self, client, mock_pdf_response, mock_response): - """Test PDF with URL and use_cache=False.""" - with patch.object(client.session, "get") as mock_get: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_get.return_value = response - - client.PDF(url="https://example.com/document.pdf", use_cache=False) - - call_args = mock_get.call_args - assert call_args[1]["params"]["use_cache"] == "false" - - def test_summary_streaming_callback_called_for_each_line(self, client): - """Test that streaming callback is called for each non-empty line.""" - stream_lines = [ - b'key_sentences: ["S1"]', - b"delta: Line 1", - b"delta: Line 2", - b"delta: Line 3", - ] - - with patch.object(client.session, "post") as mock_post: - streaming_response = Mock(spec=requests.Response) - streaming_response.status_code = 200 - streaming_response.iter_lines.return_value = stream_lines - streaming_response.headers = {} - mock_post.return_value = streaming_response - - callback_calls = [] - - def stream_callback(line): - callback_calls.append(line) - - client.Summary(url="https://example.com/article", stream_callback=stream_callback) - - assert len(callback_calls) == 4 - assert callback_calls[0] == 'key_sentences: ["S1"]' - assert callback_calls[1] == "delta: Line 1" - assert callback_calls[2] == "delta: Line 2" - assert callback_calls[3] == "delta: Line 3" - - def test_pdf_multipart_form_data_headers(self, client, mock_pdf_response, mock_response): - """Test that Content-Type header is removed for multipart form data.""" - pdf_file = Mock() - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=mock_pdf_response) - mock_post.return_value = response - - client.PDF(file=pdf_file) - call_args = mock_post.call_args - headers = call_args[1].get("headers", {}) - # Content-Type should not be in headers (requests will set it for multipart) - assert "Content-Type" not in headers or headers.get("Content-Type") != "application/json" +class TestOutputFormats: + @pytest.mark.parametrize( + "output, field, body_text", + [ + ("html", "html", "

HTML content

"), + ("text", "text", "Text content"), + ("markdown", "markdown", "# Markdown content"), + ], + ) + def test_article_output_formats(self, client, mock_request, output, field, body_text): + mock_request.return_value = make_response(json_data={"url": "u", field: body_text}) + article = client.Article(url="u", output=output) + assert article.body == body_text + + @pytest.mark.parametrize( + "output, field, body_text", + [ + ("html", "html", "

PDF HTML

"), + ("text", "text", "PDF Text"), + ("markdown", "markdown", "# PDF markdown"), + ], + ) + def test_pdf_output_formats(self, client, mock_request, output, field, body_text): + mock_request.return_value = make_response(json_data={"url": "u", field: body_text}) + pdf = client.PDF(url="u", output=output) + assert pdf.body == body_text + + +class TestEncodeMultipartFormdata: + def test_fields_only(self): + body, content_type = _encode_multipart_formdata({"key": "value"}, {}) + assert content_type.startswith("multipart/form-data; boundary=") + assert b'name="key"' in body + assert b"value" in body + assert body.endswith(b"--\r\n") + + def test_file_from_bytes(self): + body, content_type = _encode_multipart_formdata({}, {"file": b"PDF content"}) + assert b'name="file"' in body + assert b'filename="upload"' in body + assert b"application/octet-stream" in body + assert b"PDF content" in body + + def test_file_from_file_object(self): + f = io.BytesIO(b"file data") + body, _ = _encode_multipart_formdata({}, {"file": f}) + assert b"file data" in body + + def test_fields_and_files(self): + body, content_type = _encode_multipart_formdata( + {"output": "html", "use_cache": "false"}, + {"file": b"data"}, + ) + boundary = content_type.split("boundary=")[1] + parts = body.split(f"--{boundary}".encode()) + # parts: ['', field1, field2, file, '--\r\n'] + assert len(parts) == 5 + assert b'name="output"' in parts[1] + assert b'name="use_cache"' in parts[2] + assert b'name="file"' in parts[3] + + def test_boundary_is_unique(self): + _, ct1 = _encode_multipart_formdata({}, {}) + _, ct2 = _encode_multipart_formdata({}, {}) + assert ct1 != ct2 diff --git a/tests/test_integration.py b/tests/test_integration.py deleted file mode 100644 index 8c1a915..0000000 --- a/tests/test_integration.py +++ /dev/null @@ -1,327 +0,0 @@ -""" -Integration-style tests for Instaparser Library. - -These tests verify that the library components work together correctly, -though they still use mocking to avoid actual API calls. -""" - -from unittest.mock import Mock, patch - -import pytest -import requests - -from instaparser import InstaparserClient -from instaparser.exceptions import ( - InstaparserAPIError, - InstaparserAuthenticationError, - InstaparserRateLimitError, - InstaparserValidationError, -) - - -class TestClientArticleIntegration: - """Integration tests for Article functionality.""" - - def test_full_article_workflow(self, api_key, base_url, mock_response): - """Test complete article parsing workflow.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - article_data = { - "url": "https://example.com/article", - "title": "Test Article", - "author": "John Doe", - "words": 500, - "html": "

Article content

", - } - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=article_data) - mock_post.return_value = response - - article = client.Article(url="https://example.com/article") - - assert article.url == "https://example.com/article" - assert article.title == "Test Article" - assert article.author == "John Doe" - assert article.words == 500 - assert article.body == "

Article content

" - - def test_article_with_error_handling(self, api_key, base_url, mock_response): - """Test article parsing with error handling.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=401, json_data={"reason": "Invalid API key"}) - mock_post.return_value = response - - with pytest.raises(InstaparserAuthenticationError) as exc_info: - client.Article(url="https://example.com/article") - - assert exc_info.value.status_code == 401 - assert "Invalid API key" in str(exc_info.value) - - -class TestClientSummaryIntegration: - """Integration tests for Summary functionality.""" - - def test_full_summary_workflow(self, api_key, base_url, mock_response): - """Test complete summary generation workflow.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - summary_data = { - "key_sentences": ["Sentence 1", "Sentence 2"], - "overview": "This is the overview", - } - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=summary_data) - mock_post.return_value = response - - summary = client.Summary(url="https://example.com/article") - - assert len(summary.key_sentences) == 2 - assert summary.overview == "This is the overview" - - def test_summary_streaming_workflow(self, api_key, base_url): - """Test complete streaming summary workflow.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - stream_lines = [ - b'key_sentences: ["Key 1", "Key 2"]', - b"delta: Overview", - b"delta: text", - ] - - with patch.object(client.session, "post") as mock_post: - streaming_response = Mock(spec=requests.Response) - streaming_response.status_code = 200 - streaming_response.iter_lines.return_value = stream_lines - streaming_response.headers = {} - mock_post.return_value = streaming_response - - received = [] - - def callback(line): - received.append(line) - - summary = client.Summary(url="https://example.com/article", stream_callback=callback) - - assert len(received) == 3 - assert len(summary.key_sentences) == 2 - assert summary.overview == "Overview text" - - -class TestClientPDFIntegration: - """Integration tests for PDF functionality.""" - - def test_full_pdf_workflow_from_url(self, api_key, base_url, mock_response): - """Test complete PDF parsing workflow from URL.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - pdf_data = { - "url": "https://example.com/document.pdf", - "title": "Test PDF", - "words": 1000, - "html": "

PDF content

", - } - - with patch.object(client.session, "get") as mock_get: - response = mock_response(status_code=200, json_data=pdf_data) - mock_get.return_value = response - - pdf = client.PDF(url="https://example.com/document.pdf") - - assert pdf.url == "https://example.com/document.pdf" - assert pdf.title == "Test PDF" - assert pdf.words == 1000 - assert pdf.body == "

PDF content

" - assert pdf.is_rtl is False - assert pdf.videos == [] - - def test_full_pdf_workflow_from_file(self, api_key, base_url, mock_response): - """Test complete PDF parsing workflow from file.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - pdf_data = { - "url": "https://example.com/document.pdf", - "title": "Test PDF", - "words": 1000, - "html": "

PDF content

", - } - - pdf_file = b"PDF file content" - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=pdf_data) - mock_post.return_value = response - - pdf = client.PDF(file=pdf_file) - - assert pdf.title == "Test PDF" - assert pdf.is_rtl is False - assert pdf.videos == [] - - # Verify file was uploaded - call_args = mock_post.call_args - assert call_args[1]["files"]["file"] == pdf_file - - -class TestErrorHandlingIntegration: - """Integration tests for error handling across the library.""" - - def test_error_propagation_through_client(self, api_key, base_url): - """Test that errors are properly propagated through client methods.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - error_codes = [ - (400, InstaparserValidationError), - (401, InstaparserAuthenticationError), - (403, InstaparserAPIError), - (409, InstaparserAPIError), - (429, InstaparserRateLimitError), - (500, InstaparserAPIError), - ] - - for status_code, expected_exception in error_codes: - with patch.object(client.session, "post") as mock_post: - error_response = Mock(spec=requests.Response) - error_response.status_code = status_code - error_response.json.return_value = {"reason": f"Error {status_code}"} - error_response.headers = {} - mock_post.return_value = error_response - - with pytest.raises(expected_exception) as exc_info: - client.Article(url="https://example.com/article") - - assert exc_info.value.status_code == status_code - - -class TestMultipleClientInstances: - """Tests for using multiple client instances.""" - - def test_multiple_clients_independent(self): - """Test that multiple client instances are independent.""" - client1 = InstaparserClient(api_key="key1", base_url="https://api1.com") - client2 = InstaparserClient(api_key="key2", base_url="https://api2.com") - - assert client1.api_key == "key1" - assert client2.api_key == "key2" - assert client1.base_url == "https://api1.com" - assert client2.base_url == "https://api2.com" - assert client1.session is not client2.session - - def test_client_reuse_for_multiple_requests(self, api_key, base_url, mock_response): - """Test reusing a client for multiple requests.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - article_data = { - "url": "https://example.com/article", - "title": "Test Article", - "html": "

Content

", - } - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=article_data) - mock_post.return_value = response - - # Make multiple requests - article1 = client.Article(url="https://example.com/article1") - article2 = client.Article(url="https://example.com/article2") - - assert mock_post.call_count == 2 - assert article1.title == "Test Article" - assert article2.title == "Test Article" - - -class TestOutputFormats: - """Tests for different output formats.""" - - def test_article_html_vs_text_output(self, api_key, base_url, mock_response): - """Test Article with HTML and text outputs.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - html_data = { - "url": "https://example.com/article", - "html": "

HTML content

", - } - - text_data = { - "url": "https://example.com/article", - "text": "Text content", - } - - with patch.object(client.session, "post") as mock_post: - # Test HTML output - response = mock_response(status_code=200, json_data=html_data) - mock_post.return_value = response - - article_html = client.Article(url="https://example.com/article", output="html") - assert article_html.body == "

HTML content

" - - # Test text output - response = mock_response(status_code=200, json_data=text_data) - mock_post.return_value = response - article_text = client.Article(url="https://example.com/article", output="text") - assert article_text.body == "Text content" - - def test_article_markdown_output(self, api_key, base_url, mock_response): - """Test Article with markdown output.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - markdown_data = { - "url": "https://example.com/article", - "markdown": "# Article Title\n\nMarkdown content", - } - - with patch.object(client.session, "post") as mock_post: - response = mock_response(status_code=200, json_data=markdown_data) - mock_post.return_value = response - - article = client.Article(url="https://example.com/article", output="markdown") - assert article.markdown == "# Article Title\n\nMarkdown content" - assert article.body == "# Article Title\n\nMarkdown content" - - def test_pdf_html_vs_text_output(self, api_key, base_url, mock_response): - """Test PDF with HTML and text outputs.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - html_data = { - "url": "https://example.com/document.pdf", - "html": "

PDF HTML

", - } - - text_data = { - "url": "https://example.com/document.pdf", - "text": "PDF Text", - } - - with patch.object(client.session, "get") as mock_get: - # Test HTML output - response = mock_response(status_code=200, json_data=html_data) - mock_get.return_value = response - - pdf_html = client.PDF(url="https://example.com/document.pdf", output="html") - assert pdf_html.body == "

PDF HTML

" - - # Test text output - response = mock_response(status_code=200, json_data=text_data) - mock_get.return_value = response - pdf_text = client.PDF(url="https://example.com/document.pdf", output="text") - assert pdf_text.body == "PDF Text" - - def test_pdf_markdown_output(self, api_key, base_url, mock_response): - """Test PDF with markdown output.""" - client = InstaparserClient(api_key=api_key, base_url=base_url) - - markdown_data = { - "url": "https://example.com/document.pdf", - "markdown": "# PDF Title\n\nPDF markdown content", - } - - with patch.object(client.session, "get") as mock_get: - response = mock_response(status_code=200, json_data=markdown_data) - mock_get.return_value = response - - pdf = client.PDF(url="https://example.com/document.pdf", output="markdown") - assert pdf.markdown == "# PDF Title\n\nPDF markdown content" - assert pdf.body == "# PDF Title\n\nPDF markdown content"