Allowing In memory Image Instatiation for byLLM

docs/docs/communityhub/release_notes.md

@@ -5,6 +5,8 @@ This document provides a summary of new features, improvements, and bug fixes in
 
 ## jaclang 0.8.9 / jac-cloud 0.2.9 / byllm 0.4.4 (Unreleased)
 
+- byLLM: Image now accepts in-memory and path-like inputs (bytes/bytearray/memoryview, BytesIO/file-like, PIL.Image, Path), plus data/gs/http(s) URLs; auto-detects MIME (incl. WEBP), preserves URLs, and reads file-like streams without moving the cursor.
+
 ## jaclang 0.8.8 / jac-cloud 0.2.8 / byllm 0.4.3
 
 - **Better Syntax Error Messages**: Initial improvements to syntax error diagnostics, providing clearer and more descriptive messages that highlight the location and cause of errors (e.g., `Missing semicolon`).

docs/docs/learn/jac-byllm/multimodality.md

@@ -52,6 +52,44 @@ Input Image :
 
 In this example, an image of a person is provided as input to the `get_person_info` method. The method returns a `Person` object containing the extracted information from the image.
 
+### More ways to pass images
+
+`Image` accepts multiple input forms beyond file paths:
+
+- URLs: `http://`, `https://`, `gs://` (left as-is)
+- Data URLs: `data:image/...;base64,...` (left as-is)
+- Path-like: `pathlib.Path` (resolved to a local file)
+- In-memory: `bytes`, `bytearray`, `memoryview`, `io.BytesIO` or any file-like object returning bytes
+- PIL: `PIL.Image.Image`
+
+Python example for in-memory usage:
+
+```jac
+import from byllm {Image}
+import io;
+Import from PIL {Image as PILImage}
+
+with entry {
+    pil_img = PILImage.open("photo.jpg");
+
+    # BytesIO buffer
+    buf = io.BytesIO();
+    pil_img.save(buf, format="PNG");
+    img_a = Image(buf);
+
+    # Raw bytes
+    raw = buf.getvalue();
+    img_b = Image(raw);
+
+    # PIL image instance
+    img_c = Image(pil_img);
+
+    # You can also pass data URLs and gs:// links directly
+    img_d = Image("data:image/png;base64,<...>");
+    img_e = Image("gs://bucket/path/image.png");
+}
+```
+
 ## Video
 
 byLLM supports video inputs through the `Video` format. Videos can be provided as input to byLLM functions or methods:

jac-byllm/byllm/types.py

@@ -12,8 +12,9 @@
 from dataclasses import dataclass
 from enum import StrEnum
 from io import BytesIO
-from typing import Callable, TypeAlias, get_type_hints
+from typing import Callable, IO, TypeAlias, get_type_hints
 
+from PIL.Image import Image as PILImageCls
 from PIL.Image import open as open_image
 
 from litellm.types.utils import Message as LiteLLMMessage
@@ -255,30 +256,126 @@ def to_dict(self) -> list[dict]:
 class Image(Media):
     """Class representing an image."""
 
-    url: str
+    url: (
+        "str | bytes | bytearray | memoryview | BytesIO | IO[bytes] | "
+        "os.PathLike[str] | os.PathLike[bytes] | PILImageCls"
+    )  # type: ignore[name-defined]
     mime_type: str | None = None
 
+    def _format_to_mime(self, fmt: str | None) -> str:
+        """Map a PIL format name to a MIME type with sensible fallbacks."""
+        fmt = (fmt or "PNG").upper()
+        if fmt == "WEBP":
+            return "image/webp"
+        if fmt == "JPEG" or fmt == "JPG":
+            return "image/jpeg"
+        if fmt == "PNG":
+            return "image/png"
+        # Try mimetypes (uses extension mapping)
+        mime = mimetypes.types_map.get("." + fmt.lower())
+        return mime or "image/png"
+
+    def _data_url_from_bytes(self, data: bytes, fmt: str | None) -> str:
+        mime = self.mime_type or self._format_to_mime(fmt)
+        # Ensure mime_type is set on the instance for downstream usage
+        self.mime_type = mime
+        b64 = base64.b64encode(data).decode("utf-8")
+        return f"data:{mime};base64,{b64}"
+
     def __post_init__(self) -> None:
-        """Post-initialization to ensure the URL is a string."""
-        if self.url.startswith(("http://", "https://", "gs://")):
-            self.url = self.url.strip()
-        else:
-            if not os.path.exists(self.url):
-                raise ValueError(f"Image file does not exist: {self.url}")
-            image = open_image(self.url)
-
-            # python<3.13 mimetypes doesn't support `webp` format as it wasn't an IANA standard
-            # until November 2024 (RFC-9649:  https://www.rfc-editor.org/rfc/rfc9649.html).
-            if (image.format and image.format.lower()) == "webp":
-                self.mime_type = "image/webp"
-            else:
-                self.mime_type = mimetypes.types_map.get(
-                    "." + (image.format or "png").lower()
-                )
+        """Normalize input into a data URL or leave remote/data URLs as-is.
+
+        Supported inputs:
+        - HTTP(S)/GS URLs (left as-is)
+        - Data URLs (data:...)
+        - Local file paths (opened and encoded to data URL)
+        - Bytes / bytearray / memoryview
+        - File-like objects (BytesIO or any IO[bytes])
+        - os.PathLike
+        - PIL.Image.Image instances
+        """
+        value = self.url
+
+        # Handle path-like inputs by converting to string
+        if isinstance(value, os.PathLike):
+            value = os.fspath(value)
+
+        # Remote or data URLs: keep as-is (trim whitespace)
+        if isinstance(value, str):
+            s = value.strip()
+            if s.startswith(("http://", "https://", "gs://", "data:")):
+                self.url = s
+                return
+            # Treat as local file path
+            if not os.path.exists(s):
+                raise ValueError(f"Image file does not exist: {s}")
+            image = open_image(s)
+            fmt = image.format or "PNG"
+            # Determine MIME type with WEBP special-case for py<3.13
+            self.mime_type = self._format_to_mime(fmt)
             with BytesIO() as buffer:
-                image.save(buffer, format=image.format, quality=100)
-                base64_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
-                self.url = f"data:{self.mime_type};base64,{base64_image}"
+                image.save(buffer, format=fmt)
+                data = buffer.getvalue()
+            self.url = self._data_url_from_bytes(data, fmt)
+            return
+
+        # PIL Image instance
+        if isinstance(value, PILImageCls):
+            fmt = value.format or "PNG"
+            with BytesIO() as buffer:
+                value.save(buffer, format=fmt)
+                data = buffer.getvalue()
+            self.url = self._data_url_from_bytes(data, fmt)
+            return
+
+        # Bytes-like object
+        if isinstance(value, (bytes, bytearray, memoryview)):
+            raw = bytes(value)
+            # Probe format via PIL to set correct MIME
+            img = open_image(BytesIO(raw))
+            fmt = img.format or "PNG"
+            # Use bytes as-is (avoid re-encode) if PIL detects same format as content
+            # Otherwise, re-encode to the detected format to be safe.
+            try:
+                self.url = self._data_url_from_bytes(raw, fmt)
+            except Exception:
+                with BytesIO() as buffer:
+                    img.save(buffer, format=fmt)
+                    self.url = self._data_url_from_bytes(buffer.getvalue(), fmt)
+            return
+
+        # File-like object (e.g., BytesIO, IO[bytes])
+        if hasattr(value, "read") and callable(value.read):
+            # Safely read without permanently moving the cursor
+            stream: IO[bytes] = value  # type: ignore[assignment]
+            pos = None
+            try:
+                pos = stream.tell()  # type: ignore[attr-defined]
+            except Exception:
+                pos = None
+            try:
+                # Prefer getvalue if available (e.g., BytesIO)
+                if hasattr(stream, "getvalue") and callable(stream.getvalue):
+                    raw = stream.getvalue()  # type: ignore[call-arg]
+                else:
+                    if hasattr(stream, "seek"):
+                        with suppress(Exception):
+                            stream.seek(0)
+                    raw = stream.read()
+                img = open_image(BytesIO(raw))
+                fmt = img.format or "PNG"
+                self.url = self._data_url_from_bytes(raw, fmt)
+            finally:
+                if pos is not None and hasattr(stream, "seek"):
+                    with suppress(Exception):
+                        stream.seek(pos)
+            return
+
+        # If we reach here, the input type isn't supported
+        raise TypeError(
+            "Unsupported Image input type. Provide a URL/path string, data URL, bytes, "
+            "BytesIO, file-like object, os.PathLike, or PIL.Image.Image."
+        )
 
     def to_dict(self) -> list[dict]:
         """Convert the image to a dictionary."""

jac-byllm/tests/fixtures/image_types.jac

@@ -0,0 +1,60 @@
+import from byllm {Image}
+import from PIL {Image as PILImage}
+import io;
+import os;
+import base64;
+import from pathlib {Path}
+
+glob img_file_path = os.path.join(os.path.dirname(__file__), "alan-m-turing.jpg");
+
+with entry {
+    print("PIL Image");
+    pil_image = PILImage.open(img_file_path);
+    image = Image(pil_image);
+
+    print("Image from file path");
+    image2 = Image(img_file_path);
+    print("Image from URL");
+    image3 = Image("https://en.wikipedia.org/wiki/File:Alan_turing_header.jpg");
+
+    print("Image from BytesIO");
+    bytes_buf = io.BytesIO();
+    pil_image.save(bytes_buf, format="PNG");
+    image4 = Image(bytes_buf);
+
+    print("Image from raw bytes");
+    bytes_buf2 = io.BytesIO();
+    pil_image.save(bytes_buf2, format="PNG");
+    raw_bytes = bytes_buf2.getvalue();
+    image5 = Image(raw_bytes);
+
+    print("Image from memoryview");
+    mv = memoryview(raw_bytes);
+    image6 = Image(mv);
+
+    print("Image from data URL");
+    bytes_buf3 = io.BytesIO();
+    pil_image.save(bytes_buf3, format="PNG");
+    b64 = base64.b64encode(bytes_buf3.getvalue()).decode("utf-8");
+    data_url = "data:image/png;base64," + b64;
+    image7 = Image(data_url);
+
+    print("Image from PathLike");
+    p = Path(img_file_path);
+    image8 = Image(p);
+
+    print("Image from file-like without getvalue");
+    # Use a BufferedReader to simulate a stream lacking getvalue
+    tmp_buf = io.BytesIO();
+    pil_image.save(tmp_buf, format="PNG");
+    reader = io.BufferedReader(io.BytesIO(tmp_buf.getvalue()));
+    image9 = Image(reader);
+
+    print("Image from bytearray");
+    ba = bytearray(raw_bytes);
+    image10 = Image(ba);
+
+    print("Image from gs:// URL");
+    image11 = Image("gs://bucket/path/alan_turing.png");
+
+}

jac-byllm/tests/test_byllm.py

@@ -252,4 +252,27 @@ def test_enum_without_value(self) -> None:
         sys.stdout = sys.__stdout__
         stdout_value = captured_output.getvalue()
         self.assertIn("YES", stdout_value)
-        self.assertIn("NO", stdout_value)
+        self.assertIn("NO", stdout_value)
+
+    def test_fixtures_image_types(self) -> None:
+        """Test various image input types in Jaclang."""
+        captured_output = io.StringIO()
+        sys.stdout = captured_output
+        jac_import("image_types", base_path=self.fixture_abs_path("./"))
+        sys.stdout = sys.__stdout__
+        stdout_value = captured_output.getvalue()
+        expected_labels = [
+            "PIL Image",
+            "Image from file path",
+            "Image from URL",
+            "Image from BytesIO",
+            "Image from raw bytes",
+            "Image from memoryview",
+            "Image from data URL",
+            "Image from PathLike",
+            "Image from file-like without getvalue",
+            "Image from bytearray",
+            "Image from gs:// URL",
+        ]
+        for label in expected_labels:
+            self.assertIn(label, stdout_value)