Merge pull request #31 from Azure-Samples/docx

Add markitdown example and optimize dev container for Codespaces

Author: pamelafox

.devcontainer/Dockerfile

@@ -0,0 +1,5 @@
+FROM mcr.microsoft.com/devcontainers/python:3.12-bookworm
+
+COPY requirements.txt /tmp/pip-tmp/
+RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+    && rm -rf /tmp/pip-tmp

.devcontainer/devcontainer.json

@@ -1,6 +1,9 @@
 {
-    "name": "azure-openai-keyless",
-    "image": "mcr.microsoft.com/devcontainers/python:3.12-bullseye",
+    "name": "azure-openai-entity-extraction",
+    "build": {
+        "dockerfile": "Dockerfile",
+        "context": ".."
+    },
     "features": {
         "ghcr.io/azure/azure-dev/azd:latest": {}
     },
@@ -15,6 +18,8 @@
                 "humao.rest-client"
             ],
             "python.defaultInterpreterPath": "/usr/local/bin/python",
+            "python.analysis.typeCheckingMode": "off",
+            "remote.autoForwardPorts": false,
             "[python]": {
                 "editor.formatOnSave": true,
                 "editor.codeActionsOnSave": {
@@ -24,9 +29,5 @@
 			}
         }
     },
-    "postCreateCommand": "pip install -r requirements.txt",
-    "remoteUser": "vscode",
-    "hostRequirements": {
-        "memory": "8gb"
-    }
+    "remoteUser": "vscode"
 }

basic_githubmodels.py

@@ -29,7 +29,7 @@ class CalendarEvent(BaseModel):
 )
 
 message = completion.choices[0].message
-if (message.refusal):
+if message.refusal:
     rich.print(message.refusal)
 else:
     rich.print(message.parsed)

example_doc.docx

@@ -59,7 +59,7 @@ class Receipt(BaseModel):
 completion = client.beta.chat.completions.parse(
     model=model_name,
     messages=[
-        {"role": "system", "content": "Extract the information the receipt"},
+        {"role": "system", "content": "Extract the information from the receipt"},
         {"role": "user", "content": md_text},
     ],
     response_format=Receipt,

extract_pdf_receipt.py

@@ -0,0 +1,65 @@
+import logging
+import os
+
+import openai
+from dotenv import load_dotenv
+from markitdown import MarkItDown
+from pydantic import BaseModel
+from rich import print
+
+logging.basicConfig(level=logging.WARNING)
+load_dotenv(override=True)
+
+if os.getenv("OPENAI_HOST", "github") == "azure":
+    import azure.identity
+
+    if not os.getenv("AZURE_OPENAI_SERVICE") or not os.getenv("AZURE_OPENAI_GPT_DEPLOYMENT"):
+        logging.warning("AZURE_OPENAI_SERVICE and AZURE_OPENAI_GPT_DEPLOYMENT env variables are empty. See README.")
+        exit(1)
+    credential = azure.identity.AzureDeveloperCliCredential(tenant_id=os.getenv("AZURE_TENANT_ID"))
+    token_provider = azure.identity.get_bearer_token_provider(
+        credential, "https://cognitiveservices.azure.com/.default"
+    )
+    client = openai.OpenAI(
+        base_url=f"https://{os.getenv('AZURE_OPENAI_SERVICE')}.openai.azure.com/openai/v1",
+        api_key=token_provider,
+    )
+    model_name = os.getenv("AZURE_OPENAI_GPT_DEPLOYMENT")
+else:
+    if not os.getenv("GITHUB_TOKEN"):
+        logging.warning("GITHUB_TOKEN env variable is empty. See README.")
+        exit(1)
+    client = openai.OpenAI(
+        base_url="https://models.github.ai/inference",
+        api_key=os.environ["GITHUB_TOKEN"],
+        default_query={"api-version": "2024-08-01-preview"},
+    )
+    model_name = "openai/gpt-4o"
+
+
+# Define models for Structured Outputs
+class DocumentMetadata(BaseModel):
+    title: str
+    author: str | None
+    headings: list[str]
+
+
+# Use markitdown to convert docx to markdown
+md = MarkItDown(enable_plugins=False)
+markdown_text = md.convert("example_doc.docx").text_content
+
+# Send request to LLM to extract using Structured Outputs
+completion = client.beta.chat.completions.parse(
+    model=model_name,
+    messages=[
+        {"role": "system", "content": "Extract the document title, author, and a list of all headings."},
+        {"role": "user", "content": markdown_text},
+    ],
+    response_format=DocumentMetadata,
+)
+
+message = completion.choices[0].message
+if message.refusal:
+    print(message.refusal)
+else:
+    print(message.parsed)

extract_word_docx.py

@@ -7,3 +7,4 @@ beautifulsoup4
 pymupdf4llm
 azure-ai-inference
 dotenv-azd
+markitdown[docx]