Merge pull request #499 from guardrails-ai/0.3.0-backup

0.3.0

Author: CalebCourier

.github/workflows/ci.yml

@@ -9,6 +9,7 @@ on:
         branches:
             - main
             - dev
+            - '0.3.0'
 
     # Allows you to run this workflow manually from the Actions tab
     workflow_dispatch:

.github/workflows/scripts/run_notebooks.sh

@@ -9,7 +9,8 @@ cd docs/examples
 # Function to process a notebook
 process_notebook() {
     notebook="$1"
-    if [ "$notebook" != "valid_chess_moves.ipynb" ] && [ "$notebook" != "translation_with_quality_check.ipynb" ] && [ "$notebook" != "competitors_check.ipynb" ]; then
+    invalid_notebooks=("valid_chess_moves.ipynb" "translation_with_quality_check.ipynb" "llamaindex-output-parsing.ipynb" "competitors_check.ipynb")
+    if [[ ! " ${invalid_notebooks[@]} " =~ " ${notebook} " ]]; then
         echo "Processing $notebook..."
         poetry run jupyter nbconvert --to notebook --execute "$notebook"
         if [ $? -ne 0 ]; then

.gitignore

@@ -20,6 +20,7 @@ dist/*
 .cache
 scratch/
 .coverage*
+coverage.xml
 test.db
 test.index
 htmlcov

Makefile

@@ -47,6 +47,9 @@ test-cov:
 view-test-cov:
 	poetry run pytest tests/ --cov=./guardrails/ --cov-report html && open htmlcov/index.html
 
+view-test-cov-file:
+	poetry run pytest tests/unit_tests/test_logger.py --cov=./guardrails/ --cov-report html && open htmlcov/index.html
+
 docs-serve:
 	poetry run mkdocs serve -a $(MKDOCS_SERVE_ADDR)
 
@@ -59,6 +62,9 @@ dev:
 full:
 	poetry install --all-extras
 
+self-install:
+	pip install -e .
+
 all: autoformat type lint docs test
 
 precommit:

README.md

@@ -154,7 +154,7 @@ Call the `Guard` object with the LLM API call as the first argument and add any
 import openai
 
 # Wrap the OpenAI API call with the `guard` object
-raw_llm_output, validated_output = guard(
+raw_llm_output, validated_output, *rest = guard(
     openai.Completion.create,
     engine="text-davinci-003",
     max_tokens=1024,

docs/api_reference/helper_classes.md

@@ -0,0 +1,4 @@
+::: guardrails.classes.generic
+    options:
+        members:
+            - "Stack"

docs/api_reference/history_and_logs.md

@@ -0,0 +1,8 @@
+::: guardrails.classes.history
+    options:
+        members:
+            - "Call"
+            - "CallInputs"
+            - "Inputs"
+            - "Iteration"
+            - "Outputs"

docs/concepts/guard.md

@@ -19,7 +19,7 @@ from guardrails import Guard
 
 guard = Guard.from_rail(...)
 
-raw_output, validated_output = guard(
+raw_output, validated_output, *rest = guard(
     openai.Completion.create,
     engine="text-davinci-003",
     max_tokens=1024,

docs/concepts/logs.md

@@ -1,78 +1,191 @@
 # Inspecting logs
 
-All `gd.Guard` calls are logged internally, and can be accessed via two methods, `gd.Guard.guard_state` or `guardrails.log`.
+All `Guard` calls are logged internally, and can be accessed via the guard history.
 
-## 🪵 Accessing logs via `guardrails.log`
+## 🇻🇦 Accessing logs via `Guard.history`
 
-This is the simplest way to access logs. It returns a list of all `gd.Guard` calls, in the order they were made.
+`history` is an attribute of the `Guard` class. It implements a standard `Stack` interface with a few extra helper methods and properties.  For more information on our `Stack` implementation see the [Helper Classes](/api_reference/helper_classes) page.
 
-In order to access logs, run:
+Each entry in the history stack is a `Call` log which will contain information specific to a particular `Guard.__call__` or `Guard.parse` call in the order that they were executed within the current session.
 
-```bash
-
-eliot-tree --output-format=ascii guardrails.log
+For example, if you have a guard:
 
+```py
+my_guard = Guard.from_rail(...)
 ```
 
-## 🇻🇦 Accessing logs via `gd.Guard.guard_state`
+and you call it multiple times:
+
+```py
+response_1 = my_guard(...)
 
-`guard_state` is an attribute of the `gd.Guard` class. It contains:
+response_2 = my_guard.parse(...)
+```
 
-1. A list of all `gd.Guard` calls, in the order they were made.
-2. For each call, reasks needed and their results.
+Then `guard.history` will have two call logs with the first representing the first call `response_1 = my_guard(...)` and the second representing the following `parse` call `response_2 = my_guard.parse(...)`.
 
-To pretty print logs, run:
+To pretty print logs for the latest call, run:
 
 ```python
 from rich import print
 
-print(guard.state.most_recent_call.tree)
+print(guard.history.last.tree)
 ```
+--8<--
 
-![guard_state](../img/guard_history.png)
+docs/html/single-step-history.html
 
-To access fine-grained logs on field validation, see the FieldValidationLogs object:
+--8<--
 
-```python
-validation_logs = guard.guard_state.all_histories[0].history[0].field_validation_logs
-print(validation_logs.json(indent=2))
+The `Call` log will contain initial and final information about a particular guard call.
+
+```py
+first_call = my_guard.history.first
+```
+
+For example, it tracks the initial inputs as provided:
+```py
+print("prompt\n-----")
+print(first_call.prompt)
+print("prompt params\n------------- ")
+print(first_call.prompt_params)
+```
+```log
+prompt
+-----
+
+You are a human in an enchanted forest. You come across opponents of different types. You should fight smaller opponents, run away from bigger ones, and freeze if the opponent is a bear.
+
+You run into a ${opp_type}. What do you do?
+
+${gr.complete_json_suffix_v2}
+
+
+Here are a few examples
+
+goblin: {"action": {"chosen_action": "fight", "weapon": "crossbow"}}
+troll: {"action": {"chosen_action": "fight", "weapon": "sword"}}
+giant: {"action": {"chosen_action": "flight", "flight_direction": "north", "distance": 1}}
+dragon: {"action": {"chosen_action": "flight", "flight_direction": "south", "distance": 4}}
+black bear: {"action": {"chosen_action": "freeze", "duration": 3}}
+beets: {"action": {"chosen_action": "fight", "weapon": "fork"}}
+
+prompt params
+------------- 
+{'opp_type': 'grizzly'}
 ```
 
-```json
+as well as the final outputs:
+```py
+print("status: ", first_call.status) # The final status of this guard call
+print("validated response:", first_call.validated_output) # The final valid output of this guard call
+```
+```log
+status:  pass
+validated response: {'action': {'chosen_action': 'freeze', 'duration': 3}}
+```
+
+
+The `Call` log also tracks cumulative values from any iterations that happen within the call.
+
+For example, if the first response from the LLM fails validation and a reask occurs, the `Call` log can provide total tokens consumed (*currently only for OpenAI models), as well as access to all of the raw outputs from the LLM:
+```py
+print("prompt token usage: ", first_call.prompt_tokens_consumed) # Total number of prompt tokens consumed across iterations within this call
+print("completion token usage: ", first_call.completion_tokens_consumed) # Total number of completion tokens consumed across iterations within this call
+print("total token usage: ",first_call.tokens_consumed) # Total number of tokens consumed; equal to the sum of the two values above
+print("llm responses\n-------------") # An Stack of the LLM responses in order that they were received
+for r in first_call.raw_outputs:
+  print(r)
+```
+```log
+prompt token usage:  909
+completion token usage:  57
+total token usage:  966
+
+llm responses
+-------------
+{"action": {"chosen_action": "freeze"}}
 {
-  "validator_logs": [],
-  "children": {
-    "name": {
-      "validator_logs": [
-        {
-          "validator_name": "TwoWords",
-          "value_before_validation": "peter parker the second",
-          "validation_result": {
-            "outcome": "fail",
-            "metadata": null,
-            "error_message": "must be exactly two words",
-            "fix_value": "peter parker"
-          },
-          "value_after_validation": {
-            "incorrect_value": "peter parker the second",
-            "fail_results": [
-              {
-                "outcome": "fail",
-                "metadata": null,
-                "error_message": "must be exactly two words",
-                "fix_value": "peter parker"
-              }
-            ],
-            "path": [
-              "name"
-            ]
-          }
-        }
-      ],
-      "children": {}
-    }
+  "action": {
+    "chosen_action": "freeze",
+    "duration": null
   }
 }
+{
+  "action": {
+    "chosen_action": "freeze",
+    "duration": 1
+  }
+}
+```
+
+For more information on `Call`, see the [History & Logs](/api_reference/history_and_logs/#guardrails.classes.history.Call) page.
+
+## 🇻🇦 Accessing logs from individual steps
+In addition to the cumulative values available directly on the `Call` log, it also contains a `Stack` of `Iteration`'s.  Each `Iteration` represent the logs from within a step in the guardrails process.  This includes the call to the LLM, as well as parsing and validating the LLM's response.
+
+Each `Iteration` is treated as a stateless entity so it will only contain information about the inputs and outputs of the particular step it represents.
+
+For example, in order to see the raw LLM response as well as the logs for the specific validations that failed during the first step of a call, we can access this information via that steps `Iteration`:
+
+```py
+first_step = first_call.iterations.first
 
+first_llm_output = first_step.raw_output
+print("First LLM response\n------------------")
+print(first_llm_output)
+print(" ")
+
+validation_logs = first_step.validator_logs
+print("\nValidator Logs\n--------------")
+for log in validation_logs:
+    print(log.json(indent=2))
+```
+```log
+First LLM response
+------------------
+{"action": {"chosen_action": "fight", "weapon": "spoon"}}
+ 
+
+Validator Logs
+--------------
+{
+  "validator_name": "ValidChoices",
+  "value_before_validation": "spoon",
+  "validation_result": {
+    "outcome": "fail",
+    "metadata": null,
+    "error_message": "Value spoon is not in choices ['crossbow', 'axe', 'sword', 'fork'].",
+    "fix_value": null
+  },
+  "value_after_validation": {
+    "incorrect_value": "spoon",
+    "fail_results": [
+      {
+        "outcome": "fail",
+        "metadata": null,
+        "error_message": "Value spoon is not in choices ['crossbow', 'axe', 'sword', 'fork'].",
+        "fix_value": null
+      }
+    ],
+    "path": [
+      "action",
+      "weapon"
+    ]
+  }
+}
+```
+
+Similar to the `Call` log, we can also see the token usage for just this step:
+```py
+print("prompt token usage: ", first_step.prompt_tokens_consumed)
+print("completion token usage: ", first_step.completion_tokens_consumed)
+print("token usage for this step: ",first_step.tokens_consumed)
+```
+```log
+prompt token usage:  617
+completion token usage:  16
+token usage for this step:  633
+```
 
-```
+For more information on the properties available on `Iteration`, ee the [History & Logs](/api_reference/history_and_logs/#guardrails.classes.history.Iteration) page.

docs/concepts/validators.md

@@ -35,7 +35,7 @@ Sometimes validators need addtional parameters that are only availble during run
 ```python
 guard = Guard.from_rail("my_railspec.rail")
 
-raw_output, guarded_output = guard(
+raw_output, guarded_output, *rest = guard(
     llm_api=openai.ChatCompletion.create,
     model="gpt-3.5-turbo",
     num_reasks=3,
@@ -134,7 +134,7 @@ ${guardrails.complete_json_suffix}
 
 guard = Guard.from_rail_string(rail_string=rail_str)
 
-raw_output, guarded_output = guard(
+raw_output, guarded_output, *rest = guard(
     llm_api=openai.ChatCompletion.create,
     model="gpt-3.5-turbo"
 )