From 98310fecace2dd5f2bb0342dd93e0e91b7da0c40 Mon Sep 17 00:00:00 2001
From: Wesley Lima <wesley@humansignal.com>
Date: Mon, 3 Nov 2025 12:49:56 -0500
Subject: [PATCH] Adding tutorial for multi-turn ai chat

---
 ...i_chat_using_label_studio_enterprise.ipynb | 1288 +++++++++++++++++
 1 file changed, 1288 insertions(+)
 create mode 100644 tutorials/multi_turn_feedback_for_ai_chat_using_label_studio_enterprise/multi_turn_feedback_for_ai_chat_using_label_studio_enterprise.ipynb

diff --git a/tutorials/multi_turn_feedback_for_ai_chat_using_label_studio_enterprise/multi_turn_feedback_for_ai_chat_using_label_studio_enterprise.ipynb b/tutorials/multi_turn_feedback_for_ai_chat_using_label_studio_enterprise/multi_turn_feedback_for_ai_chat_using_label_studio_enterprise.ipynb
new file mode 100644
index 0000000..3d2c13b
--- /dev/null
+++ b/tutorials/multi_turn_feedback_for_ai_chat_using_label_studio_enterprise/multi_turn_feedback_for_ai_chat_using_label_studio_enterprise.ipynb
@@ -0,0 +1,1288 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "id": "8b187611",
+      "metadata": {
+        "id": "8b187611"
+      },
+      "source": [
+        "# Multi-turn Feedback for AI Chat using Label Studio Enterprise\n",
+        "\n",
+        "This notebook demonstrates how to create a Label Studio Enterprise project for evaluating chatbot conversations using the Chatbot Evaluation template.\n",
+        "\n",
+        "The allows you to:\n",
+        "- Review multi-turn conversations\n",
+        "- Rate assistant responses for accuracy, clarity, and helpfulness\n",
+        "- Evaluate grounding in documentation\n",
+        "- Assess tone and style\n",
+        "- Track whether questions were answered\n",
+        "\n",
+        "Reference: [Chatbot Evaluation Template](https://docs.humansignal.com/templates/chatbot)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "f3a5bae0",
+      "metadata": {
+        "id": "f3a5bae0"
+      },
+      "source": [
+        "## Setup and Installation\n",
+        "\n",
+        "First, install the Label Studio SDK if you haven't already.\n",
+        "\n",
+        "For more information about the SDK, see the [Label Studio Python SDK documentation](https://labelstud.io/guide/sdk).\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "eb1c5401",
+      "metadata": {
+        "vscode": {
+          "languageId": "shellscript"
+        },
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "eb1c5401",
+        "outputId": "71d09d2c-1acb-4c0e-81aa-d55abc408ce5"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Collecting label-studio-sdk\n",
+            "  Downloading label_studio_sdk-2.0.11-py3-none-any.whl.metadata (7.0 kB)\n",
+            "Requirement already satisfied: Pillow>=11.3.0 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (11.3.0)\n",
+            "Collecting appdirs>=1.4.3 (from label-studio-sdk)\n",
+            "  Downloading appdirs-1.4.4-py2.py3-none-any.whl.metadata (9.0 kB)\n",
+            "Collecting datamodel-code-generator==0.26.1 (from label-studio-sdk)\n",
+            "  Downloading datamodel_code_generator-0.26.1-py3-none-any.whl.metadata (24 kB)\n",
+            "Requirement already satisfied: httpx>=0.21.2 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (0.28.1)\n",
+            "Collecting ijson>=3.2.3 (from label-studio-sdk)\n",
+            "  Downloading ijson-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)\n",
+            "Collecting jsf<0.12.0,>=0.11.2 (from label-studio-sdk)\n",
+            "  Downloading jsf-0.11.2-py3-none-any.whl.metadata (5.0 kB)\n",
+            "Requirement already satisfied: jsonschema>=4.23.0 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (4.25.1)\n",
+            "Requirement already satisfied: lxml>=4.2.5 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (5.4.0)\n",
+            "Requirement already satisfied: nltk<4.0.0,>=3.9.1 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (3.9.1)\n",
+            "Requirement already satisfied: numpy<3.0.0,>=1.26.4 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (2.0.2)\n",
+            "Requirement already satisfied: opencv-python<5.0.0,>=4.12.0 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (4.12.0.88)\n",
+            "Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (2.2.2)\n",
+            "Requirement already satisfied: pydantic>=1.9.2 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (2.11.9)\n",
+            "Requirement already satisfied: pydantic-core<3.0.0,>=2.18.2 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (2.33.2)\n",
+            "Requirement already satisfied: pyjwt<3.0.0,>=2.10.1 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (2.10.1)\n",
+            "Requirement already satisfied: requests>=2.22.0 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (2.32.4)\n",
+            "Collecting requests-mock==1.12.1 (from label-studio-sdk)\n",
+            "  Downloading requests_mock-1.12.1-py2.py3-none-any.whl.metadata (4.1 kB)\n",
+            "Requirement already satisfied: typing_extensions>=4.0.0 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (4.15.0)\n",
+            "Collecting ujson>=5.8.0 (from label-studio-sdk)\n",
+            "  Downloading ujson-5.11.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (9.4 kB)\n",
+            "Requirement already satisfied: urllib3>=2.5.0 in /usr/local/lib/python3.12/dist-packages (from label-studio-sdk) (2.5.0)\n",
+            "Collecting xmljson==0.2.1 (from label-studio-sdk)\n",
+            "  Downloading xmljson-0.2.1-py2.py3-none-any.whl.metadata (12 kB)\n",
+            "Collecting argcomplete<4.0,>=1.10 (from datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading argcomplete-3.6.2-py3-none-any.whl.metadata (16 kB)\n",
+            "Collecting black>=19.10b0 (from datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading black-25.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl.metadata (83 kB)\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.5/83.5 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hCollecting genson<2.0,>=1.2.1 (from datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading genson-1.3.0-py3-none-any.whl.metadata (28 kB)\n",
+            "Collecting inflect<6.0,>=4.1.0 (from datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading inflect-5.6.2-py3-none-any.whl.metadata (21 kB)\n",
+            "Collecting isort<6.0,>=4.3.21 (from datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading isort-5.13.2-py3-none-any.whl.metadata (12 kB)\n",
+            "Requirement already satisfied: jinja2<4.0,>=2.10.1 in /usr/local/lib/python3.12/dist-packages (from datamodel-code-generator==0.26.1->label-studio-sdk) (3.1.6)\n",
+            "Requirement already satisfied: packaging in /usr/local/lib/python3.12/dist-packages (from datamodel-code-generator==0.26.1->label-studio-sdk) (25.0)\n",
+            "Requirement already satisfied: pyyaml>=6.0.1 in /usr/local/lib/python3.12/dist-packages (from datamodel-code-generator==0.26.1->label-studio-sdk) (6.0.3)\n",
+            "Requirement already satisfied: anyio in /usr/local/lib/python3.12/dist-packages (from httpx>=0.21.2->label-studio-sdk) (4.11.0)\n",
+            "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx>=0.21.2->label-studio-sdk) (2025.8.3)\n",
+            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx>=0.21.2->label-studio-sdk) (1.0.9)\n",
+            "Requirement already satisfied: idna in /usr/local/lib/python3.12/dist-packages (from httpx>=0.21.2->label-studio-sdk) (3.10)\n",
+            "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx>=0.21.2->label-studio-sdk) (0.16.0)\n",
+            "Collecting faker>=15.3.4 (from jsf<0.12.0,>=0.11.2->label-studio-sdk)\n",
+            "  Downloading faker-37.8.0-py3-none-any.whl.metadata (15 kB)\n",
+            "Collecting rstr>=3.2.0 (from jsf<0.12.0,>=0.11.2->label-studio-sdk)\n",
+            "  Downloading rstr-3.2.2-py3-none-any.whl.metadata (7.1 kB)\n",
+            "Requirement already satisfied: smart-open>=6.3.0 in /usr/local/lib/python3.12/dist-packages (from smart-open[http]>=6.3.0->jsf<0.12.0,>=0.11.2->label-studio-sdk) (7.3.1)\n",
+            "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.12/dist-packages (from jsonschema>=4.23.0->label-studio-sdk) (25.3.0)\n",
+            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.12/dist-packages (from jsonschema>=4.23.0->label-studio-sdk) (2025.9.1)\n",
+            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.12/dist-packages (from jsonschema>=4.23.0->label-studio-sdk) (0.36.2)\n",
+            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.12/dist-packages (from jsonschema>=4.23.0->label-studio-sdk) (0.27.1)\n",
+            "Requirement already satisfied: click in /usr/local/lib/python3.12/dist-packages (from nltk<4.0.0,>=3.9.1->label-studio-sdk) (8.3.0)\n",
+            "Requirement already satisfied: joblib in /usr/local/lib/python3.12/dist-packages (from nltk<4.0.0,>=3.9.1->label-studio-sdk) (1.5.2)\n",
+            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.12/dist-packages (from nltk<4.0.0,>=3.9.1->label-studio-sdk) (2024.11.6)\n",
+            "Requirement already satisfied: tqdm in /usr/local/lib/python3.12/dist-packages (from nltk<4.0.0,>=3.9.1->label-studio-sdk) (4.67.1)\n",
+            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas>=0.24.0->label-studio-sdk) (2.9.0.post0)\n",
+            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas>=0.24.0->label-studio-sdk) (2025.2)\n",
+            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas>=0.24.0->label-studio-sdk) (2025.2)\n",
+            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic>=1.9.2->label-studio-sdk) (0.7.0)\n",
+            "Requirement already satisfied: typing-inspection>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from pydantic>=1.9.2->label-studio-sdk) (0.4.2)\n",
+            "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests>=2.22.0->label-studio-sdk) (3.4.3)\n",
+            "Collecting mypy-extensions>=0.4.3 (from black>=19.10b0->datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading mypy_extensions-1.1.0-py3-none-any.whl.metadata (1.1 kB)\n",
+            "Collecting pathspec>=0.9.0 (from black>=19.10b0->datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading pathspec-0.12.1-py3-none-any.whl.metadata (21 kB)\n",
+            "Requirement already satisfied: platformdirs>=2 in /usr/local/lib/python3.12/dist-packages (from black>=19.10b0->datamodel-code-generator==0.26.1->label-studio-sdk) (4.4.0)\n",
+            "Collecting pytokens>=0.1.10 (from black>=19.10b0->datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading pytokens-0.1.10-py3-none-any.whl.metadata (2.0 kB)\n",
+            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2<4.0,>=2.10.1->datamodel-code-generator==0.26.1->label-studio-sdk) (3.0.3)\n",
+            "Collecting email-validator>=2.0.0 (from pydantic[email]!=2.0.0,!=2.0.1,!=2.4.0,<3.0,>=1.10.0; python_version >= \"3.12\" and python_version < \"4.0\"->datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading email_validator-2.3.0-py3-none-any.whl.metadata (26 kB)\n",
+            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas>=0.24.0->label-studio-sdk) (1.17.0)\n",
+            "Requirement already satisfied: wrapt in /usr/local/lib/python3.12/dist-packages (from smart-open>=6.3.0->smart-open[http]>=6.3.0->jsf<0.12.0,>=0.11.2->label-studio-sdk) (1.17.3)\n",
+            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from anyio->httpx>=0.21.2->label-studio-sdk) (1.3.1)\n",
+            "Collecting dnspython>=2.0.0 (from email-validator>=2.0.0->pydantic[email]!=2.0.0,!=2.0.1,!=2.4.0,<3.0,>=1.10.0; python_version >= \"3.12\" and python_version < \"4.0\"->datamodel-code-generator==0.26.1->label-studio-sdk)\n",
+            "  Downloading dnspython-2.8.0-py3-none-any.whl.metadata (5.7 kB)\n",
+            "Downloading label_studio_sdk-2.0.11-py3-none-any.whl (600 kB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m600.3/600.3 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading datamodel_code_generator-0.26.1-py3-none-any.whl (111 kB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.0/111.0 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading requests_mock-1.12.1-py2.py3-none-any.whl (27 kB)\n",
+            "Downloading xmljson-0.2.1-py2.py3-none-any.whl (10 kB)\n",
+            "Downloading appdirs-1.4.4-py2.py3-none-any.whl (9.6 kB)\n",
+            "Downloading ijson-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (148 kB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m148.3/148.3 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading jsf-0.11.2-py3-none-any.whl (49 kB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.3/49.3 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading ujson-5.11.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (57 kB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.4/57.4 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading argcomplete-3.6.2-py3-none-any.whl (43 kB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading black-25.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl (1.7 MB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m44.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading faker-37.8.0-py3-none-any.whl (2.0 MB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m52.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading genson-1.3.0-py3-none-any.whl (21 kB)\n",
+            "Downloading inflect-5.6.2-py3-none-any.whl (33 kB)\n",
+            "Downloading isort-5.13.2-py3-none-any.whl (92 kB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.3/92.3 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hDownloading rstr-3.2.2-py3-none-any.whl (10 kB)\n",
+            "Downloading email_validator-2.3.0-py3-none-any.whl (35 kB)\n",
+            "Downloading mypy_extensions-1.1.0-py3-none-any.whl (5.0 kB)\n",
+            "Downloading pathspec-0.12.1-py3-none-any.whl (31 kB)\n",
+            "Downloading pytokens-0.1.10-py3-none-any.whl (12 kB)\n",
+            "Downloading dnspython-2.8.0-py3-none-any.whl (331 kB)\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m331.1/331.1 kB\u001b[0m \u001b[31m18.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25hInstalling collected packages: xmljson, genson, appdirs, ujson, rstr, pytokens, pathspec, mypy-extensions, isort, inflect, ijson, faker, dnspython, argcomplete, requests-mock, email-validator, black, jsf, datamodel-code-generator, label-studio-sdk\n",
+            "  Attempting uninstall: inflect\n",
+            "    Found existing installation: inflect 7.5.0\n",
+            "    Uninstalling inflect-7.5.0:\n",
+            "      Successfully uninstalled inflect-7.5.0\n",
+            "Successfully installed appdirs-1.4.4 argcomplete-3.6.2 black-25.9.0 datamodel-code-generator-0.26.1 dnspython-2.8.0 email-validator-2.3.0 faker-37.8.0 genson-1.3.0 ijson-3.4.0 inflect-5.6.2 isort-5.13.2 jsf-0.11.2 label-studio-sdk-2.0.11 mypy-extensions-1.1.0 pathspec-0.12.1 pytokens-0.1.10 requests-mock-1.12.1 rstr-3.2.2 ujson-5.11.0 xmljson-0.2.1\n"
+          ]
+        }
+      ],
+      "source": [
+        "%pip install label-studio-sdk"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "e39a041e",
+      "metadata": {
+        "id": "e39a041e"
+      },
+      "source": [
+        "## Configure Credentials\n",
+        "To support loading credentials from Google Colab Secrets with fallback to .env and environment sourced variables the following cell can be used."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "85a7aa7b",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "85a7aa7b",
+        "outputId": "97cc6c56-5546-4387-ffac-225ccaeb13f5"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.12/dist-packages (1.1.1)\n"
+          ]
+        }
+      ],
+      "source": [
+        "%pip install python-dotenv\n",
+        "\n",
+        "# Load configuration with Google Colab Secrets support + fallback\n",
+        "IS_GOOGLE_COLAB = False\n",
+        "\n",
+        "# Load from .env file if available (for local development)\n",
+        "try:\n",
+        "    from dotenv import load_dotenv\n",
+        "    load_dotenv()\n",
+        "except:\n",
+        "    pass  # will use system env vars\n",
+        "\n",
+        "def get_credential(key, default=None):\n",
+        "    global IS_GOOGLE_COLAB\n",
+        "    \"\"\"Get credential from Colab Secrets first, then environment variables\"\"\"\n",
+        "    try:\n",
+        "        # Try Google Colab Secrets first (most secure)\n",
+        "        from google.colab import userdata\n",
+        "        IS_GOOGLE_COLAB = True\n",
+        "        return userdata.get(key)\n",
+        "    except:\n",
+        "        from os import environ\n",
+        "        IS_GOOGLE_COLAB = False\n",
+        "        # Fallback to environment variables (for local Jupyter)\n",
+        "        return environ.get(key, default)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "1430bf66",
+      "metadata": {
+        "id": "1430bf66"
+      },
+      "source": [
+        "Set your environment variables before running:\n",
+        "\n",
+        "```bash\n",
+        "export LABEL_STUDIO_URL=\"https://app.humansignal.com\"  # or your Label Studio URL\n",
+        "export LABEL_STUDIO_API_KEY=\"your-api-key-here\"\n",
+        "```\n",
+        "\n",
+        "**How to get your API key:**\n",
+        "1. Open Label Studio in your browser\n",
+        "2. Click on your profile (top-right)\n",
+        "3. Go to \"Account & Settings\"\n",
+        "4. Click \"Access Token\" (or \"Personal Access Token\")\n",
+        "5. Copy the existing token or create a new one\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "35f4e423",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 211
+        },
+        "id": "35f4e423",
+        "outputId": "8b03126e-b135-492a-d95f-dfb99457d1f5"
+      },
+      "outputs": [
+        {
+          "output_type": "error",
+          "ename": "ValueError",
+          "evalue": "❌ Please set LABEL_STUDIO_API_KEY environment variable.",
+          "traceback": [
+            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+            "\u001b[0;32m/tmp/ipython-input-3282658643.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mls_api_key\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'❌ Please set LABEL_STUDIO_API_KEY environment variable.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;31m# Connect to Label Studio\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+            "\u001b[0;31mValueError\u001b[0m: ❌ Please set LABEL_STUDIO_API_KEY environment variable."
+          ]
+        }
+      ],
+      "source": [
+        "import os\n",
+        "from label_studio_sdk import LabelStudio\n",
+        "\n",
+        "# Get credentials from environment variables\n",
+        "ls_api_key = os.environ.get('LABEL_STUDIO_API_KEY')\n",
+        "ls_url = os.environ.get('LABEL_STUDIO_URL', 'https://app.humansignal.com')\n",
+        "\n",
+        "if not ls_api_key:\n",
+        "    raise ValueError('❌ Please set LABEL_STUDIO_API_KEY environment variable.')\n",
+        "\n",
+        "# Connect to Label Studio\n",
+        "try:\n",
+        "    ls = LabelStudio(base_url=ls_url, api_key=ls_api_key)\n",
+        "    print(f'✅ Connected to Label Studio at {ls_url}')\n",
+        "except Exception as e:\n",
+        "    raise ConnectionError(f'❌ Failed to connect to Label Studio: {str(e)}')\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "0fa79674",
+      "metadata": {
+        "id": "0fa79674"
+      },
+      "source": [
+        "## Define the Chatbot Evaluation Label Config\n",
+        "\n",
+        "This is the label config from the [Evaluate Production Conversations for RLHF\n",
+        "](https://docs.humansignal.com/templates/chat_rlhf) example. It includes:\n",
+        "- A chat interface for viewing conversations\n",
+        "- Overall quality of message rating\n",
+        "- Additinal comments"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "075fd9a6",
+      "metadata": {
+        "id": "075fd9a6",
+        "outputId": "9e8722a9-3a59-44e3-cb6e-7d9906b90f9a"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Label config loaded successfully\n"
+          ]
+        }
+      ],
+      "source": [
+        "LABEL_CONFIG = \"\"\"\n",
+        "<View>\n",
+        "  <Style>\n",
+        "    .chat {\n",
+        "      border: 1px solid #ccc;\n",
+        "      padding: 10px;\n",
+        "      border-radius: 5px;\n",
+        "    }\n",
+        "    .evaluation {\n",
+        "        border: 2px solid #cc854f;\n",
+        "        background-color: #ffe4d0;\n",
+        "        color: #664228;\n",
+        "        padding: 10px;\n",
+        "        border-radius: 5px;\n",
+        "        margin-bottom: 20px;\n",
+        "    }\n",
+        "    <!-- Choice text -->\n",
+        "    .evaluation span {\n",
+        "        color: #664228;\n",
+        "    }\n",
+        "    <!-- Star rating -->\n",
+        "    .evaluation .ant-rate-star.ant-rate-star-full span {\n",
+        "      color: #f4aa2a;\n",
+        "     }\n",
+        "\n",
+        "    <!-- Dark mode comment text and button color -->\n",
+        "    [data-color-scheme=\"dark\"] .evaluation .lsf-row p,\n",
+        "    [data-color-scheme=\"dark\"] .evaluation button span {\n",
+        "       color: #f9f8f6\n",
+        "    }\n",
+        "\n",
+        "    .overall-chat {\n",
+        "       border-bottom: 1px solid #cc854f;\n",
+        "       margin-bottom: 15px;\n",
+        "    }\n",
+        "    .instructions {\n",
+        "       color: #664228;\n",
+        "       background-color: #ffe4d0;\n",
+        "       padding-top: 15px;\n",
+        "       padding-bottom: 15px;\n",
+        "    }\n",
+        "    <!-- Allow enlarging the instruction text -->\n",
+        "    .lsf-richtext__container.lsf-htx-richtext {\n",
+        "      font-size: 16px !important;\n",
+        "      line-height: 1.6;\n",
+        "    }\n",
+        "\n",
+        "    <!-- Remove excess height from the chat to allow space for instruction text -->\n",
+        "    .htx-chat {\n",
+        "      --excess-height: 275px\n",
+        "    }\n",
+        "  </Style>\n",
+        "  <View style=\"display: flex; gap: 24px;\">\n",
+        "\n",
+        "    <!-- Left: conversation -->\n",
+        "    <View className=\"chat\" style=\"flex: 2;\">\n",
+        "      <View className=\"instructions\">\n",
+        "        <Text name=\"instructions\" value=\"Review the conversation in detail.\n",
+        "                                         As you read through it, click on individual messages to\n",
+        "                                         provide feedback about accuracy, clarity, and intent.\" />\n",
+        "      </View>\n",
+        "\n",
+        "      <Chat name=\"chat\" value=\"$chat\"\n",
+        "            minMessages=\"2\"\n",
+        "            editable=\"false\" />\n",
+        "    </View>\n",
+        "\n",
+        "    <!-- Right: conversation-level evaluation -->\n",
+        "    <View style=\"flex: 1;\" className=\"evaluation\">\n",
+        "      <View style=\"position:sticky;top:14px\">\n",
+        "\n",
+        "          <!-- Evaluate the whole conversation -->\n",
+        "      <View className=\"overall-chat\" style=\"margin-top:auto\">\n",
+        "        <Header size=\"4\">Overall quality of this conversation</Header>\n",
+        "        <Rating name=\"rating\" toName=\"chat\" />\n",
+        "                <View style=\"padding-top:15px\">\n",
+        "          <Text name=\"add_comment\" value=\"Add a comment (optional)\" />\n",
+        "          <TextArea name=\"conversation_comment\" toName=\"chat\" />\n",
+        "                </View>\n",
+        "      </View>\n",
+        "        <!-- Only visible when no message is selected -->\n",
+        "         <View visibleWhen=\"no-region-selected\">\n",
+        "          <View style=\"padding-top:15px\">\n",
+        "          </View>\n",
+        "        </View>\n",
+        "\n",
+        "        <!-- Only visible when a user message is selected, and only applies to selected message -->\n",
+        "        <View visibleWhen=\"region-selected\" whenRole=\"user\">\n",
+        "          <Header value=\"Classify the user message\"/>\n",
+        "          <Choices name=\"request_classification\" toName=\"chat\" perRegion=\"true\" >\n",
+        "            <Choice value=\"Question\" />\n",
+        "            <Choice value=\"Clarifying Question\" />\n",
+        "            <Choice value=\"Command or Request\" />\n",
+        "            <Choice value=\"Positive Feedback\" />\n",
+        "            <Choice value=\"Negative Feedback\" />\n",
+        "            <Choice value=\"Off-topic / Chit-chat\" />\n",
+        "          </Choices>\n",
+        "       </View>\n",
+        "\n",
+        "        <!-- Only visible when an assistant message is selected, and only applies to selected message -->\n",
+        "        <View visibleWhen=\"region-selected\" whenRole=\"assistant\">\n",
+        "          <Header value=\"Rate assistant's clarity\"/>\n",
+        "          <Rating name=\"assistant_response_clarity\" toName=\"chat\" perRegion=\"true\" />\n",
+        "\n",
+        "          <Header value=\"Rate assistant's accuracy\"/>\n",
+        "          <Rating name=\"assistant_response_accuracy\" toName=\"chat\" perRegion=\"true\" />\n",
+        "\n",
+        "          <Header value=\"Classify the message tone\"/>\n",
+        "          <Choices name=\"q\" toName=\"chat\" perRegion=\"true\" >\n",
+        "            <Choice value=\"Professional\" />\n",
+        "            <Choice value=\"Casual\" />\n",
+        "          </Choices>\n",
+        "\n",
+        "          <Header value=\"Add a comment (optional)\"/>\n",
+        "          <TextArea perRegion=\"true\" name=\"message_comment\" toName=\"chat\" />\n",
+        "       </View>\n",
+        "     </View>\n",
+        "   </View>\n",
+        " </View>\n",
+        "</View>\n",
+        "\"\"\"\n",
+        "\n",
+        "print(\"Label config loaded successfully\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "93eef46d",
+      "metadata": {
+        "id": "93eef46d"
+      },
+      "source": [
+        "With the label config set, we now use it to create the Chat Evaluation project"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "a009ed8a",
+      "metadata": {
+        "id": "a009ed8a"
+      },
+      "outputs": [],
+      "source": [
+        "## Create Project with Label Config\n",
+        "\n",
+        "# Define project parameters\n",
+        "PROJECT_TITLE = \"Chatbot Conversation Evaluation\"\n",
+        "PROJECT_DESCRIPTION = \"Evaluate multi-turn chatbot conversations for accuracy, clarity, and helpfulness\"\n",
+        "\n",
+        "# Create the project using Label Studio SDK\n",
+        "project = ls.projects.create(\n",
+        "    title=PROJECT_TITLE,\n",
+        "    description=PROJECT_DESCRIPTION,\n",
+        "    label_config=LABEL_CONFIG\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "143cbad3",
+      "metadata": {
+        "id": "143cbad3"
+      },
+      "outputs": [],
+      "source": [
+        "## Get Project ID and URL\n",
+        "\n",
+        "# Store project ID and build direct URL\n",
+        "project_id = project.id\n",
+        "project_url = f\"{ls_url}/projects/{project_id}\"\n",
+        "\n",
+        "# Save project ID to .env file\n",
+        "with open('.env', 'a') as f:\n",
+        "    f.write(f\"LABEL_STUDIO_PROJECT_ID={project_id}\\n\")\n",
+        "\n",
+        "print(f\"📋 Project Details:\")\n",
+        "print(f\"   ID: {project_id}\")\n",
+        "print(f\"   Direct URL: {project_url}\")\n",
+        "print(f\"\\n🔗 Click here to open the project:\")\n",
+        "print(f\"   {project_url}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "b86a65f6",
+      "metadata": {
+        "id": "b86a65f6"
+      },
+      "source": [
+        "## Part 2: Set Up Chainlit Integration\n",
+        "\n",
+        "Now we'll set up a Chainlit chatbot that automatically syncs conversations to Label Studio.\n",
+        "\n",
+        "### What We'll Build\n",
+        "- A chatbot UI using Chainlit\n",
+        "- Automatic conversation logging to JSON\n",
+        "- Auto-sync to Label Studio when users disconnect\n",
+        "- Support for conversation resumption with versioning"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "46f30999",
+      "metadata": {
+        "id": "46f30999"
+      },
+      "source": [
+        "### Step 1: Install Additional Dependencies\n",
+        "\n",
+        "We need Chainlit for the chat UI and Ollama for a local LLM.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "629f5a18",
+      "metadata": {
+        "id": "629f5a18"
+      },
+      "outputs": [],
+      "source": [
+        "%pip install chainlit ollama openai anthropic\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "28480223",
+      "metadata": {
+        "id": "28480223"
+      },
+      "source": [
+        "### Step 2: Create Helper Files\n",
+        "\n",
+        "We'll create three Python files:\n",
+        "1. `conversation_logger.py` - Saves conversations to JSON\n",
+        "2. `auto_sync.py` - Automatically syncs to Label Studio\n",
+        "3. `chatbot_ui_auto_sync.py` - Main chatbot application\n",
+        "\n",
+        "**Note:** Run these cells to create the files in your working directory.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "d4f6c7d3",
+      "metadata": {
+        "id": "d4f6c7d3"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile conversation_logger.py\n",
+        "\"\"\"Conversation logger for saving chats to JSON\"\"\"\n",
+        "import json\n",
+        "from pathlib import Path\n",
+        "from datetime import datetime\n",
+        "from typing import List, Dict, Optional\n",
+        "\n",
+        "\n",
+        "class ConversationLogger:\n",
+        "    \"\"\"Logs conversations to JSON files\"\"\"\n",
+        "\n",
+        "    def __init__(self, output_dir: Path = Path(\"data/conversations\")):\n",
+        "        self.output_dir = output_dir\n",
+        "        self.output_dir.mkdir(parents=True, exist_ok=True)\n",
+        "\n",
+        "    def save_conversation(\n",
+        "        self,\n",
+        "        messages: List[Dict[str, str]],\n",
+        "        session_id: str,\n",
+        "        model: str,\n",
+        "        metadata: Optional[Dict] = None\n",
+        "    ) -> Path:\n",
+        "        \"\"\"Save conversation to JSON file\"\"\"\n",
+        "\n",
+        "        # Check if metadata contains auto_save flag\n",
+        "        is_auto_save = metadata and metadata.get('auto_save', False)\n",
+        "\n",
+        "        conversation_data = {\n",
+        "            \"session_id\": session_id,\n",
+        "            \"timestamp\": datetime.utcnow().isoformat() + \"Z\",\n",
+        "            \"model\": model,\n",
+        "            \"messages\": messages,\n",
+        "            \"turn_count\": len([m for m in messages if m[\"role\"] == \"user\"]),\n",
+        "            \"metadata\": {k: v for k, v in (metadata or {}).items() if k != 'auto_save'}\n",
+        "        }\n",
+        "\n",
+        "        # For auto-save: use session ID only (continuous updates)\n",
+        "        # For manual save: add timestamp (creates snapshot)\n",
+        "        if is_auto_save:\n",
+        "            filename = f\"conversation_{session_id}.json\"\n",
+        "        else:\n",
+        "            filename = f\"conversation_{session_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json\"\n",
+        "\n",
+        "        filepath = self.output_dir / filename\n",
+        "\n",
+        "        with open(filepath, 'w', encoding='utf-8') as f:\n",
+        "            json.dump(conversation_data, f, indent=2, ensure_ascii=False)\n",
+        "\n",
+        "        return filepath\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "ff91f69c",
+      "metadata": {
+        "id": "ff91f69c"
+      },
+      "source": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "fedd9ffc",
+      "metadata": {
+        "id": "fedd9ffc"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile auto_sync.py\n",
+        "\"\"\"Automatic Label Studio sync helper\"\"\"\n",
+        "import os\n",
+        "import json\n",
+        "from pathlib import Path\n",
+        "from typing import Optional\n",
+        "from datetime import datetime\n",
+        "from label_studio_sdk.client import LabelStudio\n",
+        "from dotenv import load_dotenv\n",
+        "load_dotenv()\n",
+        "\n",
+        "\n",
+        "class LabelStudioSync:\n",
+        "    \"\"\"Helper class to push conversations to Label Studio\"\"\"\n",
+        "\n",
+        "    def __init__(\n",
+        "        self,\n",
+        "        url: Optional[str] = None,\n",
+        "        api_key: Optional[str] = None,\n",
+        "        project_id: Optional[int] = None\n",
+        "    ):\n",
+        "        self.url = url or os.getenv('LABEL_STUDIO_URL', 'https://app.humansignal.com')\n",
+        "        self.api_key = api_key or os.getenv('LABEL_STUDIO_API_KEY')\n",
+        "        self.project_id = project_id or int(os.getenv('LABEL_STUDIO_PROJECT_ID', None))\n",
+        "\n",
+        "\n",
+        "        if not self.project_id:\n",
+        "            print(\"⚠️  LABEL_STUDIO_PROJECT_ID not set - auto-sync disabled\")\n",
+        "            self.client = None\n",
+        "\n",
+        "        if not self.url:\n",
+        "            print(\"⚠️  LABEL_STUDIO_URL not set - auto-sync disabled\")\n",
+        "            self.client = None\n",
+        "\n",
+        "        if not self.api_key:\n",
+        "            print(\"⚠️  LABEL_STUDIO_API_KEY not set - auto-sync disabled\")\n",
+        "            self.client = None\n",
+        "        else:\n",
+        "            self.client = LabelStudio(base_url=self.url, api_key=self.api_key)\n",
+        "\n",
+        "    def is_enabled(self) -> bool:\n",
+        "        \"\"\"Check if auto-sync is enabled\"\"\"\n",
+        "        return self.client is not None and self.project_id > 0\n",
+        "\n",
+        "    async def push_conversation(self, conversation_file: Path) -> bool:\n",
+        "        \"\"\"Push a single conversation to Label Studio\"\"\"\n",
+        "        if not self.is_enabled():\n",
+        "            return False\n",
+        "\n",
+        "        try:\n",
+        "            # Load conversation\n",
+        "            with open(conversation_file, 'r') as f:\n",
+        "                data = json.load(f)\n",
+        "\n",
+        "            # Format as Label Studio task\n",
+        "            task = {\n",
+        "                'data': {\n",
+        "                    'chat': data['messages'],  # Changed from 'messages' to 'chat' to match label config\n",
+        "                    'text': 'Review the conversation below and evaluate the quality of the chat interaction.',\n",
+        "                    'session_id': data.get('session_id', 'unknown'),\n",
+        "                    'thread_id': data.get('metadata', {}).get('thread_id', data.get('session_id')),\n",
+        "                    'model': data.get('model', 'unknown'),\n",
+        "                    'turn_count': data.get('turn_count', 0),\n",
+        "                    'timestamp': data.get('timestamp', ''),\n",
+        "                    'version': data.get('metadata', {}).get('version', 1),\n",
+        "                },\n",
+        "                'meta': {\n",
+        "                    'filename': conversation_file.name,\n",
+        "                    'imported_at': datetime.utcnow().isoformat() + 'Z',\n",
+        "                    'auto_synced': True\n",
+        "                }\n",
+        "            }\n",
+        "\n",
+        "            # Check if already imported\n",
+        "            session_id = data.get('session_id')\n",
+        "            existing = self.client.tasks.list(project=self.project_id)\n",
+        "\n",
+        "            for existing_task in existing:\n",
+        "                if hasattr(existing_task, 'data') and \\\n",
+        "                   existing_task.data.get('session_id') == session_id:\n",
+        "                    print(f\"⏭️  Session {session_id} already in Label Studio\")\n",
+        "                    return False\n",
+        "\n",
+        "            # Import task\n",
+        "            self.client.projects.import_tasks(id=self.project_id, request=[task])\n",
+        "            print(f\"✅ Auto-synced {session_id} to Label Studio\")\n",
+        "            return True\n",
+        "\n",
+        "        except Exception as e:\n",
+        "            print(f\"❌ Failed to sync {conversation_file.name}: {e}\")\n",
+        "            return False\n",
+        "\n",
+        "\n",
+        "# Global instance\n",
+        "_sync = None\n",
+        "\n",
+        "def get_sync() -> LabelStudioSync:\n",
+        "    \"\"\"Get the global sync instance\"\"\"\n",
+        "    global _sync\n",
+        "    if _sync is None:\n",
+        "        _sync = LabelStudioSync()\n",
+        "    return _sync\n",
+        "\n",
+        "\n",
+        "async def auto_push_conversation(conversation_file: Path):\n",
+        "    \"\"\"Push a conversation to Label Studio (async wrapper)\"\"\"\n",
+        "    sync = get_sync()\n",
+        "    if sync.is_enabled():\n",
+        "        await sync.push_conversation(conversation_file)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "82f14666",
+      "metadata": {
+        "id": "82f14666"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile chatbot_ui_auto_sync.py\n",
+        "\"\"\"\n",
+        "Chainlit Chatbot with Automatic Label Studio Sync\n",
+        "Handles resumed conversations with versioned tasks\n",
+        "\"\"\"\n",
+        "import os\n",
+        "import uuid\n",
+        "import json\n",
+        "from pathlib import Path\n",
+        "from datetime import datetime\n",
+        "from typing import List, Dict, Optional\n",
+        "from dotenv import load_dotenv\n",
+        "load_dotenv()\n",
+        "\n",
+        "import chainlit as cl\n",
+        "\n",
+        "try:\n",
+        "    import openai\n",
+        "    OPENAI_AVAILABLE = True\n",
+        "except ImportError:\n",
+        "    OPENAI_AVAILABLE = False\n",
+        "\n",
+        "try:\n",
+        "    import anthropic\n",
+        "    ANTHROPIC_AVAILABLE = True\n",
+        "except ImportError:\n",
+        "    ANTHROPIC_AVAILABLE = False\n",
+        "\n",
+        "try:\n",
+        "    import ollama\n",
+        "    OLLAMA_AVAILABLE = True\n",
+        "except ImportError:\n",
+        "    OLLAMA_AVAILABLE = False\n",
+        "\n",
+        "from conversation_logger import ConversationLogger\n",
+        "from auto_sync import get_sync\n",
+        "\n",
+        "\n",
+        "# Configuration\n",
+        "MIN_TURNS_FOR_SYNC = 2  # Minimum conversation length to sync\n",
+        "\n",
+        "\n",
+        "def get_available_models() -> Dict[str, List[str]]:\n",
+        "    \"\"\"Return available models by provider\"\"\"\n",
+        "    models = {}\n",
+        "\n",
+        "    if OPENAI_AVAILABLE and os.getenv(\"OPENAI_API_KEY\"):\n",
+        "        models[\"OpenAI\"] = [\"gpt-4\", \"gpt-3.5-turbo\"]\n",
+        "\n",
+        "    if ANTHROPIC_AVAILABLE and os.getenv(\"ANTHROPIC_API_KEY\"):\n",
+        "        models[\"Anthropic\"] = [\"claude-3-sonnet-20240229\"]\n",
+        "\n",
+        "    if OLLAMA_AVAILABLE:\n",
+        "        try:\n",
+        "            ollama_models = ollama.list()\n",
+        "            if ollama_models and ollama_models.get('models'):\n",
+        "                models[\"Ollama\"] = [m['name'] for m in ollama_models['models']]\n",
+        "            else:\n",
+        "                models[\"Ollama\"] = [\"llama3.2:3b\"]\n",
+        "        except:\n",
+        "            models[\"Ollama\"] = [\"llama3.2:3b\"]\n",
+        "\n",
+        "    return models\n",
+        "\n",
+        "\n",
+        "async def generate_response(messages: List[Dict[str, str]], model: str) -> str:\n",
+        "    \"\"\"Generate response from specified model\"\"\"\n",
+        "    provider, model_name = model.split(\"/\", 1)\n",
+        "\n",
+        "    if provider == \"OpenAI\":\n",
+        "        client = openai.OpenAI()\n",
+        "        response = client.chat.completions.create(\n",
+        "            model=model_name,\n",
+        "            messages=messages\n",
+        "        )\n",
+        "        return response.choices[0].message.content\n",
+        "\n",
+        "    elif provider == \"Anthropic\":\n",
+        "        client = anthropic.Anthropic()\n",
+        "        response = client.messages.create(\n",
+        "            model=model_name,\n",
+        "            messages=messages,\n",
+        "            max_tokens=1024\n",
+        "        )\n",
+        "        return response.content[0].text\n",
+        "\n",
+        "    elif provider == \"Ollama\":\n",
+        "        msg = cl.Message(content=\"\")\n",
+        "        await msg.send()\n",
+        "\n",
+        "        full_response = \"\"\n",
+        "        stream = ollama.chat(\n",
+        "            model=model_name,\n",
+        "            messages=messages,\n",
+        "            stream=True\n",
+        "        )\n",
+        "\n",
+        "        for chunk in stream:\n",
+        "            content = chunk['message']['content']\n",
+        "            full_response += content\n",
+        "            await msg.stream_token(content)\n",
+        "\n",
+        "        await msg.update()\n",
+        "        return full_response\n",
+        "\n",
+        "    return \"Error: Unknown provider\"\n",
+        "\n",
+        "\n",
+        "def get_or_create_thread_id() -> str:\n",
+        "    \"\"\"Get persistent thread ID for this conversation\"\"\"\n",
+        "    thread_id = cl.user_session.get(\"thread_id\")\n",
+        "\n",
+        "    if not thread_id:\n",
+        "        thread_id = str(uuid.uuid4())[:16]\n",
+        "        cl.user_session.set(\"thread_id\", thread_id)\n",
+        "\n",
+        "    return thread_id\n",
+        "\n",
+        "\n",
+        "@cl.on_chat_start\n",
+        "async def start():\n",
+        "    \"\"\"Initialize chat session\"\"\"\n",
+        "    available_models = get_available_models()\n",
+        "\n",
+        "    if not available_models:\n",
+        "        await cl.Message(\n",
+        "            content=\"⚠️ **No LLM providers configured!**\\n\\n\"\n",
+        "            \"Set up Ollama: `brew install ollama && ollama pull llama3.2:3b`\"\n",
+        "        ).send()\n",
+        "        return\n",
+        "\n",
+        "    model_list = []\n",
+        "    for provider, models in available_models.items():\n",
+        "        for model in models:\n",
+        "            model_list.append(f\"{provider}/{model}\")\n",
+        "\n",
+        "    thread_id = get_or_create_thread_id()\n",
+        "\n",
+        "    cl.user_session.set(\"messages\", [])\n",
+        "    cl.user_session.set(\"logger\", ConversationLogger())\n",
+        "    cl.user_session.set(\"model\", model_list[0] if model_list else None)\n",
+        "    cl.user_session.set(\"available_models\", model_list)\n",
+        "    cl.user_session.set(\"is_resumed\", False)\n",
+        "\n",
+        "    sync = get_sync()\n",
+        "    sync_status = f\"✅ Auto-sync enabled (Project {sync.project_id})\" if sync.is_enabled() else \"💾 Auto-sync disabled\"\n",
+        "\n",
+        "    await cl.Message(\n",
+        "        content=f\"💬 **Multi-Turn Chat Feedback**\\n\\n\"\n",
+        "        f\"**Thread:** `{thread_id}`\\n\"\n",
+        "        f\"**Model:** `{model_list[0] if model_list else 'None'}`\\n\\n\"\n",
+        "        f\"{sync_status}\\n\\n\"\n",
+        "        f\"Ask me anything!\"\n",
+        "    ).send()\n",
+        "\n",
+        "\n",
+        "@cl.on_chat_resume\n",
+        "async def on_resume(thread: Dict):\n",
+        "    \"\"\"Handle conversation resumption\"\"\"\n",
+        "    available_models = get_available_models()\n",
+        "    model_list = []\n",
+        "    for provider, models in available_models.items():\n",
+        "        for model in models:\n",
+        "            model_list.append(f\"{provider}/{model}\")\n",
+        "\n",
+        "    thread_id = thread.get(\"id\")\n",
+        "    steps = thread.get(\"steps\", [])\n",
+        "\n",
+        "    messages = []\n",
+        "    for step in steps:\n",
+        "        if step.get(\"type\") in [\"user_message\", \"assistant_message\"]:\n",
+        "            role = \"user\" if step[\"type\"] == \"user_message\" else \"assistant\"\n",
+        "            messages.append({\"role\": role, \"content\": step.get(\"output\", \"\")})\n",
+        "\n",
+        "    cl.user_session.set(\"thread_id\", thread_id)\n",
+        "    cl.user_session.set(\"messages\", messages)\n",
+        "    cl.user_session.set(\"logger\", ConversationLogger())\n",
+        "    cl.user_session.set(\"model\", model_list[0] if model_list else None)\n",
+        "    cl.user_session.set(\"is_resumed\", True)\n",
+        "\n",
+        "    turn_count = len([m for m in messages if m[\"role\"] == \"user\"])\n",
+        "\n",
+        "    await cl.Message(\n",
+        "        content=f\"🔄 **Resumed** | Thread: `{thread_id}` | Previous turns: {turn_count}\"\n",
+        "    ).send()\n",
+        "\n",
+        "\n",
+        "@cl.on_message\n",
+        "async def main(message: cl.Message):\n",
+        "    \"\"\"Handle incoming messages\"\"\"\n",
+        "    messages = cl.user_session.get(\"messages\")\n",
+        "    model = cl.user_session.get(\"model\")\n",
+        "\n",
+        "    if not model:\n",
+        "        await cl.Message(content=\"⚠️ No model selected\").send()\n",
+        "        return\n",
+        "\n",
+        "    messages.append({\"role\": \"user\", \"content\": message.content})\n",
+        "\n",
+        "    try:\n",
+        "        response = await generate_response(messages, model)\n",
+        "        messages.append({\"role\": \"assistant\", \"content\": response})\n",
+        "        cl.user_session.set(\"messages\", messages)\n",
+        "\n",
+        "        # Auto-save after each response\n",
+        "        logger = cl.user_session.get(\"logger\")\n",
+        "        thread_id = get_or_create_thread_id()\n",
+        "\n",
+        "        logger.save_conversation(\n",
+        "            messages=messages,\n",
+        "            session_id=thread_id,\n",
+        "            model=model,\n",
+        "            metadata={\"auto_save\": True, \"last_updated\": datetime.utcnow().isoformat()}\n",
+        "        )\n",
+        "\n",
+        "    except Exception as e:\n",
+        "        await cl.Message(content=f\"❌ Error: {str(e)}\").send()\n",
+        "\n",
+        "\n",
+        "@cl.on_chat_end\n",
+        "async def on_chat_end():\n",
+        "    \"\"\"Auto-push to Label Studio (with versioning for resumes)\"\"\"\n",
+        "    messages = cl.user_session.get(\"messages\")\n",
+        "    thread_id = get_or_create_thread_id()\n",
+        "    model = cl.user_session.get(\"model\")\n",
+        "    is_resumed = cl.user_session.get(\"is_resumed\", False)\n",
+        "\n",
+        "    if not messages:\n",
+        "        return\n",
+        "\n",
+        "    turn_count = len([m for m in messages if m[\"role\"] == \"user\"])\n",
+        "\n",
+        "    if turn_count < MIN_TURNS_FOR_SYNC:\n",
+        "        print(f\"⏭️  Only {turn_count} turns, skipping sync\")\n",
+        "        return\n",
+        "\n",
+        "    sync = get_sync()\n",
+        "    if not sync.is_enabled():\n",
+        "        print(f\"ℹ️  Auto-sync disabled\")\n",
+        "        return\n",
+        "\n",
+        "    try:\n",
+        "        # Find existing versions\n",
+        "        existing_tasks = sync.client.tasks.list(project=sync.project_id)\n",
+        "        existing_versions = []\n",
+        "\n",
+        "        for task in existing_tasks:\n",
+        "            if hasattr(task, 'data'):\n",
+        "                task_thread_id = task.data.get('thread_id') or task.data.get('session_id')\n",
+        "                if task_thread_id and task_thread_id.split('_v')[0] == thread_id.split('_v')[0]:\n",
+        "                    existing_versions.append(task)\n",
+        "\n",
+        "        version = len(existing_versions) + 1\n",
+        "        versioned_session_id = f\"{thread_id}_v{version}\"\n",
+        "\n",
+        "        # Save with version\n",
+        "        logger = cl.user_session.get(\"logger\")\n",
+        "        filepath = logger.save_conversation(\n",
+        "            messages=messages,\n",
+        "            session_id=versioned_session_id,\n",
+        "            model=model,\n",
+        "            metadata={\n",
+        "                \"auto_save\": True,\n",
+        "                \"version\": version,\n",
+        "                \"thread_id\": thread_id,\n",
+        "                \"was_resumed\": is_resumed\n",
+        "            }\n",
+        "        )\n",
+        "\n",
+        "        # Create task\n",
+        "        conversation_data = json.loads(filepath.read_text())\n",
+        "        task = {\n",
+        "            'data': {\n",
+        "                'chat': conversation_data['messages'],  # Changed from 'messages' to 'chat' to match label config\n",
+        "                'text': 'Review the conversation below and evaluate the quality of the chat interaction.',\n",
+        "                'session_id': versioned_session_id,\n",
+        "                'thread_id': thread_id,\n",
+        "                'model': model,\n",
+        "                'turn_count': turn_count,\n",
+        "                'version': version,\n",
+        "                'timestamp': datetime.utcnow().isoformat() + 'Z',\n",
+        "            },\n",
+        "            'meta': {\n",
+        "                'auto_synced': True,\n",
+        "                'is_resume': is_resumed,\n",
+        "                'version': version,\n",
+        "            }\n",
+        "        }\n",
+        "\n",
+        "        sync.client.projects.import_tasks(id=sync.project_id, request=[task])\n",
+        "\n",
+        "        if version > 1:\n",
+        "            print(f\"✅ Created version {version} (RESUME)\")\n",
+        "        else:\n",
+        "            print(f\"✅ Created version 1 (NEW)\")\n",
+        "\n",
+        "    except Exception as e:\n",
+        "        print(f\"❌ Sync failed: {e}\")\n",
+        "\n",
+        "\n",
+        "if __name__ == \"__main__\":\n",
+        "    pass\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "7311f845",
+      "metadata": {
+        "id": "7311f845"
+      },
+      "source": [
+        "### Step 3: Set Up Ollama (Local LLM)\n",
+        "\n",
+        "For this example, we'll use Ollama which runs a local LLM on your machine.\n",
+        "\n",
+        "**Install Ollama:**\n",
+        "```bash\n",
+        "# macOS\n",
+        "brew install ollama\n",
+        "\n",
+        "# Or download from https://ollama.ai\n",
+        "```\n",
+        "\n",
+        "**Pull a model:**\n",
+        "```bash\n",
+        "ollama pull llama3.2:3b  # 3B model works on 16GB RAM\n",
+        "```\n",
+        "\n",
+        "**Verify it's running:**\n",
+        "```bash\n",
+        "ollama list\n",
+        "```\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "c060e7a3",
+      "metadata": {
+        "id": "c060e7a3"
+      },
+      "source": [
+        "### Step 4: Set Environment Variables\n",
+        "\n",
+        "Set the project ID from earlier so the chatbot knows where to sync conversations.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "a46bddd4",
+      "metadata": {
+        "id": "a46bddd4"
+      },
+      "outputs": [],
+      "source": [
+        "# Set environment variables for auto-sync\n",
+        "import os\n",
+        "\n",
+        "# Set all required environment variables for auto-sync\n",
+        "os.environ['LABEL_STUDIO_URL'] = ls_url\n",
+        "os.environ['LABEL_STUDIO_API_KEY'] = ls_api_key\n",
+        "os.environ['LABEL_STUDIO_PROJECT_ID'] = str(project_id)\n",
+        "\n",
+        "print(f\"✅ Environment configured:\")\n",
+        "print(f\"   LABEL_STUDIO_URL: {ls_url}\")\n",
+        "print(f\"   LABEL_STUDIO_PROJECT_ID: {project_id}\")\n",
+        "print(f\"   LABEL_STUDIO_API_KEY: {'*' * 20}... (hidden)\")\n",
+        "print(f\"\\n🔄 Auto-sync: ENABLED\")\n",
+        "print(f\"   Conversations will automatically sync to Label Studio when you close the chat!\")\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "5c5a2001",
+      "metadata": {
+        "id": "5c5a2001"
+      },
+      "source": [
+        "### Step 5: Run the Chainlit Chatbot\n",
+        "\n",
+        "Now run the chatbot! It will automatically sync conversations to Label Studio.\n",
+        "\n",
+        "**To run from terminal:**\n",
+        "```bash\n",
+        "chainlit run chatbot_ui_auto_sync.py --port 8087\n",
+        "```\n",
+        "\n",
+        "Then:\n",
+        "1. Open http://localhost:8087 in your browser\n",
+        "2. Have a conversation (at least 2 turns)\n",
+        "3. Close the browser tab\n",
+        "4. Check Label Studio - your conversation will be there!\n",
+        "\n",
+        "**Features:**\n",
+        "- ✅ Automatic conversation capture\n",
+        "- ✅ Auto-sync to Label Studio on disconnect\n",
+        "- ✅ Conversation resumption with versioning\n",
+        "- ✅ Local LLM (free and private!)\n",
+        "\n",
+        "**What happens:**\n",
+        "- Each message auto-saves to `data/conversations/`\n",
+        "- When you close the chat, it pushes to Label Studio\n",
+        "- If you resume the chat later, it creates a new version (v2, v3, etc.)\n",
+        "\n",
+        "**Ready to annotate!** Visit your Label Studio project to start evaluating the conversations.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "aac715e6",
+      "metadata": {
+        "id": "aac715e6"
+      },
+      "source": [
+        "### Verify Files Created\n",
+        "\n",
+        "Let's check that all necessary files were created.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "c9f797b3",
+      "metadata": {
+        "id": "c9f797b3"
+      },
+      "outputs": [],
+      "source": [
+        "from pathlib import Path\n",
+        "\n",
+        "required_files = [\n",
+        "    'conversation_logger.py',\n",
+        "    'auto_sync.py',\n",
+        "    'chatbot_ui_auto_sync.py'\n",
+        "]\n",
+        "\n",
+        "print(\"📁 Checking files...\")\n",
+        "for file in required_files:\n",
+        "    if Path(file).exists():\n",
+        "        print(f\"   ✅ {file}\")\n",
+        "    else:\n",
+        "        print(f\"   ❌ {file} - MISSING!\")\n",
+        "\n",
+        "print(f\"\\n📋 Project URL: {project_url}\")\n",
+        "print(f\"\\n🚀 Ready to run:\")\n",
+        "print(f\"   chainlit run chatbot_ui_auto_sync.py --port 8087\")\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "ee2c0432",
+      "metadata": {
+        "id": "ee2c0432"
+      },
+      "source": [
+        "## Summary\n",
+        "\n",
+        "You've successfully set up:\n",
+        "\n",
+        "1. ✅ **Label Studio Project** - Created with chatbot evaluation template\n",
+        "2. ✅ **Conversation Logger** - Saves chats to JSON automatically\n",
+        "3. ✅ **Auto-Sync** - Pushes conversations to Label Studio\n",
+        "4. ✅ **Chainlit Chatbot** - Full UI with local LLM support\n",
+        "\n",
+        "**Complete workflow:**\n",
+        "```\n",
+        "User chats → Auto-save to JSON → Close browser → Auto-push to Label Studio → Ready to annotate!\n",
+        "```\n",
+        "\n",
+        "**Key Features:**\n",
+        "- 🔒 **Private** - Local LLM, no data sent to cloud\n",
+        "- 🔄 **Versioning** - Resume conversations safely\n",
+        "- ⚡ **Automatic** - Zero manual export needed\n",
+        "\n",
+        "**Next Steps:**\n",
+        "1. Run the chatbot: `chainlit run chatbot_ui_auto_sync.py --port 8087`\n",
+        "2. Have some conversations\n",
+        "3. Annotate them in Label Studio\n",
+        "4. Export annotations for model training or system prompt adjustment"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": ".venv",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.12.10"
+    },
+    "colab": {
+      "provenance": []
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}
\ No newline at end of file