"""P0 shadow rollout: run the LLM intent classifier beside rule routing, log only. Observation-only — the LLM result never influences routing. Disagreements are logged as `chat_intent_shadow` events to build the P1 golden dataset. """ from __future__ import annotations import dataclasses import logging from typing import Any from .chat_intent_classifier import ( ChatIntentClassifier, LLMChatIntentClassifier, RuleBasedChatIntentClassifier, ) from .chat_intents import CHAT_INTENT_REGISTRY_VERSION from .llm_services import OpenAICompatibleStructuredClient, log_ai_event from .settings import Settings class ChatIntentShadowLogger: """Compares LLM vs rule classification per message. Never used for routing.""" def __init__(self, primary: ChatIntentClassifier, rules: ChatIntentClassifier) -> None: self.primary = primary self.rules = rules def observe(self, message: str, *, state_summary: dict[str, Any]) -> None: rule_result = self.rules.classify(message, state_summary=state_summary) try: llm_result = self.primary.classify(message, state_summary=state_summary) except Exception as exc: log_ai_event( "chat_intent_shadow_error", level=logging.WARNING, exception=type(exc).__name__, trace_id=getattr(exc, "trace_id", None), ) return log_ai_event( "chat_intent_shadow", registry_version=CHAT_INTENT_REGISTRY_VERSION, rule_intent=rule_result.intent.value, llm_intent=llm_result.intent.value, llm_confidence=llm_result.confidence, disagreement=llm_result.intent != rule_result.intent, ) def build_chat_intent_shadow(settings: Settings, client: Any | None = None) -> ChatIntentShadowLogger | None: """Shadow only when explicitly enabled *and* an LLM provider is configured.""" if settings.intent_router_mode != "shadow" or not settings.use_openai: return None llm_settings = settings if settings.intent_model: llm_settings = dataclasses.replace(settings, openai_model=settings.intent_model) completion = OpenAICompatibleStructuredClient(llm_settings, client) return ChatIntentShadowLogger(LLMChatIntentClassifier(completion), RuleBasedChatIntentClassifier())