from abvdev import ABV
from openai import OpenAI
import random
import os

# Initialize clients
abv = ABV(
    api_key=os.getenv("ABV_API_KEY"),
    host="https://app.abv.dev",
)
openai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

def run_ab_test(user_input: dict):
    """
    Run A/B test for movie critic prompt.

    Args:
        user_input: Dict with 'criticlevel' and 'movie' keys

    Returns:
        LLM response
    """
    # Fetch both variants
    prompt_a = abv.get_prompt("movie-critic", label="variant-a")
    prompt_b = abv.get_prompt("movie-critic", label="variant-b")

    # Randomly assign user to variant (50/50 split)
    selected_prompt = random.choice([prompt_a, prompt_b])

    # Compile prompt with user input
    compiled_prompt = selected_prompt.compile(
        criticlevel=user_input["criticlevel"],
        movie=user_input["movie"]
    )

    # Create generation with linked prompt
    with abv.start_as_current_observation(
        as_type="generation",
        name="movie-review-ab-test",
        prompt=selected_prompt  # Link for tracking by version
    ) as generation:
        # Call LLM
        response = openai_client.chat.completions.create(
            model="gpt-4o",
            messages=[{"role": "user", "content": compiled_prompt}]
        )

        result = response.choices[0].message.content

        # Update generation with output
        generation.update(output=result)

        return result

# Usage
if __name__ == "__main__":
    result = run_ab_test({
        "criticlevel": "expert",
        "movie": "The Lord of the Rings"
    })
    print(result)

    # Flush events for short-lived applications
    abv.flush()

# Weighted random selection
selected_prompt = random.choices(
    [prompt_a, prompt_b],
    weights=[0.9, 0.1],  # 90% variant-a, 10% variant-b
    k=1
)[0]

import dotenv from "dotenv";
dotenv.config();

import { NodeSDK } from "@opentelemetry/sdk-node";
import { ABVSpanProcessor } from "@abvdev/otel";

const sdk = new NodeSDK({
  spanProcessors: [
    new ABVSpanProcessor({
      apiKey: process.env.ABV_API_KEY,
      baseUrl: process.env.ABV_BASE_URL,
      exportMode: "immediate",
      flushAt: 1,
      flushInterval: 1,
    })
  ],
});

sdk.start();

import "./instrumentation"; // Must be first import
import { ABVClient } from "@abvdev/client";
import { startObservation } from "@abvdev/tracing";
import OpenAI from "openai";
import dotenv from "dotenv";
dotenv.config();

const abv = new ABVClient();
const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
});

async function runABTest(userInput: {
  criticlevel: string;
  movie: string;
}) {
  // Fetch both variants
  const promptA = await abv.prompt.get("movie-critic", {
    label: "variant-a",
  });

  const promptB = await abv.prompt.get("movie-critic", {
    label: "variant-b",
  });

  // Randomly assign user to variant (50/50 split)
  const selectedPrompt = Math.random() < 0.5 ? promptA : promptB;

  // Compile prompt
  const compiledMessages = selectedPrompt.compile(userInput);

  // Create generation with linked prompt
  const generation = startObservation(
    "movie-review-ab-test",
    {
      model: "gpt-4o",
      input: compiledMessages,
      prompt: selectedPrompt  // Link for tracking
    },
    { asType: "generation" }
  );

  // Call LLM
  const completion = await openai.chat.completions.create({
    model: "gpt-4o",
    messages: compiledMessages,
  });

  const result = completion.choices[0].message.content;

  // Update generation
  generation.update({
    output: { content: result },
  });

  generation.end();

  return result;
}

// Usage
async function main() {
  const result = await runABTest({
    criticlevel: "expert",
    movie: "The Lord of the Rings"
  });
  console.log(result);
}

main();

const selectedPrompt = Math.random() < 0.9 ? promptA : promptB;
// 90% get promptA, 10% get promptB

from scipy import stats

variant_a_scores = [4.2, 4.0, 4.5, 4.1, ...]  # 400 scores
variant_b_scores = [4.5, 4.3, 4.7, 4.4, ...]  # 400 scores

statistic, p_value = stats.mannwhitneyu(
    variant_a_scores,
    variant_b_scores,
    alternative='two-sided'
)

print(f"p-value: {p_value}")
if p_value < 0.05:
    print("Statistically significant difference")
else:
    print("No significant difference")

with abv.start_as_current_observation(
    as_type="generation",
    prompt=selected_prompt  # Don't forget this!
) as generation:
    ...