Quickstart

import { ABV } from "@abvdev/client";

const abv = new ABV();

// Create a dataset for email response evaluation
const dataset = await abv.datasets.create({
  name: "email-responses-v1",
  description: "Test cases for customer service email quality"
});

// Add test cases
const testCases = [
  {
    input: {
      customer_email: "My order #12345 arrived damaged. I need a refund.",
      context: "Product: Coffee Maker, Price: $89.99"
    },
    expected_output: {
      polite: true,
      addresses_refund: true,
      mentions_order_number: true
    }
  },
  {
    input: {
      customer_email: "How do I reset my password? I've tried 3 times.",
      context: "Customer tier: Premium"
    },
    expected_output: {
      polite: true,
      provides_steps: true,
      offers_additional_help: true
    }
  },
  {
    input: {
      customer_email: "Your website is terrible! Nothing works!",
      context: "Customer tier: Free"
    },
    expected_output: {
      polite: true,  // Must stay professional even when customer isn't
      acknowledges_frustration: true,
      offers_specific_help: true
    }
  }
];

// Add each test case to the dataset
for (let i = 0; i < testCases.length; i++) {
  await dataset.createItem({
    input: testCases[i].input,
    expected_output: testCases[i].expected_output,
    metadata: { case_number: i + 1 }
  });
}

console.log(`✅ Created dataset with ${testCases.length} test cases`);
console.log(`Dataset ID: ${dataset.id}`);

// Your AI function that generates email responses
function generateEmailResponse(customerEmail: string, context: string): string {
  /**
   * This is your actual AI function. For this tutorial, we'll use a simple mock.
   * In production, this would call your LLM (OpenAI, Anthropic, etc.)
   */
  // Mock response for demonstration
  // Replace this with your actual LLM call
  return `Dear valued customer, thank you for contacting us about: ${customerEmail.slice(0, 50)}...`;
}

// Run the dataset through your AI
const datasetRun = await dataset.run({
  name: "Initial evaluation",
  description: "Testing baseline email response quality",
  metadata: { model: "gpt-4", temperature: 0.7 }
});

// Process each test case
const items = await dataset.items();
for (const item of items) {
  // Generate AI response
  const inputData = item.input;
  const aiResponse = generateEmailResponse(
    inputData.customer_email,
    inputData.context
  );

  // Record the output
  await datasetRun.createObservation({
    dataset_item_id: item.id,
    output: aiResponse,
    metadata: { input_case: item.metadata.case_number }
  });
}

console.log(`✅ Completed evaluation run: ${datasetRun.id}`);
console.log("View results in the ABV platform!");

// Score politeness using LLM-as-a-Judge
function scorePolitenessLLM(response: string): { value: number; comment: string } {
  /**
   * Use an LLM to judge if the response is polite
   */
  // In production, call your LLM API
  // For demo, we'll return a mock score
  const scoreValue = 0.85;  // 0-1 scale

  return {
    value: scoreValue,
    comment: "Response uses professional language and courteous tone"
  };
}

// Apply scores to each observation
const observations = await datasetRun.observations();
for (const observation of observations) {
  // Get politeness score
  const politeness = scorePolitenesLLM(observation.output);

  // Record the score
  await observation.score({
    name: "politeness",
    value: politeness.value,
    comment: politeness.comment
  });
}

console.log("✅ Scored all responses for politeness");

function scoreCompleteness(response: string, expected: Record<string, boolean>): number {
  /**
   * Check if response addresses expected points
   * Simple keyword matching for demonstration
   */
  let score = 0;
  let checks = 0;

  // Check each expected criterion
  for (const [key, shouldHave] of Object.entries(expected)) {
    checks += 1;
    if (key === "addresses_refund" && response.toLowerCase().includes("refund")) {
      score += 1;
    } else if (key === "mentions_order_number" && response.includes("#")) {
      score += 1;
    } else if (key === "provides_steps" && ["step", "first", "then"].some(word => response.toLowerCase().includes(word))) {
      score += 1;
    }
  }

  return checks > 0 ? score / checks : 0;
}

// Apply custom scores
for (const observation of observations) {
  // Get the expected output for this case
  const item = await dataset.getItem(observation.dataset_item_id);
  const expected = item.expected_output;

  // Calculate completeness score
  const completeness = scoreCompleteness(observation.output, expected);

  // Record the score
  await observation.score({
    name: "completeness",
    value: completeness,
    comment: `Addressed ${Math.round(completeness * 100)}% of expected points`
  });
}

console.log("✅ Scored all responses for completeness");

// Improved version with better prompting
function generateEmailResponseV2(customerEmail: string, context: string): string {
  /**
   * Version 2: Improved prompt engineering
   */
  // In production, use a better prompt like:
  const prompt = `Generate a professional customer service email response.

Customer email: ${customerEmail}
Context: ${context}

Requirements:
- Be polite and professional
- Address all customer concerns
- Offer specific next steps
- Show empathy when appropriate

Response:`;

  // Mock improved response
  return "Dear Customer, I sincerely apologize for the issue with your order...";
}

// Run evaluation again with v2
const datasetRunV2 = await dataset.run({
  name: "Improved model (v2)",
  description: "Testing with better prompting",
  metadata: { model: "gpt-4", temperature: 0.7, version: "v2" }
});

for (const item of await dataset.items()) {
  const inputData = item.input;
  const aiResponse = generateEmailResponseV2(
    inputData.customer_email,
    inputData.context
  );

  await datasetRunV2.createObservation({
    dataset_item_id: item.id,
    output: aiResponse
  });
}

// Score the new version
// ... (same scoring code as before)

console.log("✅ Completed v2 evaluation - compare results in the platform!");

import { ABV } from "@abvdev/client";

const abv = new ABV();

async function runTutorial() {
  // 1. Create dataset
  const dataset = await abv.datasets.create({
    name: "email-responses-tutorial",
    description: "Quickstart tutorial dataset"
  });

  // 2. Add test cases
  const testCases = [
    {
      input: { customer_email: "Order damaged, need refund", context: "Product: Coffee Maker" },
      expected_output: { polite: true, addresses_refund: true }
    },
    {
      input: { customer_email: "Password reset help", context: "Premium customer" },
      expected_output: { polite: true, provides_steps: true }
    }
  ];

  for (const testCase of testCases) {
    await dataset.createItem({
      input: testCase.input,
      expected_output: testCase.expected_output
    });
  }

  // 3. Run evaluation
  const datasetRun = await dataset.run({ name: "Tutorial run" });
  const items = await dataset.items();

  for (const item of items) {
    const response = `Mock response to: ${item.input.customer_email}`;
    await datasetRun.createObservation({
      dataset_item_id: item.id,
      output: response
    });
  }

  // 4. Add scores
  const observations = await datasetRun.observations();
  for (const observation of observations) {
    await observation.score({
      name: "quality",
      value: 0.8,
      comment: "Good response"
    });
  }

  console.log(`✅ Complete! View results for dataset run: ${datasetRun.id}`);
}

runTutorial();