Gateway Patterns

The ABV Gateway provides an OpenAI-compatible API that routes requests to multiple LLM providers (OpenAI, Anthropic, Google)

Basic Chat Completion

Python
JavaScript

from abvdev import ABV

abv = ABV(api_key="sk-abv-...")

response = abv.gateway.complete_chat(
    provider="openai",
    model="gpt-4o-mini",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "What is machine learning?"}
    ],
    temperature=0.7,
    max_tokens=500
)

print(response.choices[0].message.content)

import { ABVClient } from "@abvdev/client";

const abv = new ABVClient({ apiKey: "sk-abv-..." });

const response = await abv.gateway.chat.completions.create({
  provider: "openai",
  model: "gpt-4o-mini",
  messages: [
    { role: "system", content: "You are a helpful assistant." },
    { role: "user", content: "What is machine learning?" },
  ],
  temperature: 0.7,
  max_tokens: 500,
});

console.log(response.choices[0].message.content);

Switch Providers

Use the same code with different providers by changing provider and model.

Python
JavaScript

# OpenAI
openai_response = abv.gateway.complete_chat(
    provider="openai",
    model="gpt-4o",
    messages=messages
)

# Anthropic
anthropic_response = abv.gateway.complete_chat(
    provider="anthropic",
    model="claude-sonnet-4-20250514",
    messages=messages
)

# Google Gemini
gemini_response = abv.gateway.complete_chat(
    provider="gemini",
    model="gemini-1.5-pro",
    messages=messages
)

// OpenAI
const openaiResponse = await abv.gateway.chat.completions.create({
  provider: "openai",
  model: "gpt-4o",
  messages,
});

// Anthropic
const anthropicResponse = await abv.gateway.chat.completions.create({
  provider: "anthropic",
  model: "claude-sonnet-4-20250514",
  messages,
});

// Google Gemini
const geminiResponse = await abv.gateway.chat.completions.create({
  provider: "gemini",
  model: "gemini-1.5-pro",
  messages,
});

Streaming Responses

Python
JavaScript

stream = abv.gateway.complete_chat_stream(
    provider="openai",
    model="gpt-4o-mini",
    messages=[{"role": "user", "content": "Write a haiku about coding"}]
)

for chunk in stream:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

const stream = await abv.gateway.chat.completions.create({
  provider: "openai",
  model: "gpt-4o-mini",
  messages: [{ role: "user", content: "Write a haiku about coding" }],
  stream: true,
});

for await (const chunk of stream) {
  const content = chunk.choices[0]?.delta?.content;
  if (content) {
    process.stdout.write(content);
  }
}

Gateway with Tracing Context

Combine gateway calls with tracing for full observability.

Python
JavaScript

from abvdev import ABV, observe

abv = ABV(api_key="sk-abv-...")

@observe()
def chat_with_user(user_id: str, message: str):
    # Update trace context
    abv.update_current_trace(
        user_id=user_id,
        tags=["chat", "production"]
    )

    # Gateway call is automatically traced as a generation
    response = abv.gateway.complete_chat(
        provider="openai",
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": message}]
    )

    return response.choices[0].message.content

result = chat_with_user("user-123", "Hello!")

import { startActiveObservation, updateActiveTrace } from "@abvdev/tracing";

await startActiveObservation("chat-with-user", async (span) => {
  updateActiveTrace({
    userId: "user-123",
    tags: ["chat", "production"],
  });

  span.update({ input: { message: "Hello!" } });

  // Gateway call is automatically traced
  const response = await abv.gateway.chat.completions.create({
    provider: "openai",
    model: "gpt-4o-mini",
    messages: [{ role: "user", content: "Hello!" }],
  });

  const content = response.choices[0].message.content;
  span.update({ output: content });
  return content;
});

await abvSpanProcessor.forceFlush();

Cost Tracking

Gateway calls automatically track token usage and costs. Access via the trace.

Python
JavaScript

from abvdev import ABV, observe

abv = ABV(api_key="sk-abv-...")

@observe(name="cost-tracked-call", as_type="generation")
def generate_with_cost_tracking(prompt: str):
    response = abv.gateway.complete_chat(
        provider="openai",
        model="gpt-4o",
        messages=[{"role": "user", "content": prompt}]
    )

    # Usage is automatically captured in the trace
    # View in ABV UI: token counts, latency, and estimated cost
    return response.choices[0].message.content

# After execution, check the ABV dashboard for:
# - Input/output tokens
# - Model used
# - Estimated cost in USD
# - Time to first token (streaming)
# - Total latency

import { observe, updateActiveObservation } from "@abvdev/tracing";

const generateWithCostTracking = observe(
  async (prompt: string) => {
    const response = await abv.gateway.chat.completions.create({
      provider: "openai",
      model: "gpt-4o",
      messages: [{ role: "user", content: prompt }],
    });

    // Update with any additional cost info if needed
    updateActiveObservation({
      usageDetails: {
        input: response.usage?.prompt_tokens,
        output: response.usage?.completion_tokens,
      },
    });

    return response.choices[0].message.content;
  },
  { name: "cost-tracked-call", asType: "generation" }
);

await generateWithCostTracking("Explain quantum computing");
await abvSpanProcessor.forceFlush();

Model Parameters

Full OpenAI-compatible parameter support.

Python
JavaScript

response = abv.gateway.complete_chat(
    provider="openai",
    model="gpt-4o-mini",
    messages=[{"role": "user", "content": "Generate creative options"}],
    temperature=0.9,       # Creativity (0-2)
    max_tokens=1000,       # Response length limit
    top_p=0.95,            # Nucleus sampling
    frequency_penalty=0.5, # Reduce repetition
    presence_penalty=0.5,  # Encourage new topics
    stop=["\n\n"],         # Stop sequences
    n=3,                   # Number of completions
)

for i, choice in enumerate(response.choices):
    print(f"Option {i+1}: {choice.message.content}")

const response = await abv.gateway.chat.completions.create({
  provider: "openai",
  model: "gpt-4o-mini",
  messages: [{ role: "user", content: "Generate creative options" }],
  temperature: 0.9,
  max_tokens: 1000,
  top_p: 0.95,
  frequency_penalty: 0.5,
  presence_penalty: 0.5,
  stop: ["\n\n"],
  n: 3,
});

response.choices.forEach((choice, i) => {
  console.log(`Option ${i + 1}: ${choice.message.content}`);
});

Provider Fallback Pattern

Implement fallback logic when a provider fails.

Python
JavaScript

from abvdev import ABV, observe

abv = ABV(api_key="sk-abv-...")

PROVIDERS = [
    {"provider": "openai", "model": "gpt-4o-mini"},
    {"provider": "anthropic", "model": "claude-sonnet-4-20250514"},
    {"provider": "gemini", "model": "gemini-1.5-flash"},
]

@observe(name="llm-with-fallback")
def generate_with_fallback(messages: list):
    for config in PROVIDERS:
        try:
            response = abv.gateway.complete_chat(
                provider=config["provider"],
                model=config["model"],
                messages=messages
            )
            return response.choices[0].message.content
        except Exception as e:
            abv.update_current_span(
                level="WARNING",
                status_message=f"{config['provider']} failed: {e}"
            )
            continue

    raise Exception("All providers failed")

result = generate_with_fallback([{"role": "user", "content": "Hello"}])

import { observe, updateActiveObservation } from "@abvdev/tracing";

const PROVIDERS = [
  { provider: "openai", model: "gpt-4o-mini" },
  { provider: "anthropic", model: "claude-sonnet-4-20250514" },
  { provider: "gemini", model: "gemini-1.5-flash" },
];

const generateWithFallback = observe(
  async (messages: any[]) => {
    for (const config of PROVIDERS) {
      try {
        const response = await abv.gateway.chat.completions.create({
          provider: config.provider as any,
          model: config.model,
          messages,
        });
        return response.choices[0].message.content;
      } catch (error) {
        updateActiveObservation({
          level: "WARNING",
          statusMessage: `${config.provider} failed: ${error.message}`,
        });
        continue;
      }
    }
    throw new Error("All providers failed");
  },
  { name: "llm-with-fallback" }
);

const result = await generateWithFallback([
  { role: "user", content: "Hello" },
]);

Using Managed Prompts with Gateway

Fetch prompts from ABV and use them with the gateway.

Python
JavaScript

from abvdev import ABV

abv = ABV(api_key="sk-abv-...")

# Fetch managed prompt
prompt = abv.get_prompt("customer-support-v2")

# Compile with variables
messages = prompt.compile(
    customer_name="Alice",
    issue="billing question"
)

# Use with gateway
response = abv.gateway.complete_chat(
    provider="openai",
    model=prompt.config.get("model", "gpt-4o-mini"),
    messages=messages,
    temperature=prompt.config.get("temperature", 0.7)
)

const abv = new ABVClient({ apiKey: "sk-abv-..." });

// Fetch managed prompt
const prompt = await abv.prompt.get("customer-support-v2");

// Compile with variables
const messages = prompt.compile({
  customer_name: "Alice",
  issue: "billing question",
});

// Use with gateway
const response = await abv.gateway.chat.completions.create({
  provider: "openai",
  model: prompt.config?.model ?? "gpt-4o-mini",
  messages,
  temperature: prompt.config?.temperature ?? 0.7,
});

Guardrails Patterns

Next: Content validation and safety

LLM Gateway Guide

Reference: Full gateway documentation

Getting Started

Basic Features

LLM Gateway

Guardrails

Evaluations

Prompt Management

Cookbook

SDKs

Platform

Support

Basic Chat Completion

Switch Providers

Streaming Responses

Gateway with Tracing Context

Cost Tracking

Model Parameters

Provider Fallback Pattern

Using Managed Prompts with Gateway

Guardrails Patterns

LLM Gateway Guide

​Basic Chat Completion

​Switch Providers

​Streaming Responses

​Gateway with Tracing Context

​Cost Tracking

​Model Parameters

​Provider Fallback Pattern

​Using Managed Prompts with Gateway

Guardrails Patterns

LLM Gateway Guide

Basic Chat Completion

Switch Providers

Streaming Responses

Gateway with Tracing Context

Cost Tracking

Model Parameters

Provider Fallback Pattern

Using Managed Prompts with Gateway