> ## Documentation Index
> Fetch the complete documentation index at: https://docs.abv.dev/llms.txt
> Use this file to discover all available pages before exploring further.

# Gateway Patterns

> Unified LLM API with multi-provider routing and automatic tracing

<Info>
  The ABV Gateway provides an OpenAI-compatible API that routes requests to multiple LLM providers (OpenAI, Anthropic, Google)
</Info>

## Basic Chat Completion

<Tabs>
  <Tab title="Python" icon="python">
    ```python theme={null}
    from abvdev import ABV

    abv = ABV(api_key="sk-abv-...")

    response = abv.gateway.complete_chat(
        provider="openai",
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "What is machine learning?"}
        ],
        temperature=0.7,
        max_tokens=500
    )

    print(response.choices[0].message.content)
    ```
  </Tab>

  <Tab title="JavaScript" icon="js">
    ```typescript theme={null}
    import { ABVClient } from "@abvdev/client";

    const abv = new ABVClient({ apiKey: "sk-abv-..." });

    const response = await abv.gateway.chat.completions.create({
      provider: "openai",
      model: "gpt-4o-mini",
      messages: [
        { role: "system", content: "You are a helpful assistant." },
        { role: "user", content: "What is machine learning?" },
      ],
      temperature: 0.7,
      max_tokens: 500,
    });

    console.log(response.choices[0].message.content);
    ```
  </Tab>
</Tabs>

***

## Switch Providers

Use the same code with different providers by changing `provider` and `model`.

<Tabs>
  <Tab title="Python" icon="python">
    ```python theme={null}
    # OpenAI
    openai_response = abv.gateway.complete_chat(
        provider="openai",
        model="gpt-4o",
        messages=messages
    )

    # Anthropic
    anthropic_response = abv.gateway.complete_chat(
        provider="anthropic",
        model="claude-sonnet-4-20250514",
        messages=messages
    )

    # Google Gemini
    gemini_response = abv.gateway.complete_chat(
        provider="gemini",
        model="gemini-1.5-pro",
        messages=messages
    )
    ```
  </Tab>

  <Tab title="JavaScript" icon="js">
    ```typescript theme={null}
    // OpenAI
    const openaiResponse = await abv.gateway.chat.completions.create({
      provider: "openai",
      model: "gpt-4o",
      messages,
    });

    // Anthropic
    const anthropicResponse = await abv.gateway.chat.completions.create({
      provider: "anthropic",
      model: "claude-sonnet-4-20250514",
      messages,
    });

    // Google Gemini
    const geminiResponse = await abv.gateway.chat.completions.create({
      provider: "gemini",
      model: "gemini-1.5-pro",
      messages,
    });
    ```
  </Tab>
</Tabs>

***

## Streaming Responses

<Tabs>
  <Tab title="Python" icon="python">
    ```python theme={null}
    stream = abv.gateway.complete_chat_stream(
        provider="openai",
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": "Write a haiku about coding"}]
    )

    for chunk in stream:
        if chunk.choices[0].delta.content:
            print(chunk.choices[0].delta.content, end="", flush=True)
    ```
  </Tab>

  <Tab title="JavaScript" icon="js">
    ```typescript theme={null}
    const stream = await abv.gateway.chat.completions.create({
      provider: "openai",
      model: "gpt-4o-mini",
      messages: [{ role: "user", content: "Write a haiku about coding" }],
      stream: true,
    });

    for await (const chunk of stream) {
      const content = chunk.choices[0]?.delta?.content;
      if (content) {
        process.stdout.write(content);
      }
    }
    ```
  </Tab>
</Tabs>

***

## Gateway with Tracing Context

Combine gateway calls with tracing for full observability.

<Tabs>
  <Tab title="Python" icon="python">
    ```python theme={null}
    from abvdev import ABV, observe

    abv = ABV(api_key="sk-abv-...")

    @observe()
    def chat_with_user(user_id: str, message: str):
        # Update trace context
        abv.update_current_trace(
            user_id=user_id,
            tags=["chat", "production"]
        )

        # Gateway call is automatically traced as a generation
        response = abv.gateway.complete_chat(
            provider="openai",
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": message}]
        )

        return response.choices[0].message.content

    result = chat_with_user("user-123", "Hello!")
    ```
  </Tab>

  <Tab title="JavaScript" icon="js">
    ```typescript theme={null}
    import { startActiveObservation, updateActiveTrace } from "@abvdev/tracing";

    await startActiveObservation("chat-with-user", async (span) => {
      updateActiveTrace({
        userId: "user-123",
        tags: ["chat", "production"],
      });

      span.update({ input: { message: "Hello!" } });

      // Gateway call is automatically traced
      const response = await abv.gateway.chat.completions.create({
        provider: "openai",
        model: "gpt-4o-mini",
        messages: [{ role: "user", content: "Hello!" }],
      });

      const content = response.choices[0].message.content;
      span.update({ output: content });
      return content;
    });

    await abvSpanProcessor.forceFlush();
    ```
  </Tab>
</Tabs>

***

## Cost Tracking

Gateway calls automatically track token usage and costs. Access via the trace.

<Tabs>
  <Tab title="Python" icon="python">
    ```python theme={null}
    from abvdev import ABV, observe

    abv = ABV(api_key="sk-abv-...")

    @observe(name="cost-tracked-call", as_type="generation")
    def generate_with_cost_tracking(prompt: str):
        response = abv.gateway.complete_chat(
            provider="openai",
            model="gpt-4o",
            messages=[{"role": "user", "content": prompt}]
        )

        # Usage is automatically captured in the trace
        # View in ABV UI: token counts, latency, and estimated cost
        return response.choices[0].message.content

    # After execution, check the ABV dashboard for:
    # - Input/output tokens
    # - Model used
    # - Estimated cost in USD
    # - Time to first token (streaming)
    # - Total latency
    ```
  </Tab>

  <Tab title="JavaScript" icon="js">
    ```typescript theme={null}
    import { observe, updateActiveObservation } from "@abvdev/tracing";

    const generateWithCostTracking = observe(
      async (prompt: string) => {
        const response = await abv.gateway.chat.completions.create({
          provider: "openai",
          model: "gpt-4o",
          messages: [{ role: "user", content: prompt }],
        });

        // Update with any additional cost info if needed
        updateActiveObservation({
          usageDetails: {
            input: response.usage?.prompt_tokens,
            output: response.usage?.completion_tokens,
          },
        });

        return response.choices[0].message.content;
      },
      { name: "cost-tracked-call", asType: "generation" }
    );

    await generateWithCostTracking("Explain quantum computing");
    await abvSpanProcessor.forceFlush();
    ```
  </Tab>
</Tabs>

***

## Model Parameters

Full OpenAI-compatible parameter support.

<Tabs>
  <Tab title="Python" icon="python">
    ```python theme={null}
    response = abv.gateway.complete_chat(
        provider="openai",
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": "Generate creative options"}],
        temperature=0.9,       # Creativity (0-2)
        max_tokens=1000,       # Response length limit
        top_p=0.95,            # Nucleus sampling
        frequency_penalty=0.5, # Reduce repetition
        presence_penalty=0.5,  # Encourage new topics
        stop=["\n\n"],         # Stop sequences
        n=3,                   # Number of completions
    )

    for i, choice in enumerate(response.choices):
        print(f"Option {i+1}: {choice.message.content}")
    ```
  </Tab>

  <Tab title="JavaScript" icon="js">
    ```typescript theme={null}
    const response = await abv.gateway.chat.completions.create({
      provider: "openai",
      model: "gpt-4o-mini",
      messages: [{ role: "user", content: "Generate creative options" }],
      temperature: 0.9,
      max_tokens: 1000,
      top_p: 0.95,
      frequency_penalty: 0.5,
      presence_penalty: 0.5,
      stop: ["\n\n"],
      n: 3,
    });

    response.choices.forEach((choice, i) => {
      console.log(`Option ${i + 1}: ${choice.message.content}`);
    });
    ```
  </Tab>
</Tabs>

***

## Provider Fallback Pattern

Implement fallback logic when a provider fails.

<Tabs>
  <Tab title="Python" icon="python">
    ```python theme={null}
    from abvdev import ABV, observe

    abv = ABV(api_key="sk-abv-...")

    PROVIDERS = [
        {"provider": "openai", "model": "gpt-4o-mini"},
        {"provider": "anthropic", "model": "claude-sonnet-4-20250514"},
        {"provider": "gemini", "model": "gemini-1.5-flash"},
    ]

    @observe(name="llm-with-fallback")
    def generate_with_fallback(messages: list):
        for config in PROVIDERS:
            try:
                response = abv.gateway.complete_chat(
                    provider=config["provider"],
                    model=config["model"],
                    messages=messages
                )
                return response.choices[0].message.content
            except Exception as e:
                abv.update_current_span(
                    level="WARNING",
                    status_message=f"{config['provider']} failed: {e}"
                )
                continue

        raise Exception("All providers failed")

    result = generate_with_fallback([{"role": "user", "content": "Hello"}])
    ```
  </Tab>

  <Tab title="JavaScript" icon="js">
    ```typescript theme={null}
    import { observe, updateActiveObservation } from "@abvdev/tracing";

    const PROVIDERS = [
      { provider: "openai", model: "gpt-4o-mini" },
      { provider: "anthropic", model: "claude-sonnet-4-20250514" },
      { provider: "gemini", model: "gemini-1.5-flash" },
    ];

    const generateWithFallback = observe(
      async (messages: any[]) => {
        for (const config of PROVIDERS) {
          try {
            const response = await abv.gateway.chat.completions.create({
              provider: config.provider as any,
              model: config.model,
              messages,
            });
            return response.choices[0].message.content;
          } catch (error) {
            updateActiveObservation({
              level: "WARNING",
              statusMessage: `${config.provider} failed: ${error.message}`,
            });
            continue;
          }
        }
        throw new Error("All providers failed");
      },
      { name: "llm-with-fallback" }
    );

    const result = await generateWithFallback([
      { role: "user", content: "Hello" },
    ]);
    ```
  </Tab>
</Tabs>

***

## Using Managed Prompts with Gateway

Fetch prompts from ABV and use them with the gateway.

<Tabs>
  <Tab title="Python" icon="python">
    ```python theme={null}
    from abvdev import ABV

    abv = ABV(api_key="sk-abv-...")

    # Fetch managed prompt
    prompt = abv.get_prompt("customer-support-v2")

    # Compile with variables
    messages = prompt.compile(
        customer_name="Alice",
        issue="billing question"
    )

    # Use with gateway
    response = abv.gateway.complete_chat(
        provider="openai",
        model=prompt.config.get("model", "gpt-4o-mini"),
        messages=messages,
        temperature=prompt.config.get("temperature", 0.7)
    )
    ```
  </Tab>

  <Tab title="JavaScript" icon="js">
    ```typescript theme={null}
    const abv = new ABVClient({ apiKey: "sk-abv-..." });

    // Fetch managed prompt
    const prompt = await abv.prompt.get("customer-support-v2");

    // Compile with variables
    const messages = prompt.compile({
      customer_name: "Alice",
      issue: "billing question",
    });

    // Use with gateway
    const response = await abv.gateway.chat.completions.create({
      provider: "openai",
      model: prompt.config?.model ?? "gpt-4o-mini",
      messages,
      temperature: prompt.config?.temperature ?? 0.7,
    });
    ```
  </Tab>
</Tabs>

<CardGroup cols={2}>
  <Card title="Guardrails Patterns" icon="arrow-right" href="/developer/cookbook/guardrails-patterns">
    Next: Content validation and safety
  </Card>

  <Card title="LLM Gateway Guide" icon="book" href="/developer/llm-gateway/overview">
    Reference: Full gateway documentation
  </Card>
</CardGroup>