LLM Gateway: The Complete Architecture, Comparison & Guide

import { OpenAI } from "openai";

export interface GatewayExecutionRequest {
  prompt: string;
  maxTokens?: number;
}

export class ResilientGatewayDispatcher {
  private gatewayClient: OpenAI;
  private primaryModel: string = "claude-sonnet-5";
  private failoverModel: string = "deepseek-v4";

  constructor(apiKey: string, baseURL: string = "https://apivel.com/v1") {
    this.gatewayClient = new OpenAI({ apiKey, baseURL });
  }

  async executeWithAutoFailover(req: GatewayExecutionRequest): Promise {
    const candidateModels = [this.primaryModel, this.failoverModel];

    for (const model of candidateModels) {
      try {
        const response = await this.gatewayClient.chat.completions.create({
          model,
          messages: [{ role: "user", content: req.prompt }],
          max_tokens: req.maxTokens || 2048,
        });

        const text = response.choices[0]?.message?.content;
        if (text) return text;
      } catch (err: any) {
        const status = err?.status || err?.statusCode;
        console.warn(`[Gateway Warning] Model ${model} returned error ${status}. Executing fast failover...`);

        // If rate limited (429) or upstream server error (5xx), instantly proceed to next model in pool
        if (status === 429 || status >= 500) {
          continue;
        }
        throw err;
      }
    }

    throw new Error("[Gateway Deadlock] All model providers in the pool failed to respond.");
  }
}