Implementasi Routing API AI Multi-Provider dengan Fallback Otomatis

// router.ts
import axios, { AxiosError } from 'axios';

interface ModelProviderConfig {
  name: string;
  endpoint: string;
  apiKey: string;
  model: string;
  timeoutMs: number;
}

interface ChatMessage {
  role: 'system' | 'user' | 'assistant';
  content: string;
}

export class ResilientAIRouter {
  private providers: ModelProviderConfig[];
  private failureCount: Map<string, number> = new Map();
  private circuitOpenUntil: Map<string, number> = new Map();
  private readonly FAILURE_THRESHOLD = 3;
  private readonly COOLDOWN_PERIOD_MS = 60000; // 1 Menit

  constructor(providers: ModelProviderConfig[]) {
    if (!providers || providers.length === 0) {
      throw new Error("Minimal harus ada satu provider yang dikonfigurasi.");
    }
    this.providers = providers;
  }

  private isCircuitOpen(providerName: string): boolean {
    const openUntil = this.circuitOpenUntil.get(providerName) || 0;
    if (Date.now() < openUntil) {
      return true; // Circuit masih terbuka, jangan panggil provider ini
    }
    return false;
  }

  private recordSuccess(providerName: string) {
    this.failureCount.set(providerName, 0);
    this.circuitOpenUntil.delete(providerName);
  }

  private recordFailure(providerName: string) {
    const currentFailures = (this.failureCount.get(providerName) || 0) + 1;
    this.failureCount.set(providerName, currentFailures);

    if (currentFailures >= this.FAILURE_THRESHOLD) {
      this.circuitOpenUntil.set(providerName, Date.now() + this.COOLDOWN_PERIOD_MS);
      console.warn(`[CircuitBreaker] Provider ${providerName} dialihkan ke status OPEN selama 60 detik.`);
    }
  }

  private isTransientError(error: any): boolean {
    if (error.code === 'ECONNABORTED' || error.message?.includes('timeout')) return true;
    if (axios.isAxiosError(error) && error.response) {
      const status = error.response.status;
      return status === 429 || (status >= 500 && status <= 599);
    }
    return false;
  }

  public async complete(messages: ChatMessage[]): Promise<{ text: string; providerUsed: string; latencyMs: number }> {
    let lastError: any = null;

    for (const provider of this.providers) {
      if (this.isCircuitOpen(provider.name)) {
        console.warn(`[Router] Melewati ${provider.name} karena circuit breaker sedang AKTIF(Open).`);
        continue;
      }

      const startTime = Date.now();
      try {
        console.log(`[Router] Mencoba mengeksekusi request via: ${provider.name}...`);
        
        const response = await axios.post(
          provider.endpoint,
          {
            model: provider.model,
            messages: messages,
            temperature: 0.7,
          },
          {
            headers: {
              'Authorization': `Bearer ${provider.apiKey}`,
              'Content-Type': 'application/json',
            },
            timeout: provider.timeoutMs,
          }
        );

        const latencyMs = Date.now() - startTime;
        this.recordSuccess(provider.name);
        
        // Asumsi struktur format respons standar OpenAI-compatible
        const text = response.data.choices[0]?.message?.content || "";
        
        return {
          text,
          providerUsed: provider.name,
          latencyMs
        };

      } catch(err: any) {
        const latencyMs = Date.now() - startTime;
        lastError = err;
        
        console.error(`[Router] Gagal pada provider ${provider.name} (${latencyMs}ms): ${err.message}`);

        if (this.isTransientError(err)) {
          this.recordFailure(provider.name);
          // Lanjut ke iterasi provider berikutnya di dalam loop
          continue;
        } else {
          // Eror non-transient (misal: Bad Request 400), hentikan fallback loop
          throw new Error(`Terminal Error pada ${provider.name}: ${err.response?.data?.error?.message || err.message}`);
        }
      }
    }

    throw new Error(`Seluruh provider gagal mengeksekusi permintaan. Eror terakhir: ${lastError?.message}`);
  }
}

// index.ts
import { ResilientAIRouter } from './router';

async function main() {
  // Susun rantai redundansi berdasarkan prioritas performa dan biaya
  const router = new ResilientAIRouter([
    {
      name: 'Primary-GPT4o',
      endpoint: 'https://api.openai.com/v1/chat/completions',
      apiKey: process.env.OPENAI_API_KEY || '',
      model: 'gpt-4o',
      timeoutMs: 4000 // Batas maksimal toleransi latensi primer: 4 detik
    },
    {
      name: 'Secondary-Claude-Direct',
      endpoint: 'https://api.anthropic.com/v1/messages', // Perlu adapter format payload
      apiKey: process.env.ANTHROPIC_API_KEY || '',
      model: 'claude-3-5-sonnet-20241022',
      timeoutMs: 6000
    },
    {
      name: 'Backup-DeepSeek-Fast',
      endpoint: 'https://api.deepseek.com/v1/chat/completions',
      apiKey: process.env.DEEPSEEK_API_KEY || '',
      model: 'deepseek-chat',
      timeoutMs: 8000
    }
  ]);

  try {
    const result = await router.complete([
      { role: 'system', content: 'Anda adalah asisten coding ahli.' },
      { role: 'user', content: 'Tuliskan fungsi debounce sederhana di JavaScript.' }
    ]);

    console.log(`\nPermintaan berhasil diproses oleh [${result.providerUsed}] dalam ${result.latencyMs}ms:`);
    console.log(result.text);
  } catch(error: any) {
    console.error("Fatal Failure:", error.message);
  }
}

main();

[ Input Prompt Klien ]
         │
         ├───► Format JSON Schema Khusus(Perlu Normalisasi)
         ├───► System Prompt Tone Sensitivity(Variasi Kepatuhan Instruksi)
         └───► Token Length Discrepancy(Perbedaan Tokenizer)

┌────────────────────────────────┐
│      Aplikasi Backend Anda     │
└───────────────┬────────────────┘
                │ (Cukup 1 OpenAI SDK & 1 API Key)
                ▼
┌────────────────────────────────────────────────────────┐
│             AiStudio.id Unified API Gateway            │
│  - Multi-Provider Automatic Failover                   │
│  - Payload & Structured Output Normalization           │
│  - Low Latency Edge Proxying                           │
│  - Terpusat: Pembayaran IDR(QRIS / VA) & Satu Saldo   │
└───────────────┬────────────────────────┬───────────────┘
                │                        │
        ┌───────┴────────┐       ┌───────┴────────┐
        ▼                ▼       ▼                ▼
   [ OpenAI ]       [ Claude ] [ Gemini ]   [ DeepSeek ]