Streaming

const client = new GatelitClient({
  gatewayUrl: "https://gateway.yourapp.com",
  getToken: async () => ({ token: "sk_live_...", scheme: "GatelitKey" }),
})

for await (const chunk of client.stream({
  model: "anthropic/claude-3-5-haiku-20241022",
  messages: [{ role: "user", content: "Write a haiku about clouds." }],
})) {
  process.stdout.write(chunk.delta)
}
// → White mountains of air
// → Drifting across the blue sky
// → Rain waits in their shade

let fullText = ""
let usage = null

for await (const chunk of client.stream({
  model: "openai/gpt-4o",
  messages: [{ role: "user", content: "Explain TCP in one paragraph." }],
})) {
  fullText += chunk.delta
  if (chunk.usage) usage = chunk.usage
}

console.log(fullText)
console.log(`Tokens: ${usage?.totalTokens}`)

const controller = new AbortController()

for await (const chunk of client.stream({
  model: "openai/gpt-4o",
  messages: [{ role: "user", content: "List all prime numbers." }],
  signal: controller.signal,
})) {
  process.stdout.write(chunk.delta)
  if (fullText.length > 500) controller.abort()
}

function useStream(model: string) {
  const [text, setText] = useState("")
  const [streaming, setStreaming] = useState(false)
  const controller = useRef<AbortController>()

  async function send(message: string) {
    setStreaming(true)
    setText("")
    controller.current = new AbortController()

    try {
      for await (const chunk of client.stream({
        model,
        messages: [{ role: "user", content: message }],
        signal: controller.current.signal,
      })) {
        setText(prev => prev + chunk.delta)
      }
    } finally {
      setStreaming(false)
    }
  }

  function stop() { controller.current?.abort() }

  return { text, streaming, send, stop }
}

for await (const chunk of client.stream({
  model: "openai/gpt-4o",
  messages: [{ role: "user", content: "Hi" }],
  promptId: "my-assistant",
})) {
  process.stdout.write(chunk.delta)
}