Concurrent function calls | Boundary Documentation

We’ll use function ClassifyMessage(input: string) -> Category for our example:

classify-message.baml

enum Category {
    Refund
    CancelOrder
    TechnicalSupport
    AccountIssue
    Question
}
function ClassifyMessage(input: string) -> Category {
  client GPT4o
  prompt #"
    Classify the following INPUT into ONE
    of the following categories:
    INPUT: {{ input }}
    {{ ctx.output_format }}
    Response:
  "#
}

Python

TypeScript

Go

Ruby (beta)

Rust

You can make concurrent b.ClassifyMessage() calls like so:

main.py

import asyncio
from baml_client.async_client import b
from baml_client.types import Category
async def main():
    await asyncio.gather(
        b.ClassifyMessage("I want to cancel my order"),
        b.ClassifyMessage("I want a refund")
    )
if __name__ == '__main__':
    asyncio.run(main())

Cancelling Parallel Operations

When running multiple operations in parallel, you can use abort controllers to cancel them all at once or individually.

Cancel All Operations

Use a single abort controller to cancel all parallel operations:

TypeScript

Python

Go

Rust

import { b } from './baml_client'
const controller = new AbortController()
// Start multiple operations with the same controller
const promises = [
  b.ClassifyMessage('I want to cancel my order', { abortController: controller }),
  b.ClassifyMessage('I want a refund', { abortController: controller }),
  b.ClassifyMessage('Is my package shipped?', { abortController: controller })
]
// Cancel all operations after 2 seconds
setTimeout(() => {
  controller.abort()
  console.log('All operations cancelled')
}, 2000)
try {
  const results = await Promise.all(promises)
  console.log('All completed:', results)
} catch (error) {
  if (error.name === 'BamlAbortError') {
    console.log('Operations were cancelled')
  }
}

Cancel Individual Operations

Use separate controllers to cancel operations independently:

TypeScript

Python

const controllers = [
  new AbortController(),
  new AbortController(),
  new AbortController()
]
const promises = [
  b.ClassifyMessage('I want to cancel my order', { abortController: controllers[0] }),
  b.ClassifyMessage('I want a refund', { abortController: controllers[1] }),
  b.ClassifyMessage('Is my package shipped?', { abortController: controllers[2] })
]
// Cancel only the second operation
controllers[1].abort()
const results = await Promise.allSettled(promises)
results.forEach((result, index) => {
  if (result.status === 'fulfilled') {
    console.log(`Operation ${index} completed:`, result.value)
  } else {
    console.log(`Operation ${index} failed:`, result.reason.message)
  }
})

Fastest Request Wins

Race multiple LLM providers and cancel slower ones when the fastest completes. This pattern is useful for optimizing latency by using whichever provider responds first.

TypeScript

Python

Go

Rust

import { ClientRegistry } from '@boundaryml/baml'
async function fastestProviderWins(message: string) {
  const controllers = [
    new AbortController(),
    new AbortController(),
    new AbortController()
  ]
  // Create separate client registries for each provider
  const openaiRegistry = new ClientRegistry()
  openaiRegistry.addLlmClient('OpenAI', 'openai', {
    model: 'gpt-5-mini',
    api_key: process.env.OPENAI_API_KEY
  })
  openaiRegistry.setPrimary('OpenAI')
  const anthropicRegistry = new ClientRegistry()
  anthropicRegistry.addLlmClient('Anthropic', 'anthropic', {
    model: 'claude-3-5-haiku-20241022',
    api_key: process.env.ANTHROPIC_API_KEY
  })
  anthropicRegistry.setPrimary('Anthropic')
  const geminiRegistry = new ClientRegistry()
  geminiRegistry.addLlmClient('Gemini', 'vertex-ai', {
    model: 'gemini-3.5-flash',
    location: 'us-central1',
    credentials: process.env.GOOGLE_APPLICATION_CREDENTIALS
  })
  geminiRegistry.setPrimary('Gemini')
  const promises = [
    b.ClassifyMessage(message, {
      clientRegistry: openaiRegistry,
      abortController: controllers[0]
    }),
    b.ClassifyMessage(message, {
      clientRegistry: anthropicRegistry,
      abortController: controllers[1]
    }),
    b.ClassifyMessage(message, {
      clientRegistry: geminiRegistry,
      abortController: controllers[2]
    })
  ]
  try {
    // Wait for the first to complete
    const result = await Promise.race(promises)
    
    // Cancel the others
    controllers.forEach(c => c.abort())
    
    return result
  } catch (error) {
    // All failed - cancel any still running
    controllers.forEach(c => c.abort())
    throw error
  }
}

Implementing Timeouts for Parallel Operations

Set automatic timeouts to prevent operations from running indefinitely:

TypeScript

Python

Go

Rust

async function classifyWithTimeout(messages: string[], timeoutMs: number = 5000) {
  const controller = new AbortController()
  
  // Set timeout for all operations
  const timeoutId = setTimeout(() => {
    controller.abort()
  }, timeoutMs)
  try {
    const promises = messages.map(msg => 
      b.ClassifyMessage(msg, { abortController: controller })
    )
    
    const results = await Promise.all(promises)
    clearTimeout(timeoutId)
    return results
  } catch (error) {
    clearTimeout(timeoutId)
    if (error.name === 'BamlAbortError') {
      throw new Error(`Operations timed out after ${timeoutMs}ms`)
    }
    throw error
  }
}

Batching with Cancellation Support

Process items in batches with the ability to cancel remaining batches:

TypeScript

Python

async function processBatches<T, R>(
  items: T[],
  batchSize: number,
  processor: (item: T, controller: AbortController) => Promise<R>
): Promise<R[]> {
  const results: R[] = []
  const masterController = new AbortController()
  
  try {
    for (let i = 0; i < items.length; i += batchSize) {
      const batch = items.slice(i, i + batchSize)
      
      // Check if we should stop
      if (masterController.signal.aborted) {
        throw new Error('Batch processing cancelled')
      }
      
      // Process batch in parallel
      const batchPromises = batch.map(item => 
        processor(item, masterController)
      )
      
      const batchResults = await Promise.all(batchPromises)
      results.push(...batchResults)
      
      console.log(`Completed batch ${Math.floor(i / batchSize) + 1}`)
    }
    
    return results
  } catch (error) {
    masterController.abort()
    throw error
  }
}
// Usage
const messages = ['message1', 'message2', 'message3', /*...*/]
const results = await processBatches(
  messages,
  5, // batch size
  (msg, controller) => b.ClassifyMessage(msg, { abortController: controller })
)

For basic abort controller usage and error handling, see the Abort Controllers guide.