How do I Handle API Rate Limits When Scraping in Go?

API rate limiting is a critical aspect of responsible web scraping that prevents your applications from overwhelming target servers and ensures sustainable data collection. When scraping APIs in Go, implementing proper rate limiting strategies helps you avoid getting blocked, reduces server load, and maintains good relationships with API providers.

Understanding API Rate Limits

Rate limits are restrictions imposed by APIs to control the number of requests a client can make within a specific time window. Common rate limiting patterns include:

• Requests per second (RPS): Maximum number of requests allowed per second
• Requests per minute/hour: Longer time windows with higher quotas
• Burst limits: Short-term allowances for higher request rates
• Concurrent connections: Maximum simultaneous active requests

Basic Rate Limiting with time.Ticker

The simplest approach to rate limiting in Go uses the time.Ticker to control request intervals:

package main

import (
    "fmt"
    "net/http"
    "time"
)

type RateLimitedClient struct {
    client *http.Client
    ticker *time.Ticker
}

func NewRateLimitedClient(requestsPerSecond float64) *RateLimitedClient {
    interval := time.Duration(float64(time.Second) / requestsPerSecond)
    return &RateLimitedClient{
        client: &http.Client{Timeout: 30 * time.Second},
        ticker: time.NewTicker(interval),
    }
}

func (rlc *RateLimitedClient) Get(url string) (*http.Response, error) {
    <-rlc.ticker.C // Wait for the next tick
    return rlc.client.Get(url)
}

func (rlc *RateLimitedClient) Close() {
    rlc.ticker.Stop()
}

func main() {
    client := NewRateLimitedClient(2.0) // 2 requests per second
    defer client.Close()

    urls := []string{
        "https://api.example.com/data/1",
        "https://api.example.com/data/2",
        "https://api.example.com/data/3",
    }

    for _, url := range urls {
        resp, err := client.Get(url)
        if err != nil {
            fmt.Printf("Error fetching %s: %v\n", url, err)
            continue
        }
        resp.Body.Close()
        fmt.Printf("Successfully fetched: %s\n", url)
    }
}

Advanced Rate Limiting with Token Bucket Algorithm

For more sophisticated rate limiting that supports burst requests, implement a token bucket algorithm:

package main

import (
    "context"
    "fmt"
    "net/http"
    "sync"
    "time"
)

type TokenBucket struct {
    capacity int
    tokens   int
    refill   time.Duration
    mutex    sync.Mutex
    lastRefill time.Time
}

func NewTokenBucket(capacity int, refillRate time.Duration) *TokenBucket {
    return &TokenBucket{
        capacity:   capacity,
        tokens:     capacity,
        refill:     refillRate,
        lastRefill: time.Now(),
    }
}

func (tb *TokenBucket) Take() bool {
    tb.mutex.Lock()
    defer tb.mutex.Unlock()

    now := time.Now()
    elapsed := now.Sub(tb.lastRefill)
    tokensToAdd := int(elapsed / tb.refill)

    if tokensToAdd > 0 {
        tb.tokens = min(tb.capacity, tb.tokens+tokensToAdd)
        tb.lastRefill = now
    }

    if tb.tokens > 0 {
        tb.tokens--
        return true
    }
    return false
}

func min(a, b int) int {
    if a < b {
        return a
    }
    return b
}

type TokenBucketClient struct {
    client *http.Client
    bucket *TokenBucket
}

func NewTokenBucketClient(capacity int, refillRate time.Duration) *TokenBucketClient {
    return &TokenBucketClient{
        client: &http.Client{Timeout: 30 * time.Second},
        bucket: NewTokenBucket(capacity, refillRate),
    }
}

func (tbc *TokenBucketClient) Get(ctx context.Context, url string) (*http.Response, error) {
    for {
        if tbc.bucket.Take() {
            req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
            if err != nil {
                return nil, err
            }
            return tbc.client.Do(req)
        }

        select {
        case <-ctx.Done():
            return nil, ctx.Err()
        case <-time.After(100 * time.Millisecond):
            // Wait a bit before trying again
        }
    }
}

Implementing Exponential Backoff

When dealing with rate limit errors (HTTP 429), implement exponential backoff to gradually increase wait times:

package main

import (
    "context"
    "fmt"
    "math"
    "net/http"
    "strconv"
    "time"
)

type BackoffClient struct {
    client      *http.Client
    maxRetries  int
    baseDelay   time.Duration
    maxDelay    time.Duration
}

func NewBackoffClient(maxRetries int, baseDelay, maxDelay time.Duration) *BackoffClient {
    return &BackoffClient{
        client:     &http.Client{Timeout: 30 * time.Second},
        maxRetries: maxRetries,
        baseDelay:  baseDelay,
        maxDelay:   maxDelay,
    }
}

func (bc *BackoffClient) GetWithRetry(ctx context.Context, url string) (*http.Response, error) {
    var resp *http.Response
    var err error

    for attempt := 0; attempt <= bc.maxRetries; attempt++ {
        req, reqErr := http.NewRequestWithContext(ctx, "GET", url, nil)
        if reqErr != nil {
            return nil, reqErr
        }

        resp, err = bc.client.Do(req)
        if err != nil {
            return nil, err
        }

        // Success case
        if resp.StatusCode < 400 {
            return resp, nil
        }

        // Handle rate limiting
        if resp.StatusCode == 429 {
            resp.Body.Close()

            if attempt == bc.maxRetries {
                return nil, fmt.Errorf("max retries exceeded for URL: %s", url)
            }

            delay := bc.calculateDelay(attempt, resp)
            fmt.Printf("Rate limited, waiting %v before retry %d\n", delay, attempt+1)

            select {
            case <-ctx.Done():
                return nil, ctx.Err()
            case <-time.After(delay):
                continue
            }
        }

        // Other errors
        resp.Body.Close()
        return nil, fmt.Errorf("HTTP error %d for URL: %s", resp.StatusCode, url)
    }

    return resp, err
}

func (bc *BackoffClient) calculateDelay(attempt int, resp *http.Response) time.Duration {
    // Check for Retry-After header
    if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" {
        if seconds, err := strconv.Atoi(retryAfter); err == nil {
            return time.Duration(seconds) * time.Second
        }
    }

    // Exponential backoff with jitter
    delay := time.Duration(math.Pow(2, float64(attempt))) * bc.baseDelay
    if delay > bc.maxDelay {
        delay = bc.maxDelay
    }

    // Add jitter (±25%)
    jitter := time.Duration(float64(delay) * 0.25 * (2*time.Now().UnixNano()%2 - 1) / 1e9)
    return delay + jitter
}

Comprehensive Rate Limiting Solution

Here's a production-ready implementation that combines multiple strategies:

package main

import (
    "context"
    "fmt"
    "net/http"
    "sync"
    "time"
)

type RateLimiter struct {
    bucket     *TokenBucket
    lastReq    time.Time
    minDelay   time.Duration
    mutex      sync.Mutex
}

func NewRateLimiter(rps float64, burst int) *RateLimiter {
    interval := time.Duration(float64(time.Second) / rps)
    return &RateLimiter{
        bucket:   NewTokenBucket(burst, interval),
        minDelay: interval,
    }
}

func (rl *RateLimiter) Wait(ctx context.Context) error {
    rl.mutex.Lock()
    defer rl.mutex.Unlock()

    now := time.Now()
    elapsed := now.Sub(rl.lastReq)

    if elapsed < rl.minDelay {
        waitTime := rl.minDelay - elapsed
        select {
        case <-ctx.Done():
            return ctx.Err()
        case <-time.After(waitTime):
        }
    }

    // Wait for token if bucket is empty
    for !rl.bucket.Take() {
        select {
        case <-ctx.Done():
            return ctx.Err()
        case <-time.After(50 * time.Millisecond):
        }
    }

    rl.lastReq = time.Now()
    return nil
}

type ScrapingClient struct {
    client      *http.Client
    rateLimiter *RateLimiter
    backoff     *BackoffClient
}

func NewScrapingClient(rps float64, burst int) *ScrapingClient {
    return &ScrapingClient{
        client:      &http.Client{Timeout: 30 * time.Second},
        rateLimiter: NewRateLimiter(rps, burst),
        backoff:     NewBackoffClient(3, 1*time.Second, 30*time.Second),
    }
}

func (sc *ScrapingClient) Get(ctx context.Context, url string) (*http.Response, error) {
    // Apply rate limiting
    if err := sc.rateLimiter.Wait(ctx); err != nil {
        return nil, err
    }

    // Make request with retry logic
    return sc.backoff.GetWithRetry(ctx, url)
}

// Example usage with concurrent scraping
func main() {
    client := NewScrapingClient(5.0, 10) // 5 RPS, burst of 10

    urls := []string{
        "https://api.example.com/users/1",
        "https://api.example.com/users/2",
        "https://api.example.com/users/3",
        "https://api.example.com/users/4",
        "https://api.example.com/users/5",
    }

    ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
    defer cancel()

    var wg sync.WaitGroup
    semaphore := make(chan struct{}, 3) // Limit concurrent requests

    for _, url := range urls {
        wg.Add(1)
        go func(u string) {
            defer wg.Done()

            semaphore <- struct{}{} // Acquire
            defer func() { <-semaphore }() // Release

            resp, err := client.Get(ctx, u)
            if err != nil {
                fmt.Printf("Error fetching %s: %v\n", u, err)
                return
            }
            defer resp.Body.Close()

            fmt.Printf("Successfully fetched %s (Status: %d)\n", u, resp.StatusCode)
        }(url)
    }

    wg.Wait()
    fmt.Println("All requests completed")
}

Monitoring and Observability

Add logging and metrics to monitor your rate limiting effectiveness:

package main

import (
    "context"
    "fmt"
    "log"
    "net/http"
    "sync/atomic"
    "time"
)

type Metrics struct {
    RequestsTotal    int64
    RequestsLimited  int64
    RequestsRetried  int64
    RequestsFailed   int64
}

func (m *Metrics) LogStats() {
    total := atomic.LoadInt64(&m.RequestsTotal)
    limited := atomic.LoadInt64(&m.RequestsLimited)
    retried := atomic.LoadInt64(&m.RequestsRetried)
    failed := atomic.LoadInt64(&m.RequestsFailed)

    fmt.Printf("Stats - Total: %d, Limited: %d, Retried: %d, Failed: %d\n", 
        total, limited, retried, failed)
}

type MonitoredClient struct {
    client  *ScrapingClient
    metrics *Metrics
}

func NewMonitoredClient(rps float64, burst int) *MonitoredClient {
    return &MonitoredClient{
        client:  NewScrapingClient(rps, burst),
        metrics: &Metrics{},
    }
}

func (mc *MonitoredClient) Get(ctx context.Context, url string) (*http.Response, error) {
    atomic.AddInt64(&mc.metrics.RequestsTotal, 1)

    start := time.Now()
    resp, err := mc.client.Get(ctx, url)
    duration := time.Since(start)

    if err != nil {
        atomic.AddInt64(&mc.metrics.RequestsFailed, 1)
        log.Printf("Request failed for %s after %v: %v", url, duration, err)
        return nil, err
    }

    if resp.StatusCode == 429 {
        atomic.AddInt64(&mc.metrics.RequestsLimited, 1)
    }

    log.Printf("Request to %s completed in %v (Status: %d)", url, duration, resp.StatusCode)
    return resp, nil
}

func (mc *MonitoredClient) GetMetrics() *Metrics {
    return mc.metrics
}

Best Practices and Tips

1. Respect Rate Limit Headers

Always check and respect standard rate limiting headers: - X-RateLimit-Limit: Maximum requests allowed - X-RateLimit-Remaining: Remaining requests in current window - X-RateLimit-Reset: Time when the limit resets - Retry-After: Seconds to wait before next request

2. Use Context for Cancellation

Always use context.Context to handle timeouts and cancellations gracefully in your rate-limited requests.

3. Implement Circuit Breakers

For production systems, consider implementing circuit breaker patterns to temporarily stop requests when error rates exceed thresholds.

4. Monitor and Adjust

Continuously monitor your scraping performance and adjust rate limits based on API provider feedback and error rates.

Conclusion

Handling API rate limits effectively in Go requires a combination of proper timing control, retry mechanisms, and monitoring. By implementing token bucket algorithms, exponential backoff, and comprehensive error handling, you can build robust scraping applications that respect API constraints while maintaining high performance.

When implementing rate limiting for web scraping, consider combining these Go techniques with browser automation tools for JavaScript-heavy sites. For advanced scenarios involving monitoring network requests in Puppeteer or handling timeouts in Puppeteer, you may need to coordinate rate limiting across different scraping technologies.

Remember that responsible scraping not only prevents your applications from being blocked but also helps maintain a healthy ecosystem for data access across the web.