How do I implement retry logic for failed HTTP requests in Go?

When building robust web applications and scrapers in Go, implementing retry logic for HTTP requests is essential for handling transient network failures, server overload, and rate limiting. This comprehensive guide covers various retry strategies and patterns for building resilient HTTP clients.

Basic Retry Implementation

The simplest retry mechanism involves wrapping your HTTP request in a loop with a maximum retry count:

package main

import (
    "fmt"
    "net/http"
    "time"
)

func makeRequestWithRetry(url string, maxRetries int) (*http.Response, error) {
    var resp *http.Response
    var err error

    for attempt := 0; attempt <= maxRetries; attempt++ {
        resp, err = http.Get(url)
        if err == nil && resp.StatusCode < 500 {
            return resp, nil
        }

        if resp != nil {
            resp.Body.Close()
        }

        if attempt < maxRetries {
            fmt.Printf("Request failed (attempt %d/%d), retrying...\n", attempt+1, maxRetries+1)
            time.Sleep(time.Second * time.Duration(attempt+1))
        }
    }

    return nil, fmt.Errorf("request failed after %d attempts: %v", maxRetries+1, err)
}

Exponential Backoff Strategy

Exponential backoff increases the delay between retries exponentially, reducing load on the server and improving success rates:

package main

import (
    "context"
    "fmt"
    "math"
    "net/http"
    "time"
)

type RetryConfig struct {
    MaxRetries  int
    BaseDelay   time.Duration
    MaxDelay    time.Duration
    Multiplier  float64
}

func makeRequestWithExponentialBackoff(ctx context.Context, url string, config RetryConfig) (*http.Response, error) {
    var resp *http.Response
    var err error

    for attempt := 0; attempt <= config.MaxRetries; attempt++ {
        resp, err = http.Get(url)

        // Check if request was successful
        if err == nil && resp.StatusCode < 500 {
            return resp, nil
        }

        if resp != nil {
            resp.Body.Close()
        }

        // Don't wait after the last attempt
        if attempt < config.MaxRetries {
            delay := calculateBackoffDelay(attempt, config)

            select {
            case <-time.After(delay):
                // Continue to next attempt
            case <-ctx.Done():
                return nil, ctx.Err()
            }
        }
    }

    return nil, fmt.Errorf("request failed after %d attempts: %v", config.MaxRetries+1, err)
}

func calculateBackoffDelay(attempt int, config RetryConfig) time.Duration {
    delay := float64(config.BaseDelay) * math.Pow(config.Multiplier, float64(attempt))
    if delay > float64(config.MaxDelay) {
        delay = float64(config.MaxDelay)
    }
    return time.Duration(delay)
}

Advanced Retry Client with Jitter

Adding jitter (random variation) to retry delays helps prevent the "thundering herd" problem when multiple clients retry simultaneously:

package main

import (
    "context"
    "fmt"
    "math"
    "math/rand"
    "net/http"
    "time"
)

type HTTPRetryClient struct {
    client     *http.Client
    maxRetries int
    baseDelay  time.Duration
    maxDelay   time.Duration
    jitter     bool
}

func NewHTTPRetryClient(maxRetries int, baseDelay, maxDelay time.Duration) *HTTPRetryClient {
    return &HTTPRetryClient{
        client:     &http.Client{Timeout: 30 * time.Second},
        maxRetries: maxRetries,
        baseDelay:  baseDelay,
        maxDelay:   maxDelay,
        jitter:     true,
    }
}

func (c *HTTPRetryClient) Do(ctx context.Context, req *http.Request) (*http.Response, error) {
    var resp *http.Response
    var err error

    for attempt := 0; attempt <= c.maxRetries; attempt++ {
        // Clone the request for each attempt
        reqClone := req.Clone(ctx)

        resp, err = c.client.Do(reqClone)

        if c.shouldRetry(resp, err, attempt) {
            if resp != nil {
                resp.Body.Close()
            }

            if attempt < c.maxRetries {
                delay := c.calculateDelay(attempt)
                select {
                case <-time.After(delay):
                    continue
                case <-ctx.Done():
                    return nil, ctx.Err()
                }
            }
        } else {
            return resp, err
        }
    }

    return nil, fmt.Errorf("request failed after %d attempts: %v", c.maxRetries+1, err)
}

func (c *HTTPRetryClient) shouldRetry(resp *http.Response, err error, attempt int) bool {
    if attempt >= c.maxRetries {
        return false
    }

    // Retry on network errors
    if err != nil {
        return true
    }

    // Retry on specific HTTP status codes
    if resp != nil {
        switch resp.StatusCode {
        case http.StatusTooManyRequests, // 429
             http.StatusInternalServerError, // 500
             http.StatusBadGateway, // 502
             http.StatusServiceUnavailable, // 503
             http.StatusGatewayTimeout: // 504
            return true
        }
    }

    return false
}

func (c *HTTPRetryClient) calculateDelay(attempt int) time.Duration {
    delay := float64(c.baseDelay) * math.Pow(2, float64(attempt))

    if delay > float64(c.maxDelay) {
        delay = float64(c.maxDelay)
    }

    // Add jitter (±25% random variation)
    if c.jitter {
        jitterRange := delay * 0.25
        jitter := (rand.Float64() - 0.5) * 2 * jitterRange
        delay += jitter
    }

    return time.Duration(math.Max(delay, float64(c.baseDelay)))
}

Circuit Breaker Pattern

For applications that make many requests, implementing a circuit breaker can prevent cascading failures:

package main

import (
    "context"
    "fmt"
    "net/http"
    "sync"
    "time"
)

type CircuitState int

const (
    Closed CircuitState = iota
    Open
    HalfOpen
)

type CircuitBreaker struct {
    mu                sync.RWMutex
    state            CircuitState
    failureCount     int
    successCount     int
    failureThreshold int
    resetTimeout     time.Duration
    lastFailureTime  time.Time
}

func NewCircuitBreaker(failureThreshold int, resetTimeout time.Duration) *CircuitBreaker {
    return &CircuitBreaker{
        state:            Closed,
        failureThreshold: failureThreshold,
        resetTimeout:     resetTimeout,
    }
}

func (cb *CircuitBreaker) Call(ctx context.Context, fn func() (*http.Response, error)) (*http.Response, error) {
    cb.mu.RLock()
    state := cb.state
    cb.mu.RUnlock()

    if state == Open {
        if time.Since(cb.lastFailureTime) > cb.resetTimeout {
            cb.mu.Lock()
            cb.state = HalfOpen
            cb.successCount = 0
            cb.mu.Unlock()
        } else {
            return nil, fmt.Errorf("circuit breaker is open")
        }
    }

    resp, err := fn()

    cb.mu.Lock()
    defer cb.mu.Unlock()

    if err != nil || (resp != nil && resp.StatusCode >= 500) {
        cb.onFailure()
        return resp, err
    }

    cb.onSuccess()
    return resp, err
}

func (cb *CircuitBreaker) onFailure() {
    cb.failureCount++
    cb.lastFailureTime = time.Now()

    if cb.failureCount >= cb.failureThreshold {
        cb.state = Open
    }
}

func (cb *CircuitBreaker) onSuccess() {
    cb.failureCount = 0

    if cb.state == HalfOpen {
        cb.successCount++
        if cb.successCount >= 3 { // Require 3 successes to close
            cb.state = Closed
        }
    }
}

Practical Usage Examples

Here's how to use the retry client in real-world scenarios:

func main() {
    // Basic usage
    config := RetryConfig{
        MaxRetries: 3,
        BaseDelay:  100 * time.Millisecond,
        MaxDelay:   5 * time.Second,
        Multiplier: 2.0,
    }

    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    resp, err := makeRequestWithExponentialBackoff(ctx, "https://api.example.com/data", config)
    if err != nil {
        fmt.Printf("Request failed: %v\n", err)
        return
    }
    defer resp.Body.Close()

    // Advanced client usage
    retryClient := NewHTTPRetryClient(5, 100*time.Millisecond, 10*time.Second)

    req, err := http.NewRequest("GET", "https://api.example.com/data", nil)
    if err != nil {
        fmt.Printf("Failed to create request: %v\n", err)
        return
    }

    resp, err = retryClient.Do(ctx, req)
    if err != nil {
        fmt.Printf("Request failed: %v\n", err)
        return
    }
    defer resp.Body.Close()

    fmt.Printf("Request successful: %d\n", resp.StatusCode)
}

Integration with Web Scraping

When building web scrapers, retry logic becomes crucial for handling rate limiting and timeouts. Here's how to integrate retry logic with a scraping workflow:

package main

import (
    "context"
    "fmt"
    "io"
    "net/http"
    "strings"
    "time"
)

type Scraper struct {
    client *HTTPRetryClient
}

func NewScraper() *Scraper {
    return &Scraper{
        client: NewHTTPRetryClient(3, 200*time.Millisecond, 5*time.Second),
    }
}

func (s *Scraper) ScrapeURL(ctx context.Context, url string) (string, error) {
    req, err := http.NewRequest("GET", url, nil)
    if err != nil {
        return "", err
    }

    // Set headers to appear more like a regular browser
    req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; GoScraper/1.0)")
    req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")

    resp, err := s.client.Do(ctx, req)
    if err != nil {
        return "", fmt.Errorf("failed to fetch %s: %v", url, err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
    }

    body, err := io.ReadAll(resp.Body)
    if err != nil {
        return "", fmt.Errorf("failed to read response body: %v", err)
    }

    return string(body), nil
}

Testing Retry Logic

Testing retry behavior is important for ensuring your implementation works correctly:

package main

import (
    "context"
    "net/http"
    "net/http/httptest"
    "testing"
    "time"
)

func TestRetryLogic(t *testing.T) {
    attempts := 0
    server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        attempts++
        if attempts < 3 {
            w.WriteHeader(http.StatusInternalServerError)
            return
        }
        w.WriteHeader(http.StatusOK)
        w.Write([]byte("success"))
    }))
    defer server.Close()

    client := NewHTTPRetryClient(5, 10*time.Millisecond, 100*time.Millisecond)
    req, _ := http.NewRequest("GET", server.URL, nil)

    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    defer cancel()

    resp, err := client.Do(ctx, req)
    if err != nil {
        t.Fatalf("Expected success, got error: %v", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        t.Fatalf("Expected 200, got %d", resp.StatusCode)
    }

    if attempts != 3 {
        t.Fatalf("Expected 3 attempts, got %d", attempts)
    }
}

Best Practices

1. Set appropriate timeouts: Always use context with timeout for overall request duration
2. Handle specific error types: Distinguish between retryable and non-retryable errors
3. Implement backoff strategies: Use exponential backoff with jitter to avoid overwhelming servers
4. Log retry attempts: Include detailed logging for debugging and monitoring
5. Consider rate limits: Respect server rate limits and implement appropriate delays
6. Use circuit breakers: Prevent cascading failures in high-traffic applications

Conclusion

Implementing robust retry logic in Go requires careful consideration of various factors including backoff strategies, error handling, and timeout management. The patterns shown above provide a solid foundation for building resilient HTTP clients that can gracefully handle network failures and server errors. When combined with proper error handling techniques, these retry mechanisms ensure your Go applications remain stable and reliable even under adverse network conditions.