Reqwest's async functionality enables efficient concurrent HTTP requests in Rust applications. By leveraging Rust's async/await syntax with the Tokio runtime, you can perform multiple HTTP requests simultaneously, significantly improving performance compared to sequential requests.
Project Setup
Add the required dependencies to your Cargo.toml:
[dependencies]
reqwest = { version = "0.11", features = ["json"] }
tokio = { version = "1", features = ["full"] }
futures = "0.3"
Note: Use the default Tokio runtime instead of async-std-runtime for better compatibility.
Basic Concurrent Requests with join_all
The simplest approach uses futures::future::join_all to wait for all requests to complete:
use reqwest;
use tokio;
use futures::future::join_all;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let urls = vec![
"https://httpbin.org/delay/1",
"https://httpbin.org/delay/2",
"https://httpbin.org/delay/1",
];
let client = reqwest::Client::new();
// Create futures for all requests
let futures: Vec<_> = urls.into_iter()
.map(|url| {
let client = client.clone();
async move {
let response = client.get(url).send().await?;
let status = response.status();
let text = response.text().await?;
Ok::<(reqwest::StatusCode, String), reqwest::Error>((status, text))
}
})
.collect();
// Execute all requests concurrently
let results = join_all(futures).await;
for (i, result) in results.into_iter().enumerate() {
match result {
Ok((status, body)) => println!("Request {}: {} - {} bytes", i, status, body.len()),
Err(e) => eprintln!("Request {} failed: {}", i, e),
}
}
Ok(())
}
Using tokio::spawn for True Parallelism
For better concurrency control and true parallel execution, use tokio::spawn:
use reqwest;
use tokio;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let urls = vec![
"https://httpbin.org/json",
"https://httpbin.org/uuid",
"https://httpbin.org/ip",
];
let client = reqwest::Client::new();
let mut handles = Vec::new();
// Spawn tasks for concurrent execution
for (i, url) in urls.into_iter().enumerate() {
let client = client.clone();
let handle = tokio::spawn(async move {
let start = std::time::Instant::now();
let result = client.get(url).send().await;
let duration = start.elapsed();
match result {
Ok(response) => {
let status = response.status();
let json: serde_json::Value = response.json().await?;
Ok::<(usize, reqwest::StatusCode, serde_json::Value, std::time::Duration), reqwest::Error>(
(i, status, json, duration)
)
}
Err(e) => Err(e),
}
});
handles.push(handle);
}
// Collect results
for handle in handles {
match handle.await? {
Ok((id, status, json, duration)) => {
println!("Task {}: {} ({:?}) - {:?}", id, status, duration, json);
}
Err(e) => eprintln!("Task failed: {}", e),
}
}
Ok(())
}
Controlled Concurrency with Semaphores
To limit the number of concurrent requests and avoid overwhelming servers:
use reqwest;
use tokio;
use tokio::sync::Semaphore;
use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let urls = (1..=20).map(|i| format!("https://httpbin.org/delay/{}", i % 3 + 1)).collect::<Vec<_>>();
let client = reqwest::Client::new();
let semaphore = Arc::new(Semaphore::new(5)); // Limit to 5 concurrent requests
let mut handles = Vec::new();
for (i, url) in urls.into_iter().enumerate() {
let client = client.clone();
let semaphore = semaphore.clone();
let handle = tokio::spawn(async move {
let _permit = semaphore.acquire().await.unwrap();
println!("Starting request {} to {}", i, url);
let start = std::time::Instant::now();
let result = client
.get(&url)
.timeout(std::time::Duration::from_secs(10))
.send()
.await;
let duration = start.elapsed();
match result {
Ok(response) => {
println!("Completed request {} in {:?} - Status: {}", i, duration, response.status());
Ok(response.text().await?)
}
Err(e) => {
eprintln!("Request {} failed: {}", i, e);
Err(e)
}
}
});
handles.push(handle);
}
// Wait for all tasks to complete
for handle in handles {
let _ = handle.await?;
}
Ok(())
}
POST Requests with JSON Data
Concurrent POST requests with different payloads:
use reqwest;
use tokio;
use serde_json::json;
use futures::future::join_all;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let client = reqwest::Client::new();
let payloads = vec![
json!({"name": "Alice", "age": 30}),
json!({"name": "Bob", "age": 25}),
json!({"name": "Charlie", "age": 35}),
];
let futures: Vec<_> = payloads.into_iter()
.enumerate()
.map(|(i, payload)| {
let client = client.clone();
async move {
let response = client
.post("https://httpbin.org/post")
.json(&payload)
.send()
.await?;
let status = response.status();
let response_json: serde_json::Value = response.json().await?;
Ok::<(usize, reqwest::StatusCode, serde_json::Value), reqwest::Error>(
(i, status, response_json)
)
}
})
.collect();
let results = join_all(futures).await;
for result in results {
match result {
Ok((id, status, json)) => {
println!("POST {}: {} - Echoed data: {:?}",
id, status, json.get("json"));
}
Err(e) => eprintln!("POST failed: {}", e),
}
}
Ok(())
}
Error Handling and Retry Logic
Robust concurrent requests with retry mechanisms:
use reqwest;
use tokio;
use std::time::Duration;
async fn fetch_with_retry(
client: &reqwest::Client,
url: &str,
max_retries: u32
) -> Result<String, reqwest::Error> {
let mut attempts = 0;
loop {
match client.get(url).send().await {
Ok(response) if response.status().is_success() => {
return response.text().await;
}
Ok(response) => {
eprintln!("HTTP error {}: {}", response.status(), url);
}
Err(e) if attempts < max_retries => {
eprintln!("Attempt {} failed for {}: {}", attempts + 1, url, e);
}
Err(e) => return Err(e),
}
attempts += 1;
if attempts > max_retries {
return Err(reqwest::Error::from(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"Max retries exceeded"
)));
}
// Exponential backoff
let delay = Duration::from_millis(100 * 2_u64.pow(attempts));
tokio::time::sleep(delay).await;
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let urls = vec![
"https://httpbin.org/status/200",
"https://httpbin.org/status/500", // Will trigger retries
"https://httpbin.org/status/200",
];
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(30))
.build()?;
let futures: Vec<_> = urls.into_iter()
.map(|url| fetch_with_retry(&client, url, 3))
.collect();
let results = futures::future::join_all(futures).await;
for (i, result) in results.into_iter().enumerate() {
match result {
Ok(content) => println!("Success {}: {} bytes", i, content.len()),
Err(e) => eprintln!("Failed {}: {}", i, e),
}
}
Ok(())
}
Performance Tips
- Reuse Client: Always reuse the same
reqwest::Clientinstance to leverage connection pooling - Tune Connection Pool: Configure the client for your use case:
let client = reqwest::Client::builder()
.pool_max_idle_per_host(10)
.pool_idle_timeout(Duration::from_secs(30))
.timeout(Duration::from_secs(30))
.build()?;
- Use Semaphores: Limit concurrent requests to avoid overwhelming servers
- Implement Backoff: Use exponential backoff for retries
- Handle Timeouts: Set appropriate timeouts for different request types
When to Use Each Pattern
- join_all: Simple cases where you want all requests to complete
- tokio::spawn: When you need true parallelism and independent task management
- Semaphores: When you need to limit concurrency to respect rate limits
- Streams: For processing large numbers of URLs with backpressure control
Concurrent requests with Reqwest significantly improve performance for I/O-bound operations. Choose the appropriate pattern based on your specific requirements for error handling, concurrency limits, and resource management.
