Session and state management is crucial when working with APIs that require authentication or when scraping websites that track user sessions. Reqwest provides powerful tools for maintaining HTTP state across multiple requests through cookie jars, custom headers, and persistent client configurations.
Cookie-Based Session Management
Basic Cookie Jar Setup
Reqwest's cookie::Jar automatically handles cookie storage and transmission across requests:
use reqwest::cookie::Jar;
use reqwest::{Client, Url};
use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
// Create a shared cookie jar
let jar = Arc::new(Jar::default());
// Build client with cookie support
let client = Client::builder()
.cookie_provider(Arc::clone(&jar))
.build()?;
// Login request - cookies will be automatically stored
let login_response = client
.post("https://example.com/login")
.form(&[("username", "user"), ("password", "pass")])
.send()
.await?;
// Subsequent requests automatically include session cookies
let protected_data = client
.get("https://example.com/protected")
.send()
.await?;
println!("Protected page: {}", protected_data.text().await?);
Ok(())
}
Manual Cookie Management
For more control, you can manually add and inspect cookies:
use reqwest::cookie::Jar;
use reqwest::{Client, Url};
use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
let jar = Arc::new(Jar::default());
let url = Url::parse("https://api.example.com")?;
// Manually add authentication cookie
jar.add_cookie_str("auth_token=abc123; Path=/; HttpOnly", &url);
jar.add_cookie_str("session_id=xyz789; Path=/; Secure", &url);
let client = Client::builder()
.cookie_provider(Arc::clone(&jar))
.build()?;
// Make authenticated request
let response = client.get(&url).send().await?;
// Inspect cookies for debugging
if let Some(cookies) = jar.cookies(&url) {
println!("Current cookies: {}", cookies.to_str().unwrap());
}
Ok(())
}
Header-Based State Management
Persistent Headers with Client Builder
Configure default headers that persist across all requests:
use reqwest::{Client, header::{HeaderMap, HeaderValue, AUTHORIZATION, USER_AGENT}};
#[tokio::main]
async fn main() -> Result<(), reqwest::Error> {
let mut headers = HeaderMap::new();
headers.insert(AUTHORIZATION, HeaderValue::from_static("Bearer token123"));
headers.insert(USER_AGENT, HeaderValue::from_static("MyApp/1.0"));
headers.insert("X-API-Version", HeaderValue::from_static("v2"));
let client = Client::builder()
.default_headers(headers)
.build()?;
// All requests automatically include default headers
let response = client.get("https://api.example.com/data").send().await?;
Ok(())
}
Dynamic Header Management
Update headers based on authentication flow or API responses:
use reqwest::{Client, header::{HeaderMap, HeaderValue, AUTHORIZATION}};
use serde_json::Value;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let client = Client::new();
// Step 1: Authenticate and get token
let auth_response: Value = client
.post("https://api.example.com/auth")
.json(&serde_json::json!({
"username": "user",
"password": "pass"
}))
.send()
.await?
.json()
.await?;
let token = auth_response["access_token"].as_str().unwrap();
// Step 2: Use token for subsequent requests
let mut headers = HeaderMap::new();
headers.insert(
AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {}", token))?
);
let protected_response = client
.get("https://api.example.com/protected")
.headers(headers)
.send()
.await?;
Ok(())
}
Session Persistence Across Application Runs
JSON-Based Cookie Storage
Save and restore cookie state using JSON serialization:
use reqwest::cookie::Jar;
use reqwest::{Client, Url};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use std::fs;
#[derive(Serialize, Deserialize)]
struct CookieData {
name: String,
value: String,
domain: String,
path: String,
}
async fn save_session(jar: &Jar, base_url: &Url) -> Result<(), Box<dyn std::error::Error>> {
let cookies = jar.cookies(base_url).unwrap();
let cookie_str = cookies.to_str().unwrap();
// Parse and structure cookie data for JSON storage
let cookie_data: Vec<CookieData> = cookie_str
.split(';')
.map(|cookie| {
let parts: Vec<&str> = cookie.trim().splitn(2, '=').collect();
CookieData {
name: parts[0].to_string(),
value: parts.get(1).unwrap_or(&"").to_string(),
domain: base_url.host_str().unwrap_or("").to_string(),
path: "/".to_string(),
}
})
.collect();
let json_data = serde_json::to_string_pretty(&cookie_data)?;
fs::write("session.json", json_data)?;
Ok(())
}
async fn load_session() -> Result<Arc<Jar>, Box<dyn std::error::Error>> {
let jar = Arc::new(Jar::default());
if let Ok(json_data) = fs::read_to_string("session.json") {
let cookie_data: Vec<CookieData> = serde_json::from_str(&json_data)?;
let url = Url::parse("https://example.com")?;
for cookie in cookie_data {
let cookie_str = format!("{}={}", cookie.name, cookie.value);
jar.add_cookie_str(&cookie_str, &url);
}
}
Ok(jar)
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Load existing session or create new one
let jar = load_session().await?;
let client = Client::builder()
.cookie_provider(Arc::clone(&jar))
.build()?;
// Use client for requests...
let response = client.get("https://example.com/api").send().await?;
// Save session before exit
let url = Url::parse("https://example.com")?;
save_session(&jar, &url).await?;
Ok(())
}
Advanced Session Management Patterns
Session Wrapper with Automatic Renewal
Create a wrapper that handles token refresh automatically:
use reqwest::{Client, Response};
use serde_json::Value;
use std::time::{Duration, Instant};
pub struct SessionManager {
client: Client,
access_token: Option<String>,
refresh_token: Option<String>,
token_expires: Option<Instant>,
base_url: String,
}
impl SessionManager {
pub fn new(base_url: String) -> Self {
Self {
client: Client::new(),
access_token: None,
refresh_token: None,
token_expires: None,
base_url,
}
}
pub async fn login(&mut self, username: &str, password: &str) -> Result<(), Box<dyn std::error::Error>> {
let response: Value = self.client
.post(&format!("{}/auth/login", self.base_url))
.json(&serde_json::json!({
"username": username,
"password": password
}))
.send()
.await?
.json()
.await?;
self.access_token = Some(response["access_token"].as_str().unwrap().to_string());
self.refresh_token = Some(response["refresh_token"].as_str().unwrap().to_string());
// Set token expiration (assuming expires_in is in seconds)
if let Some(expires_in) = response["expires_in"].as_u64() {
self.token_expires = Some(Instant::now() + Duration::from_secs(expires_in));
}
Ok(())
}
async fn ensure_valid_token(&mut self) -> Result<(), Box<dyn std::error::Error>> {
if let Some(expires) = self.token_expires {
if Instant::now() >= expires {
self.refresh_access_token().await?;
}
}
Ok(())
}
async fn refresh_access_token(&mut self) -> Result<(), Box<dyn std::error::Error>> {
if let Some(refresh_token) = &self.refresh_token {
let response: Value = self.client
.post(&format!("{}/auth/refresh", self.base_url))
.json(&serde_json::json!({
"refresh_token": refresh_token
}))
.send()
.await?
.json()
.await?;
self.access_token = Some(response["access_token"].as_str().unwrap().to_string());
if let Some(expires_in) = response["expires_in"].as_u64() {
self.token_expires = Some(Instant::now() + Duration::from_secs(expires_in));
}
}
Ok(())
}
pub async fn get(&mut self, endpoint: &str) -> Result<Response, Box<dyn std::error::Error>> {
self.ensure_valid_token().await?;
let mut request = self.client.get(&format!("{}{}", self.base_url, endpoint));
if let Some(token) = &self.access_token {
request = request.bearer_auth(token);
}
Ok(request.send().await?)
}
}
Best Practices
Error Handling and Retry Logic
use reqwest::{Client, StatusCode};
use std::time::Duration;
use tokio::time::sleep;
async fn make_request_with_retry(
client: &Client,
url: &str,
max_retries: u32,
) -> Result<reqwest::Response, reqwest::Error> {
let mut attempts = 0;
loop {
match client.get(url).send().await {
Ok(response) => {
match response.status() {
StatusCode::UNAUTHORIZED => {
// Handle session expiry
if attempts < max_retries {
println!("Session expired, retrying...");
sleep(Duration::from_secs(2)).await;
attempts += 1;
continue;
}
}
StatusCode::TOO_MANY_REQUESTS => {
// Handle rate limiting
if attempts < max_retries {
sleep(Duration::from_secs(10)).await;
attempts += 1;
continue;
}
}
_ => return Ok(response),
}
}
Err(e) => {
if attempts < max_retries {
sleep(Duration::from_secs(1)).await;
attempts += 1;
continue;
} else {
return Err(e);
}
}
}
break;
}
Err(reqwest::Error::from(std::io::Error::new(
std::io::ErrorKind::Other,
"Max retries exceeded"
)))
}
Security Considerations
- Always use HTTPS for sensitive session data
- Store tokens securely and avoid logging them
- Implement proper token expiration and refresh logic
- Respect rate limits and implement backoff strategies
- Use secure cookie attributes (
HttpOnly,Secure,SameSite)
Session management in Reqwest provides the foundation for building robust web scrapers and API clients that can maintain state across complex interaction flows.
