Memory management is critical when running headless Chromium, especially at scale. Poor memory management can lead to crashes, degraded performance, and system instability. Here are proven strategies to optimize memory usage effectively.
Essential Chrome Flags for Memory Optimization
Core Memory-Saving Flags
The most impactful flags for reducing memory consumption:
# Essential memory optimization flags
chromium-browser --headless \
--no-sandbox \
--disable-gpu \
--disable-dev-shm-usage \
--disable-extensions \
--disable-plugins \
--disable-background-networking \
--disable-background-timer-throttling \
--disable-renderer-backgrounding \
--disable-backgrounding-occluded-windows \
--memory-pressure-off
Advanced Memory Flags
For extreme memory optimization (use with caution):
# Advanced memory optimization (may affect stability)
chromium-browser --headless \
--single-process \
--no-zygote \
--max_old_space_size=512 \
--max-heap-size=512 \
--aggressive-cache-discard \
--disk-cache-size=1
⚠️ Warning: --single-process can cause instability. Test thoroughly before using in production.
Resource Limiting Strategies
1. Set Memory Limits at OS Level
# Using ulimit (per process)
ulimit -v 1048576 # Limit to 1GB virtual memory
# Using systemd service
[Service]
MemoryMax=512M
MemoryHigh=400M
2. Configure cgroups for Containers
# Docker memory limits
docker run --memory=512m --memory-swap=512m your-app
# Kubernetes resource limits
resources:
limits:
memory: "512Mi"
requests:
memory: "256Mi"
Browser Pool Management
Connection Pooling Pattern
class BrowserPool {
constructor(maxBrowsers = 5) {
this.maxBrowsers = maxBrowsers;
this.browsers = [];
this.usageCount = new Map();
}
async getBrowser() {
// Reuse existing browser if available
for (let browser of this.browsers) {
if (!browser.busy) {
browser.busy = true;
return browser;
}
}
// Create new browser if under limit
if (this.browsers.length < this.maxBrowsers) {
const browser = await this.createBrowser();
browser.busy = true;
this.browsers.push(browser);
return browser;
}
// Wait for available browser
return new Promise(resolve => {
const check = () => {
const available = this.browsers.find(b => !b.busy);
if (available) {
available.busy = true;
resolve(available);
} else {
setTimeout(check, 100);
}
};
check();
});
}
async createBrowser() {
return await puppeteer.launch({
headless: true,
args: [
'--no-sandbox',
'--disable-gpu',
'--disable-dev-shm-usage',
'--disable-extensions',
'--disable-background-networking',
'--memory-pressure-off'
]
});
}
releaseBrowser(browser) {
browser.busy = false;
// Restart browser after N uses to prevent memory leaks
const usage = this.usageCount.get(browser) || 0;
if (usage > 50) {
this.restartBrowser(browser);
} else {
this.usageCount.set(browser, usage + 1);
}
}
async restartBrowser(oldBrowser) {
await oldBrowser.close();
const index = this.browsers.indexOf(oldBrowser);
if (index !== -1) {
this.browsers[index] = await this.createBrowser();
this.usageCount.delete(oldBrowser);
}
}
}
Page-Level Optimization
Memory-Efficient Page Configuration
const puppeteer = require('puppeteer');
async function createOptimizedPage(browser) {
const page = await browser.newPage();
// Disable unnecessary features
await page.setRequestInterception(true);
page.on('request', (request) => {
const resourceType = request.resourceType();
// Block resource-heavy content
if (['image', 'stylesheet', 'font', 'media'].includes(resourceType)) {
request.abort();
} else {
request.continue();
}
});
// Set viewport to minimize memory
await page.setViewport({ width: 1280, height: 720 });
// Disable JavaScript if not needed
await page.setJavaScriptEnabled(false);
return page;
}
Cleanup Best Practices
async function scrapePage(url) {
const browser = await browserPool.getBrowser();
let page;
try {
page = await createOptimizedPage(browser);
await page.goto(url, { waitUntil: 'domcontentloaded' });
// Perform scraping
const data = await page.evaluate(() => {
// Extract data and return plain objects only
return {
title: document.title,
links: Array.from(document.links).map(a => a.href)
};
});
return data;
} finally {
// Always cleanup
if (page) {
await page.close();
}
browserPool.releaseBrowser(browser);
}
}
Memory Monitoring and Alerting
Process Memory Monitoring
import psutil
import asyncio
from pyppeteer import launch
class MemoryMonitor:
def __init__(self, max_memory_mb=512):
self.max_memory_mb = max_memory_mb
self.browser = None
async def get_memory_usage(self):
if not self.browser:
return 0
# Get browser process memory
browser_process = psutil.Process(self.browser.process.pid)
memory_mb = browser_process.memory_info().rss / 1024 / 1024
# Include child processes
for child in browser_process.children(recursive=True):
try:
memory_mb += child.memory_info().rss / 1024 / 1024
except psutil.NoSuchProcess:
pass
return memory_mb
async def check_memory_limit(self):
memory_usage = await self.get_memory_usage()
if memory_usage > self.max_memory_mb:
print(f"Memory limit exceeded: {memory_usage}MB > {self.max_memory_mb}MB")
await self.restart_browser()
async def restart_browser(self):
if self.browser:
await self.browser.close()
self.browser = await launch(
headless=True,
args=[
'--no-sandbox',
'--disable-gpu',
'--disable-dev-shm-usage',
'--memory-pressure-off'
]
)
Docker Optimization
Dockerfile for Memory-Optimized Chromium
FROM node:16-alpine
# Install Chromium
RUN apk add --no-cache \
chromium \
nss \
freetype \
freetype-dev \
harfbuzz \
ca-certificates \
ttf-freefont
# Configure Chrome flags
ENV CHROME_BIN=/usr/bin/chromium-browser
ENV CHROME_PATH=/usr/bin/chromium-browser
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# Memory optimization
ENV NODE_OPTIONS="--max-old-space-size=512"
# Set user for security
RUN addgroup -g 1001 -S nodejs
RUN adduser -S nextjs -u 1001
USER nextjs
WORKDIR /app
COPY package*.json ./
RUN npm ci --only=production
COPY . .
CMD ["node", "index.js"]
Selenium WebDriver Memory Management
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import psutil
import time
class MemoryEfficientDriver:
def __init__(self, max_memory_mb=512):
self.max_memory_mb = max_memory_mb
self.driver = None
self.start_time = time.time()
def create_driver(self):
options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-extensions')
options.add_argument('--disable-background-networking')
options.add_argument('--memory-pressure-off')
options.add_argument('--max_old_space_size=512')
# Disable images and CSS for memory savings
prefs = {
"profile.managed_default_content_settings.images": 2,
"profile.default_content_setting_values.notifications": 2,
}
options.add_experimental_option("prefs", prefs)
self.driver = webdriver.Chrome(options=options)
self.start_time = time.time()
def get_memory_usage(self):
if not self.driver:
return 0
pid = self.driver.service.process.pid
process = psutil.Process(pid)
memory_mb = process.memory_info().rss / 1024 / 1024
# Include child processes
for child in process.children(recursive=True):
try:
memory_mb += child.memory_info().rss / 1024 / 1024
except psutil.NoSuchProcess:
pass
return memory_mb
def should_restart(self):
# Restart if memory limit exceeded or running too long
memory_usage = self.get_memory_usage()
running_time = time.time() - self.start_time
return (memory_usage > self.max_memory_mb or
running_time > 3600) # 1 hour max
def restart_if_needed(self):
if self.should_restart():
if self.driver:
self.driver.quit()
self.create_driver()
Memory Leak Prevention
Common Memory Leak Sources
- Uncleared event listeners: Always remove listeners before closing pages
- Retained DOM references: Avoid storing DOM elements in variables
- Circular references: Use WeakMap/WeakSet when appropriate
- Unclosed pages/browsers: Always use try/finally blocks
Memory-Safe Scraping Pattern
async function memoryEfficientScraping(urls) {
const results = [];
const batchSize = 5; // Process in small batches
for (let i = 0; i < urls.length; i += batchSize) {
const batch = urls.slice(i, i + batchSize);
const browser = await puppeteer.launch({
headless: true,
args: ['--no-sandbox', '--disable-gpu', '--memory-pressure-off']
});
try {
const batchResults = await Promise.all(
batch.map(async (url) => {
const page = await browser.newPage();
try {
await page.goto(url, { waitUntil: 'domcontentloaded' });
const data = await page.evaluate(() => document.title);
return { url, title: data };
} finally {
await page.close(); // Always close pages
}
})
);
results.push(...batchResults);
} finally {
await browser.close(); // Always close browser
}
// Allow garbage collection between batches
if (global.gc) global.gc();
await new Promise(resolve => setTimeout(resolve, 1000));
}
return results;
}
Production Recommendations
- Monitor continuously: Set up alerts for memory usage above 80%
- Implement circuit breakers: Stop processing if memory exceeds limits
- Use horizontal scaling: Multiple small instances instead of one large
- Regular restarts: Scheduled browser restarts every 1-2 hours
- Resource quotas: Always set memory limits in production environments
By implementing these memory management strategies, you can run headless Chromium reliably at scale while maintaining optimal performance and system stability.
