What are the best practices for managing memory usage in Headless Chromium?

Memory management is critical when running headless Chromium, especially at scale. Poor memory management can lead to crashes, degraded performance, and system instability. Here are proven strategies to optimize memory usage effectively.

Essential Chrome Flags for Memory Optimization

Core Memory-Saving Flags

The most impactful flags for reducing memory consumption:

# Essential memory optimization flags
chromium-browser --headless \
  --no-sandbox \
  --disable-gpu \
  --disable-dev-shm-usage \
  --disable-extensions \
  --disable-plugins \
  --disable-background-networking \
  --disable-background-timer-throttling \
  --disable-renderer-backgrounding \
  --disable-backgrounding-occluded-windows \
  --memory-pressure-off

Advanced Memory Flags

For extreme memory optimization (use with caution):

# Advanced memory optimization (may affect stability)
chromium-browser --headless \
  --single-process \
  --no-zygote \
  --max_old_space_size=512 \
  --max-heap-size=512 \
  --aggressive-cache-discard \
  --disk-cache-size=1

⚠️ Warning: --single-process can cause instability. Test thoroughly before using in production.

Resource Limiting Strategies

1. Set Memory Limits at OS Level

# Using ulimit (per process)
ulimit -v 1048576  # Limit to 1GB virtual memory

# Using systemd service
[Service]
MemoryMax=512M
MemoryHigh=400M

2. Configure cgroups for Containers

# Docker memory limits
docker run --memory=512m --memory-swap=512m your-app

# Kubernetes resource limits
resources:
  limits:
    memory: "512Mi"
  requests:
    memory: "256Mi"

Browser Pool Management

Connection Pooling Pattern

class BrowserPool {
  constructor(maxBrowsers = 5) {
    this.maxBrowsers = maxBrowsers;
    this.browsers = [];
    this.usageCount = new Map();
  }

  async getBrowser() {
    // Reuse existing browser if available
    for (let browser of this.browsers) {
      if (!browser.busy) {
        browser.busy = true;
        return browser;
      }
    }

    // Create new browser if under limit
    if (this.browsers.length < this.maxBrowsers) {
      const browser = await this.createBrowser();
      browser.busy = true;
      this.browsers.push(browser);
      return browser;
    }

    // Wait for available browser
    return new Promise(resolve => {
      const check = () => {
        const available = this.browsers.find(b => !b.busy);
        if (available) {
          available.busy = true;
          resolve(available);
        } else {
          setTimeout(check, 100);
        }
      };
      check();
    });
  }

  async createBrowser() {
    return await puppeteer.launch({
      headless: true,
      args: [
        '--no-sandbox',
        '--disable-gpu',
        '--disable-dev-shm-usage',
        '--disable-extensions',
        '--disable-background-networking',
        '--memory-pressure-off'
      ]
    });
  }

  releaseBrowser(browser) {
    browser.busy = false;

    // Restart browser after N uses to prevent memory leaks
    const usage = this.usageCount.get(browser) || 0;
    if (usage > 50) {
      this.restartBrowser(browser);
    } else {
      this.usageCount.set(browser, usage + 1);
    }
  }

  async restartBrowser(oldBrowser) {
    await oldBrowser.close();
    const index = this.browsers.indexOf(oldBrowser);
    if (index !== -1) {
      this.browsers[index] = await this.createBrowser();
      this.usageCount.delete(oldBrowser);
    }
  }
}

Page-Level Optimization

Memory-Efficient Page Configuration

const puppeteer = require('puppeteer');

async function createOptimizedPage(browser) {
  const page = await browser.newPage();

  // Disable unnecessary features
  await page.setRequestInterception(true);
  page.on('request', (request) => {
    const resourceType = request.resourceType();

    // Block resource-heavy content
    if (['image', 'stylesheet', 'font', 'media'].includes(resourceType)) {
      request.abort();
    } else {
      request.continue();
    }
  });

  // Set viewport to minimize memory
  await page.setViewport({ width: 1280, height: 720 });

  // Disable JavaScript if not needed
  await page.setJavaScriptEnabled(false);

  return page;
}

Cleanup Best Practices

async function scrapePage(url) {
  const browser = await browserPool.getBrowser();
  let page;

  try {
    page = await createOptimizedPage(browser);
    await page.goto(url, { waitUntil: 'domcontentloaded' });

    // Perform scraping
    const data = await page.evaluate(() => {
      // Extract data and return plain objects only
      return {
        title: document.title,
        links: Array.from(document.links).map(a => a.href)
      };
    });

    return data;
  } finally {
    // Always cleanup
    if (page) {
      await page.close();
    }
    browserPool.releaseBrowser(browser);
  }
}

Memory Monitoring and Alerting

Process Memory Monitoring

import psutil
import asyncio
from pyppeteer import launch

class MemoryMonitor:
    def __init__(self, max_memory_mb=512):
        self.max_memory_mb = max_memory_mb
        self.browser = None

    async def get_memory_usage(self):
        if not self.browser:
            return 0

        # Get browser process memory
        browser_process = psutil.Process(self.browser.process.pid)
        memory_mb = browser_process.memory_info().rss / 1024 / 1024

        # Include child processes
        for child in browser_process.children(recursive=True):
            try:
                memory_mb += child.memory_info().rss / 1024 / 1024
            except psutil.NoSuchProcess:
                pass

        return memory_mb

    async def check_memory_limit(self):
        memory_usage = await self.get_memory_usage()
        if memory_usage > self.max_memory_mb:
            print(f"Memory limit exceeded: {memory_usage}MB > {self.max_memory_mb}MB")
            await self.restart_browser()

    async def restart_browser(self):
        if self.browser:
            await self.browser.close()

        self.browser = await launch(
            headless=True,
            args=[
                '--no-sandbox',
                '--disable-gpu',
                '--disable-dev-shm-usage',
                '--memory-pressure-off'
            ]
        )

Docker Optimization

Dockerfile for Memory-Optimized Chromium

FROM node:16-alpine

# Install Chromium
RUN apk add --no-cache \
    chromium \
    nss \
    freetype \
    freetype-dev \
    harfbuzz \
    ca-certificates \
    ttf-freefont

# Configure Chrome flags
ENV CHROME_BIN=/usr/bin/chromium-browser
ENV CHROME_PATH=/usr/bin/chromium-browser
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser

# Memory optimization
ENV NODE_OPTIONS="--max-old-space-size=512"

# Set user for security
RUN addgroup -g 1001 -S nodejs
RUN adduser -S nextjs -u 1001
USER nextjs

WORKDIR /app
COPY package*.json ./
RUN npm ci --only=production

COPY . .
CMD ["node", "index.js"]

Selenium WebDriver Memory Management

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import psutil
import time

class MemoryEfficientDriver:
    def __init__(self, max_memory_mb=512):
        self.max_memory_mb = max_memory_mb
        self.driver = None
        self.start_time = time.time()

    def create_driver(self):
        options = Options()
        options.add_argument('--headless')
        options.add_argument('--no-sandbox')
        options.add_argument('--disable-gpu')
        options.add_argument('--disable-dev-shm-usage')
        options.add_argument('--disable-extensions')
        options.add_argument('--disable-background-networking')
        options.add_argument('--memory-pressure-off')
        options.add_argument('--max_old_space_size=512')

        # Disable images and CSS for memory savings
        prefs = {
            "profile.managed_default_content_settings.images": 2,
            "profile.default_content_setting_values.notifications": 2,
        }
        options.add_experimental_option("prefs", prefs)

        self.driver = webdriver.Chrome(options=options)
        self.start_time = time.time()

    def get_memory_usage(self):
        if not self.driver:
            return 0

        pid = self.driver.service.process.pid
        process = psutil.Process(pid)
        memory_mb = process.memory_info().rss / 1024 / 1024

        # Include child processes
        for child in process.children(recursive=True):
            try:
                memory_mb += child.memory_info().rss / 1024 / 1024
            except psutil.NoSuchProcess:
                pass

        return memory_mb

    def should_restart(self):
        # Restart if memory limit exceeded or running too long
        memory_usage = self.get_memory_usage()
        running_time = time.time() - self.start_time

        return (memory_usage > self.max_memory_mb or 
                running_time > 3600)  # 1 hour max

    def restart_if_needed(self):
        if self.should_restart():
            if self.driver:
                self.driver.quit()
            self.create_driver()

Memory Leak Prevention

Common Memory Leak Sources

1. Uncleared event listeners: Always remove listeners before closing pages
2. Retained DOM references: Avoid storing DOM elements in variables
3. Circular references: Use WeakMap/WeakSet when appropriate
4. Unclosed pages/browsers: Always use try/finally blocks

Memory-Safe Scraping Pattern

async function memoryEfficientScraping(urls) {
  const results = [];
  const batchSize = 5; // Process in small batches

  for (let i = 0; i < urls.length; i += batchSize) {
    const batch = urls.slice(i, i + batchSize);
    const browser = await puppeteer.launch({
      headless: true,
      args: ['--no-sandbox', '--disable-gpu', '--memory-pressure-off']
    });

    try {
      const batchResults = await Promise.all(
        batch.map(async (url) => {
          const page = await browser.newPage();
          try {
            await page.goto(url, { waitUntil: 'domcontentloaded' });
            const data = await page.evaluate(() => document.title);
            return { url, title: data };
          } finally {
            await page.close(); // Always close pages
          }
        })
      );

      results.push(...batchResults);
    } finally {
      await browser.close(); // Always close browser
    }

    // Allow garbage collection between batches
    if (global.gc) global.gc();
    await new Promise(resolve => setTimeout(resolve, 1000));
  }

  return results;
}

Production Recommendations

1. Monitor continuously: Set up alerts for memory usage above 80%
2. Implement circuit breakers: Stop processing if memory exceeds limits
3. Use horizontal scaling: Multiple small instances instead of one large
4. Regular restarts: Scheduled browser restarts every 1-2 hours
5. Resource quotas: Always set memory limits in production environments

By implementing these memory management strategies, you can run headless Chromium reliably at scale while maintaining optimal performance and system stability.