How to monitor network requests in Puppeteer?

Network monitoring in Puppeteer is essential for debugging web applications, analyzing API behavior, and optimizing performance. Puppeteer provides comprehensive tools to intercept and analyze all network traffic.

Basic Network Request Monitoring

Monitor all network requests using the request event:

const puppeteer = require('puppeteer');

async function monitorNetwork() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    // Listen for all network requests
    page.on('request', request => {
        console.log(`${request.method()} ${request.url()}`);
    });

    await page.goto('https://example.com');
    await browser.close();
}

monitorNetwork();

Comprehensive Request and Response Monitoring

Track both requests and responses with detailed information:

const puppeteer = require('puppeteer');

async function comprehensiveMonitoring() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    // Monitor requests
    page.on('request', request => {
        console.log('REQUEST:', {
            url: request.url(),
            method: request.method(),
            headers: request.headers(),
            resourceType: request.resourceType()
        });
    });

    // Monitor responses
    page.on('response', async response => {
        console.log('RESPONSE:', {
            url: response.url(),
            status: response.status(),
            statusText: response.statusText(),
            headers: response.headers(),
            fromCache: response.fromCache(),
            size: response.headers()['content-length'] || 'unknown'
        });
    });

    await page.goto('https://example.com');
    await browser.close();
}

comprehensiveMonitoring();

Filtering Network Traffic

Monitor specific types of requests using resource type filtering:

async function filterRequests() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    page.on('request', request => {
        const resourceType = request.resourceType();

        // Filter for API calls (XHR and Fetch)
        if (resourceType === 'xhr' || resourceType === 'fetch') {
            console.log('API Request:', request.url());
        }

        // Filter for images
        if (resourceType === 'image') {
            console.log('Image loaded:', request.url());
        }

        // Filter for stylesheets
        if (resourceType === 'stylesheet') {
            console.log('CSS loaded:', request.url());
        }
    });

    await page.goto('https://example.com');
    await browser.close();
}

Analyzing Request and Response Bodies

Capture and analyze request/response payloads:

async function analyzePayloads() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    page.on('request', request => {
        // Log POST data
        if (request.method() === 'POST') {
            console.log('POST Data:', request.postData());
        }
    });

    page.on('response', async response => {
        // Only analyze JSON responses
        const contentType = response.headers()['content-type'];
        if (contentType && contentType.includes('application/json')) {
            try {
                const responseBody = await response.text();
                console.log('JSON Response:', JSON.parse(responseBody));
            } catch (error) {
                console.log('Failed to parse JSON response');
            }
        }
    });

    await page.goto('https://example.com');
    await browser.close();
}

Failed Request Monitoring

Track failed requests and network errors:

async function monitorFailures() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    page.on('requestfailed', request => {
        console.log('FAILED REQUEST:', {
            url: request.url(),
            method: request.method(),
            failure: request.failure()
        });
    });

    page.on('response', response => {
        // Log HTTP errors
        if (response.status() >= 400) {
            console.log('HTTP ERROR:', {
                url: response.url(),
                status: response.status(),
                statusText: response.statusText()
            });
        }
    });

    await page.goto('https://example.com');
    await browser.close();
}

Performance Monitoring

Track request timing and performance metrics:

async function monitorPerformance() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    const requestTimes = new Map();

    page.on('request', request => {
        requestTimes.set(request.url(), Date.now());
    });

    page.on('response', response => {
        const startTime = requestTimes.get(response.url());
        if (startTime) {
            const duration = Date.now() - startTime;
            console.log(`${response.url()} - ${duration}ms`);
            requestTimes.delete(response.url());
        }
    });

    await page.goto('https://example.com');
    await browser.close();
}

Practical Use Cases

API Monitoring for Web Scraping

async function monitorApiCalls() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    const apiCalls = [];

    page.on('response', async response => {
        // Capture API responses
        if (response.url().includes('/api/') || 
            response.headers()['content-type']?.includes('application/json')) {

            try {
                const data = await response.json();
                apiCalls.push({
                    url: response.url(),
                    status: response.status(),
                    data: data
                });
            } catch (error) {
                // Not JSON response
            }
        }
    });

    await page.goto('https://example.com');

    // Process captured API data
    console.log('Captured API calls:', apiCalls);

    await browser.close();
}

Important Notes

• Event Timing: The request event fires when a request is initiated, before data is sent
• Response Bodies: Use response.text(), response.json(), or response.buffer() to read response data
• Memory Management: Be careful with large responses to avoid memory issues
• Async Handling: Always use await when reading response bodies
• Resource Types: Available types include document, stylesheet, image, media, font, script, texttrack, xhr, fetch, eventsource, websocket, manifest, other

Network monitoring in Puppeteer is powerful for debugging, performance analysis, and data extraction. Use these patterns to gain insights into web application behavior and optimize your scraping strategies.