Web Scraping with JavaScript

node -v    # Should show Node.js version (e.g., v18.17.0)
npm -v     # Should show npm version (e.g., 9.6.7)

mkdir web-scraper-project
cd web-scraper-project
npm init -y

npm install puppeteer

const puppeteer = require('puppeteer');

async function scrapeWebsite() {
    // Launch browser
    const browser = await puppeteer.launch({
        headless: true, // Set to false to see browser window
        defaultViewport: { width: 1280, height: 720 }
    });

    try {
        const page = await browser.newPage();

        // Navigate to website
        await page.goto('https://example.com', { 
            waitUntil: 'networkidle2' 
        });

        // Extract page title
        const title = await page.title();
        console.log('Page title:', title);

        // Extract all links
        const links = await page.evaluate(() => {
            return Array.from(document.querySelectorAll('a[href]')).map(link => ({
                text: link.textContent.trim(),
                url: link.href
            }));
        });

        console.log('Found links:', links);

    } catch (error) {
        console.error('Error during scraping:', error);
    } finally {
        await browser.close();
    }
}

scrapeWebsite();

const puppeteer = require('puppeteer');

async function scrapeProducts(searchTerm) {
    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();

    try {
        // Set user agent to avoid detection
        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');

        await page.goto('https://example-store.com');

        // Wait for search box and enter search term
        await page.waitForSelector('#search-input');
        await page.type('#search-input', searchTerm);
        await page.click('#search-button');

        // Wait for results to load
        await page.waitForSelector('.product-item');

        // Extract product data
        const products = await page.evaluate(() => {
            return Array.from(document.querySelectorAll('.product-item')).map(item => ({
                name: item.querySelector('.product-name')?.textContent.trim(),
                price: item.querySelector('.product-price')?.textContent.trim(),
                image: item.querySelector('.product-image')?.src,
                rating: item.querySelector('.product-rating')?.textContent.trim()
            }));
        });

        console.log(`Found ${products.length} products:`, products);
        return products;

    } finally {
        await browser.close();
    }
}

scrapeProducts('laptop');

npm install selenium-webdriver

# For Chrome
npm install chromedriver

# For Firefox  
npm install geckodriver

const { Builder, By, until } = require('selenium-webdriver');
const chrome = require('selenium-webdriver/chrome');

async function automateLogin() {
    // Set Chrome options
    const options = new chrome.Options();
    options.addArguments('--headless'); // Run in background
    options.addArguments('--no-sandbox');
    options.addArguments('--disable-dev-shm-usage');

    const driver = await new Builder()
        .forBrowser('chrome')
        .setChromeOptions(options)
        .build();

    try {
        // Navigate to login page
        await driver.get('https://example.com/login');

        // Wait for and fill login form
        await driver.wait(until.elementLocated(By.id('username')), 10000);
        await driver.findElement(By.id('username')).sendKeys('your-username');
        await driver.findElement(By.id('password')).sendKeys('your-password');

        // Submit form
        await driver.findElement(By.css('button[type="submit"]')).click();

        // Wait for dashboard to load
        await driver.wait(until.titleContains('Dashboard'), 10000);

        // Extract user data
        const userInfo = await driver.findElement(By.css('.user-info')).getText();
        console.log('User info:', userInfo);

    } catch (error) {
        console.error('Automation failed:', error);
    } finally {
        await driver.quit();
    }
}

automateLogin();

const { Builder, By, until, Key } = require('selenium-webdriver');

async function scrapeInfiniteScroll() {
    const driver = await new Builder().forBrowser('chrome').build();

    try {
        await driver.get('https://example.com/infinite-scroll');

        let itemCount = 0;
        let previousCount = -1;
        const allItems = [];

        // Keep scrolling until no new content loads
        while (itemCount !== previousCount) {
            previousCount = itemCount;

            // Scroll to bottom
            await driver.executeScript('window.scrollTo(0, document.body.scrollHeight)');

            // Wait for new content to load
            await driver.sleep(2000);

            // Count current items
            const items = await driver.findElements(By.css('.content-item'));
            itemCount = items.length;

            console.log(`Loaded ${itemCount} items...`);
        }

        // Extract all item data
        const items = await driver.findElements(By.css('.content-item'));
        for (let item of items) {
            const title = await item.findElement(By.css('.item-title')).getText();
            const description = await item.findElement(By.css('.item-desc')).getText();
            allItems.push({ title, description });
        }

        console.log(`Scraped ${allItems.length} total items`);
        return allItems;

    } finally {
        await driver.quit();
    }
}

scrapeInfiniteScroll();

npm install nightmare

const Nightmare = require('nightmare');

const nightmare = Nightmare({ 
    show: false, // Set to true to see browser window
    width: 1280,
    height: 720 
});

nightmare
    .goto('https://example.com')
    .wait('.content')
    .evaluate(() => {
        return {
            title: document.title,
            links: Array.from(document.querySelectorAll('a')).map(a => a.href)
        };
    })
    .end()
    .then(result => {
        console.log('Scraped data:', result);
    })
    .catch(error => {
        console.error('Error:', error);
    });

npm install axios cheerio

const axios = require('axios');
const cheerio = require('cheerio');

async function scrapeNews() {
    try {
        const { data } = await axios.get('https://example-news.com', {
            headers: {
                'User-Agent': 'Mozilla/5.0 (compatible; NewsBot/1.0)'
            }
        });

        const $ = cheerio.load(data);
        const articles = [];

        $('.article-item').each((index, element) => {
            const article = {
                title: $(element).find('.article-title').text().trim(),
                summary: $(element).find('.article-summary').text().trim(),
                url: $(element).find('a').attr('href'),
                author: $(element).find('.article-author').text().trim(),
                publishDate: $(element).find('.publish-date').text().trim()
            };

            articles.push(article);
        });

        console.log(`Found ${articles.length} articles`);
        return articles;

    } catch (error) {
        console.error('Scraping failed:', error.message);
        return [];
    }
}

scrapeNews().then(articles => {
    console.log(articles);
});

const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs').promises;

class PriceMonitor {
    constructor() {
        this.products = [];
    }

    async scrapeProduct(url) {
        try {
            const { data } = await axios.get(url, {
                headers: {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                    'Accept-Language': 'en-US,en;q=0.5',
                    'Accept-Encoding': 'gzip, deflate',
                    'Connection': 'keep-alive'
                },
                timeout: 10000
            });

            const $ = cheerio.load(data);

            return {
                title: $('h1.product-title, .product-name, [data-testid="product-title"]').first().text().trim(),
                price: this.parsePrice($('.price, .product-price, [data-testid="price"]').first().text()),
                availability: $('.availability, .stock-status').text().trim(),
                rating: $('.rating-value, .star-rating').text().trim(),
                imageUrl: $('img.product-image, .main-image img').first().attr('src'),
                scrapedAt: new Date().toISOString()
            };

        } catch (error) {
            console.error(`Failed to scrape ${url}:`, error.message);
            return null;
        }
    }

    parsePrice(priceText) {
        const match = priceText.match(/[\d,]+\.?\d*/);
        return match ? parseFloat(match[0].replace(/,/g, '')) : null;
    }

    async monitorPrices(urls) {
        console.log(`Monitoring ${urls.length} products...`);

        for (const url of urls) {
            const product = await this.scrapeProduct(url);
            if (product) {
                this.products.push({ url, ...product });
                console.log(`✓ ${product.title}: $${product.price}`);
            }

            // Rate limiting - wait 1 second between requests
            await new Promise(resolve => setTimeout(resolve, 1000));
        }

        await this.saveResults();
    }

    async saveResults() {
        await fs.writeFile('price-monitor.json', JSON.stringify(this.products, null, 2));
        console.log(`Saved ${this.products.length} products to price-monitor.json`);
    }
}

// Usage
const monitor = new PriceMonitor();
const productUrls = [
    'https://example-store.com/product1',
    'https://example-store.com/product2'
];

monitor.monitorPrices(productUrls);

const axios = require('axios');
const cheerio = require('cheerio');

// Handle different response encodings
async function scrapeWithEncoding(url) {
    const { data } = await axios.get(url, {
        responseType: 'arraybuffer'
    });

    const html = data.toString('utf8');
    const $ = cheerio.load(html);
    return $;
}

// Handle relative URLs
function resolveUrl(baseUrl, relativeUrl) {
    return new URL(relativeUrl, baseUrl).href;
}

// Clean extracted text
function cleanText(text) {
    return text
        .replace(/\s+/g, ' ')  // Replace multiple whitespace with single space
        .replace(/\n/g, '')    // Remove newlines
        .trim();               // Remove leading/trailing space
}

npm install jsdom axios

const axios = require('axios');
const { JSDOM } = require('jsdom');

async function scrapeWithJSDom(url) {
    try {
        const { data } = await axios.get(url);

        // Create DOM from HTML
        const dom = new JSDOM(data, {
            runScripts: "dangerously", // Allow script execution
            resources: "usable"        // Allow external resources
        });

        const { window } = dom;
        const { document } = window;

        // Wait for scripts to execute
        await new Promise(resolve => setTimeout(resolve, 1000));

        // Extract data using DOM methods
        const title = document.title;
        const links = Array.from(document.querySelectorAll('a[href]')).map(link => ({
            text: link.textContent.trim(),
            href: link.href
        }));

        // Access window objects if needed
        const userAgent = window.navigator.userAgent;

        return { title, links, userAgent };

    } catch (error) {
        console.error('JSDom scraping failed:', error.message);
        return null;
    }
}

scrapeWithJSDom('https://example.com').then(result => {
    console.log(result);
});

const axios = require('axios');
const { JSDOM } = require('jsdom');

class JSDomScraper {
    constructor() {
        this.results = [];
    }

    async scrapeWithScripts(url) {
        const { data } = await axios.get(url, {
            headers: {
                'User-Agent': 'Mozilla/5.0 (compatible; JSDomBot/1.0)'
            }
        });

        const dom = new JSDOM(data, {
            runScripts: "dangerously",
            resources: "usable",
            pretendToBeVisual: true,
            beforeParse(window) {
                // Mock any browser APIs if needed
                window.localStorage = {
                    getItem: () => null,
                    setItem: () => {},
                    removeItem: () => {}
                };
            }
        });

        const { window } = dom;
        const { document } = window;

        // Give scripts time to execute
        await this.waitForContent(window);

        return this.extractData(document);
    }

    async waitForContent(window, maxWait = 5000) {
        const startTime = Date.now();

        while (Date.now() - startTime < maxWait) {
            // Check if dynamic content has loaded
            if (window.document.querySelector('.dynamic-content')) {
                break;
            }
            await new Promise(resolve => setTimeout(resolve, 100));
        }
    }

    extractData(document) {
        // Extract dynamically generated content
        const dynamicElements = Array.from(document.querySelectorAll('.dynamic-item'));

        return dynamicElements.map(element => ({
            id: element.dataset.id,
            title: element.querySelector('.item-title')?.textContent.trim(),
            description: element.querySelector('.item-desc')?.textContent.trim(),
            timestamp: element.dataset.timestamp
        }));
    }
}

// Usage
const scraper = new JSDomScraper();
scraper.scrapeWithScripts('https://example.com/dynamic-content')
    .then(results => {
        console.log('Extracted dynamic content:', results);
    });

// Structure your scrapers as classes for better organization
class WebScraper {
    constructor(options = {}) {
        this.delay = options.delay || 1000;
        this.retries = options.retries || 3;
        this.userAgent = options.userAgent || 'Mozilla/5.0 (compatible; Bot/1.0)';
    }

    async scrape(url) {
        // Implement your scraping logic here
    }

    async handleErrors(error, url) {
        // Centralized error handling
    }

    async saveResults(data, filename) {
        // Centralized data persistence
    }
}