使用Puppeteer高效抓取TripAdvisor景点数据：完整指南

npm init -y
npm install puppeteer

const puppeteer = require("puppeteer");

(async () => {
    const browser = await puppeteer.launch({ headless: true }); // 生产环境建议设置为true
    const page = await browser.newPage();
    const url = 'https://www.tripadvisor.com/Attractions-g297476-Activities-c42-Cartagena_Cartagena_District_Bolivar_Department.html';

    await page.goto(url, { waitUntil: 'load', timeout: 30000 });
    await page.waitForSelector('main', { timeout: 10000 }); // 等待主内容区加载

    // 方法一：使用page.evaluate和querySelectorAll
    const titles1 = await page.evaluate(() => 
        Array.from(document.querySelectorAll('article .VLKGO a:not([class])'), (e) => ({
            title: e.innerText.split('.').pop().trim() // 移除序号，清理文本
        }))
    );
    console.log('Titles (Method 1):', titles1);

    // 方法二：使用page.$$eval
    const titles2 = await page.$$eval('article .VLKGO span > div', el => 
        el.map(x => x.textContent.split('.').pop().trim())
    );
    console.log('Titles (Method 2):', titles2);

    await browser.close();
})().catch(err => console.error(err));

const puppeteer = require("puppeteer");

let browser; // 声明浏览器实例以便在finally块中关闭
(async () => {
    browser = await puppeteer.launch({ headless: true }); // 建议生产环境设为true
    const page = await browser.newPage();

    const url = 'https://www.tripadvisor.com/Attractions-g297476-Activities-c42-Cartagena_Cartagena_District_Bolivar_Department.html';

    await page.goto(url, { waitUntil: 'load', timeout: 30000 });
    await page.waitForSelector('main', { timeout: 10000 }); // 等待主要内容区加载

    // 获取所有景点卡片的元素句柄
    let places = await page.$$('article');

    let data = [];
    for (let place of places) {
        let header = {};
        let image = '';
        let desc = '';
        let by = {};
        let price = '';
        let priceTxt = '';

        try {
            // 提取标题和链接
            header = await place.$eval('.VLKGO a:not([class])', el => {
                // 移除标题前的序号（如 "1."）
                const name = el.textContent.split('.').pop().trim();
                const link = el.getAttribute('href');
                return { name, link: `https://www.tripadvisor.com${link}` }; // 拼接完整链接
            });
        } catch (e) {
            console.warn("Title/Link not found for a card:", e.message);
            // 可以选择跳过或赋予默认值
        }

        try {
            // 提取图片URL，通常在srcset中包含多尺寸图片，取最大的
            image = await place.$eval('picture > img[srcset]', el => {
                const srcset = el.getAttribute('srcset');
                if (srcset) {
                    // 获取最后一个（通常是最大尺寸）的图片URL，并移除' 2x'等描述
                    return srcset.split(',').pop().replace(/2x/gi, '').trim();
                }
                return '';
            });
        } catch (e) {
            console.warn("Image not found for a card:", e.message);
        }

        try {
            // 提取描述
            desc = await place.$eval('a:not([class]) > div > span', el => el.textContent.trim());
        } catch (e) {
            console.warn("Description not found for a card:", e.message);
        }

        try {
            // 提取提供者/作者信息
            by = await place.$eval('.VLKGO div > div > div > a', el => {
                const name = el.textContent.replace('By ', '').trim();
                const link = el.getAttribute('href');
                return { name, link: `https://www.tripadvisor.com${link}` };
            });
        } catch (e) {
            // 提供者信息可能不是所有卡片都有
            // console.warn("Provider info not found for a card:", e.message);
        }

        try {
            // 提取价格（如果存在）
            price = await place.$eval('[data-automation=cardPrice]', el => el.textContent.trim());
        } catch (e) {
            // 价格信息可能不是所有卡片都有
        }

        try {
            // 提取价格文本（如 "起" 或 "每人"）
            priceTxt = await place.$eval('div:nth-child(1) > div:nth-child(3):not([class])', el => el.textContent.trim());
        } catch (e) {
            // 价格文本可能不是所有卡片都有
        }

        // 将提取到的数据添加到结果数组
        data.push({
            name: header.name || 'N/A',
            link: header.link || 'N/A',
            desc: desc || 'N/A',
            image: image || 'N/A',
            price: price || 'N/A',
            priceTxt: priceTxt || 'N/A',
            by: by.name ? by : { name: 'N/A', link: 'N/A' }
        });
    }

    console.log(JSON.stringify(data, null, 2)); // 打印结构化的JSON数据

})().catch(err => console.error("An error occurred:", err)).finally(() => {
    // 确保浏览器在任何情况下都被关闭
    if (browser) {
        browser.close();
    }
});