php数据如何制作Web爬虫抓取数据_php数据爬虫程序开发入门

function fetchPage($url) {
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; PHP Crawler)');
    curl_setopt($ch, CURLOPT_TIMEOUT, 10);
    $html = curl_exec($ch);
    if (curl_error($ch)) {
        echo 'Curl error: ' . curl_error($ch);
    }
    curl_close($ch);
    return $html;
}
$html = fetchPage('https://example.com');

$dom = new DOMDocument();
libxml_use_internal_errors(true); // 忽略HTML格式错误
$dom->loadHTML($html);

$xpath = new DOMXPath($dom);
$links = $xpath->query('//a[@href]');

foreach ($links as $link) {
    echo $link->getAttribute('href') . "\n";
}

curl_setopt($ch, CURLOPT_COOKIEJAR, 'cookie.txt');
curl_setopt($ch, CURLOPT_COOKIEFILE, 'cookie.txt');

$stmt = $pdo->prepare("INSERT INTO articles (title, url) VALUES (?, ?)");
$stmt->execute([$title, $url]);

$data = ['title' => $title, 'url' => $link];
file_put_contents('data.json', json_encode($data, JSON_UNESCAPED_UNICODE));