使用 Beautiful Soup 和 Scrapy 进行网页抓取：高效、负责任地提取数据

<code class="bash">pip install beautifulsoup4 requests</code>

<code class="python">import requests
from bs4 import BeautifulSoup

url = 'https://example-blog.com'
response = requests.get(url)

if response.status_code == 200:
    soup = BeautifulSoup(response.text, 'html.parser')
    titles = soup.find_all('h1', class_='entry-title')
    if titles:
        for title in titles:
            print(title.get_text(strip=True))
    else:
        print("未找到标题，请检查HTML结构和选择器。")
else:
    print(f"页面获取失败，状态码：{response.status_code}")</code>

<code class="bash">pip install scrapy</code>

<code class="bash">scrapy startproject quotes_scraper
cd quotes_scraper</code>

<code class="python">import scrapy

class QuotesSpider(scrapy.Spider):
    name = 'quotes'
    start_urls = ['http://quotes.toscrape.com']

    def parse(self, response):
        for quote in response.css('div.quote'):
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
            }
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, callback=self.parse)</code>

<code class="bash">scrapy crawl quotes -o quotes.json</code>