scrapy爬虫数据入库教程

<code class="python"># 数据库引擎
DATABASE_ENGINE = 'mysql'
# 数据库用户名
DATABASE_USER = 'root'
# 数据库密码
DATABASE_PASSWORD = 'mypassword'
# 数据库主机地址
DATABASE_HOST = 'localhost'
# 数据库名称
DATABASE_NAME = 'scrapydata'</code>

<code class="python">from scrapy.pipelines.images import ImagesPipeline
from itemadapter import ItemAdapter
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine

class DatabasePipeline(ImagesPipeline):
    def __init__(self, *args, **kwargs):
        # 创建 SQLAlchemy 引擎
        self.engine = create_engine("mysql+pymysql://{}:{}@{}:{}/{}".format(
            DATABASE_USER, DATABASE_PASSWORD, DATABASE_HOST, DATABASE_PORT, DATABASE_NAME))
        # 创建会话工厂
        self.session_factory = sessionmaker(bind=self.engine)

    def process_item(self, item, spider):
        # 获取 Item 适配器
        adapter = ItemAdapter(item)

        # 创建数据库会话
        session = self.session_factory()

        # 将 Item 数据保存到数据库
        # ...

        # 提交会话并关闭连接
        session.commit()
        session.close()

        return item</code>

<code class="python">ITEM_PIPELINES = {
    'myproject.pipelines.DatabasePipeline': 300,
}</code>

<code class="python"># 将 Item 中的标题字段保存到数据库
title = adapter.get('title')
new_record = MyModel(title=title)
session.add(new_record)</code>