如何使用Python脚本为站群实现蜘蛛统计和创建独立分析页面？

import sys
import pandas as pd

logfile = sys.argv[1] if len(sys.argv) > 1 else "access.log"

# 读取日志文件
try:
    with open(logfile, 'r', encoding='utf-8') as f:  # 添加编码处理，避免乱码
        log_data = f.readlines()
except FileNotFoundError:
    print(f"Error: Log file '{logfile}' not found.")
    sys.exit(1)

# 解析日志数据 (根据你的日志格式调整)
log_entries = []
for line in log_data:
    try:
        parts = line.split()
        if len(parts) > 10:
            log_entries.append({
                "ip": parts[0],
                "date": parts[3][1:],
                "request": parts[6], # 调整字段索引以匹配你的日志格式
                "status": parts[8],
                "agent": " ".join(parts[11:]) # 调整字段索引以匹配你的日志格式
            })
    except IndexError:
        print(f"Warning: Skipping malformed log line: {line.strip()}")

df = pd.DataFrame(log_entries)

# 定义蜘蛛 User-Agent
spider_agents = [
    "googlebot",
    "bingbot",
    "baiduspider",
    "yandexbot",
    "sogou",
    # 添加其他蜘蛛 User-Agent
]

# 筛选蜘蛛请求
df['spider'] = df['agent'].str.extract(f"({'|'.join(spider_agents)})", expand=False)
spider_df = df[df['spider'].notna()]

# 按蜘蛛类型和日期汇总统计
spider_summary = spider_df.groupby(['spider', 'date']).size().reset_index(name='count')

print(spider_summary)

# 可选：保存数据到 CSV 文件
spider_summary.to_csv('spider_stats.csv', index=False)

python3 spider_stats.py access.log

from flask import Flask, render_template, jsonify
import pandas as pd

app = Flask(__name__)

@app.route('/')
def index():
    try:
        df = pd.read_csv('spider_stats.csv')
        data = df.to_dict(orient='records')
        return render_template('index.html', data=data)
    except FileNotFoundError:
        return "Error: spider_stats.csv not found."


if __name__ == '__main__':
    app.run(debug=True) # debug模式方便开发，上线时需关闭