解决BeautifulSoup爬取网页表格中动态内容缺失问题

<tr data-partnumber="B93297">
<td><div>Compute – Ampere A1 – OCPU</div></td>
<td class="rw-theme-40bg"><div data-minrange="3000" data-type="vcpu"><br/>
</div></td>
<td><span data-minrange="3000"><span data-model="pay_as_you_go"></span></span></td>
<td><div>OCPU per hour</div></td>
</tr>

<tr data-partnumber="B93297">
<th scope="row"><div>Compute – Ampere A1 – OCPU</div></th>
<td class="rw-theme-40bg"><div data-minrange="3000" data-type="vcpu" class="">$0.01<span></span></div></td>
<td><span data-minrange="3000" class="">$0.01<span></span></span></td>
<td><div>OCPU per hour</div></td>
</tr>

from bs4 import BeautifulSoup
import requests
import re
import pandas as pd

# 1. 获取动态价格数据
# ----------------------------------------------------------------------
json_data_url = 'https://www.oracle.com/a/ocom/docs/pricing/cloud-price-list.json'
try:
    json_data = requests.get(json_data_url).json()
except requests.exceptions.RequestException as e:
    print(f"Error fetching JSON data: {e}")
    json_data = {} # Fallback to empty dict
currency = 'USD' # 假设我们关注美元价格

# 2. 获取静态HTML页面内容
# ----------------------------------------------------------------------
url = 'https://www.oracle.com/uk/cloud/compute/pricing/#compute-vm'
try:
    oracle_website = requests.get(url).text
    soup = BeautifulSoup(oracle_website, "html.parser")
except requests.exceptions.RequestException as e:
    print(f"Error fetching HTML page: {e}")
    exit() # Exit if we can't get the page

# 3. 遍历表格并整合数据
# ----------------------------------------------------------------------
rows_data = []
# 定位包含虚拟机定价表的div
virtual_machine_table_div = soup.find("div", class_="rc34w5 rw-neutral-00bg")

if virtual_machine_table_div and virtual_machine_table_div.table:
    virtual_machine_table = virtual_machine_table_div.table
    for compute_products_tbody in virtual_machine_table.find_all("tbody"):
        trs = compute_products_tbody.find_all("tr")

        for tr in trs:
            part_number = None
            # 尝试从tr标签的data-partnumber属性获取
            if 'data-partnumber' in tr.attrs:
                part_number = tr['data-partnumber']
            else:
                # 尝试从td内部的div获取data-partnumber
                # 注意：原始问题中的第二个td是价格占位符，这里我们找第一个可能包含partNumber的td
                # 更稳健的做法是遍历所有td或直接从tr获取
                first_td_div = tr.find('td')
                if first_td_div:
                    div_with_part_num = first_td_div.find('div', {'data-partnumber': re.compile('.*')})
                    if div_with_part_num:
                        part_number = div_with_part_num['data-partnumber']

            # 初始化价格
            comp_price = '-'
            unit_price = '-'

            # 根据part_number从JSON数据中查找价格
            if part_number:
                if part_number == 'B93297': # 特殊处理 'Compute – Ampere A1 – OCPU'
                    if 'vcpuRangeItems' in json_data and part_number in json_data['vcpuRangeItems']:
                        price_info = json_data['vcpuRangeItems'][part_number].get(currency)
                        if price_info:
                            comp_price = price_info[-1]['value']
                            unit_price = price_info[-1]['value']
                elif part_number not in json_data['vcpuItems'] and part_number not in json_data['items'] and part_number not in json_data['rangeItems']:
                    # 如果part_number不在任何已知价格类型中，则价格未知
                    comp_price = '-'
                    unit_price = '-'
                else:
                    # 从vcpuItems获取Comparison Price
                    if 'vcpuItems' in json_data and part_number in json_data['vcpuItems']:
                        comp_price = json_data['vcpuItems'][part_number].get(currency, '-')

                    # 从items或rangeItems获取Unit Price
                    if 'items' in json_data and part_number in json_data['items']:
                        unit_price = json_data['items'][part_number].get(currency, '-')
                    elif 'rangeItems' in json_data and part_number in json_data['rangeItems']:
                        price_info = json_data['rangeItems'][part_number].get(currency)
                        if price_info:
                            unit_price = price_info[-1]['value'] # 取最后一个（可能是最高范围或默认值）

            elif part_number is None: # 处理没有part_number的产品，如Free tier
                product_name_td = tr.find('td')
                if product_name_td and "Free" in product_name_td.text:
                    comp_price = 'Free'
                    unit_price = 'Free'

            # 提取产品名称和单位
            tds = tr.find_all('td')
            product_name = tds[0].text.strip() if len(tds) > 0 else ''
            unit = tds[-1].text.strip() if len(tds) > 0 else '' # 最后一个td是单位

            row = {
                'partNumber': part_number,
                'Product': product_name,
                'Comparison Price (/vCPU)': comp_price,
                'Unit price': unit_price,
                'Unit': unit,
            }
            rows_data.append(row)
else:
    print("Could not find the virtual machine table.")

# 4. 使用pandas生成结构化输出
# ----------------------------------------------------------------------
df = pd.DataFrame(rows_data)
print(df.to_markdown(index=False))