如何抓取 CoinMarketCap：完整的网页抓取指南

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        # 启动有头浏览器有时有助于调试
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0')
        page = context.new_page()
        
        page.goto('https://coinmarketcap.com/')
        
        # 等待动态 React 表格完全渲染
        page.wait_for_selector('table.cmc-table')
        
        # 使用特定类提取前 10 个代币名称
        coins = page.query_selector_all('.coin-item-name')
        for coin in coins[:10]:
            print(coin.inner_text())
            
        browser.close()

run()

import scrapy

class CoinSpider(scrapy.Spider):
    name = 'coin_spider'
    start_urls = ['https://coinmarketcap.com/']
    
    def parse(self, response):
        # Scrapy 选择器可以高效处理 CSS 路径
        for row in response.css('table.cmc-table tbody tr'):
            yield {
                'name': row.css('p.coin-item-name::text').get(),
                'symbol': row.css('p.coin-item-symbol::text').get(),
                'price': row.css('div.sc-131cee3c-0 span::text').get()
            }
            
        # 处理后续页面的基本分页
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.setViewport({ width: 1280, height: 800 });
  
  // 使用 networkidle2 确保大多数 React 组件已完成加载
  await page.goto('https://coinmarketcap.com/', { waitUntil: 'networkidle2' });

  const data = await page.evaluate(() => {
    const results = [];
    const rows = document.querySelectorAll('table.cmc-table tbody tr');
    rows.forEach((row, index) => {
      if (index < 10) {
        results.push({
          name: row.querySelector('.coin-item-name')?.innerText,
          price: row.querySelector('.sc-131cee3c-0')?.innerText
        });
      }
    });
    return results;
  });

  console.log(data);
  await browser.close();
})();