如何抓取 IMDb：电影数据提取全指南

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        
        # 导航至电影页面
        page.goto('https://www.imdb.com/title/tt0111161/')
        
        # 等待特定的数据元素，确保 JS 已渲染
        page.wait_for_selector('[data-testid="hero__primary-text"]')
        
        # 提取数据
        movie_title = page.locator('[data-testid="hero__primary-text"]').inner_text()
        rating_val = page.locator('[data-testid="hero-rating-bar__aggregate-rating__score"] > span').first.inner_text()
        
        print({'title': movie_title, 'rating': rating_val})
        
        browser.close()

run()

import scrapy

class ImdbSpider(scrapy.Spider):
    name = 'imdb_spider'
    allowed_domains = ['imdb.com']
    start_urls = ['https://www.imdb.com/chart/top/']
    
    def parse(self, response):
        # 遍历电影列表
        for movie in response.css('.ipc-metadata-list-summary-item'):
            yield {
                'title': movie.css('.ipc-title__text::text').get(),
                'rating': movie.css('.ipc-rating-star--rating::text').get(),
                'year': movie.css('.sc-b189961a-8::text').get(),
            }
            
        # 处理分页（如有）
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

async function scrapeIMDb() {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // 模拟真实浏览器请求头
  await page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36');
  
  await page.goto('https://www.imdb.com/title/tt0111161/', { waitUntil: 'domcontentloaded' });

  const movieInfo = await page.evaluate(() => {
    const title = document.querySelector('[data-testid="hero__primary-text"]')?.innerText;
    const rating = document.querySelector('[data-testid="hero-rating-bar__aggregate-rating__score"]')?.innerText;
    return { title, rating };
  });

  console.log(movieInfo);
  await browser.close();
}

scrapeIMDb();