IMDbのスクレイピング方法：映画データ抽出の完全ガイド

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        
        # 映画ページに移動
        page.goto('https://www.imdb.com/title/tt0111161/')
        
        # JSがレンダリングされるのを確実にするため、特定のデータ要素を待機
        page.wait_for_selector('[data-testid="hero__primary-text"]')
        
        # データを抽出
        movie_title = page.locator('[data-testid="hero__primary-text"]').inner_text()
        rating_val = page.locator('[data-testid="hero-rating-bar__aggregate-rating__score"] > span').first.inner_text()
        
        print({'title': movie_title, 'rating': rating_val})
        
        browser.close()

run()

import scrapy

class ImdbSpider(scrapy.Spider):
    name = 'imdb_spider'
    allowed_domains = ['imdb.com']
    start_urls = ['https://www.imdb.com/chart/top/']
    
    def parse(self, response):
        # トップ映画のリストをループします
        for movie in response.css('.ipc-metadata-list-summary-item'):
            yield {
                'title': movie.css('.ipc-title__text::text').get(),
                'rating': movie.css('.ipc-rating-star--rating::text').get(),
                'year': movie.css('.sc-b189961a-8::text').get(),
            }
            
        # 該当する場合、ページネーションを処理します
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

async function scrapeIMDb() {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // 本物のブラウザヘッダーを模倣
  await page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36');
  
  await page.goto('https://www.imdb.com/title/tt0111161/', { waitUntil: 'domcontentloaded' });

  const movieInfo = await page.evaluate(() => {
    const title = document.querySelector('[data-testid="hero__primary-text"]')?.innerText;
    const rating = document.querySelector('[data-testid="hero-rating-bar__aggregate-rating__score"]')?.innerText;
    return { title, rating };
  });

  console.log(movieInfo);
  await browser.close();
}

scrapeIMDb();