Chambers and Partnersのスクレイピング方法 | 法律ランキングガイド

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        # モバイルのユーザーエージェントやstealthを使用すると検出を回避しやすくなります
        page.goto('https://chambers.com/legal-guide/uk-1', wait_until='networkidle')
        
        # JavaScript経由でランキングカードが読み込まれるのを待機
        page.wait_for_selector('.ranking-card')
        
        rankings = page.eval_on_selector_all('.ranking-card', "cards => cards.map(c => c.innerText)")
        for info in rankings:
            print(info)
            
        browser.close()

run()

import scrapy

class ChambersRankingSpider(scrapy.Spider):
    name = 'chambers_spider'
    start_urls = ['https://chambers.com/search?q=litigation']
    
    custom_settings = {
        'DOWNLOAD_DELAY': 3,
        'CONCURRENT_REQUESTS': 1,
        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/118.0.0.0 Safari/537.36'
    }

    def parse(self, response):
        for item in response.css('.search-result-item'):
            yield {
                'name': item.css('.title-link::text').get(),
                'firm': item.css('.firm-link::text').get(),
                'url': response.urljoin(item.css('a::attr(href)').get())
            }
        
        # ページネーション処理
        next_page = response.css('a.pagination-next::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  await page.setViewport({ width: 1280, height: 800 });
  
  // 特定の法域のランキングに移動
  await page.goto('https://chambers.com/legal-guide/global-2', { waitUntil: 'networkidle2' });
  
  const data = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('.ranking-row'));
    return items.map(item => ({
      name: item.querySelector('.name')?.innerText,
      band: item.querySelector('.band-indicator')?.innerText
    }));
  });
  
  console.log(data);
  await browser.close();
})();