AirlineQuality.com (Skytrax) のレビューをスクレイピングする方法

from playwright.sync_api import sync_playwright

def scrape_reviews():
    with sync_playwright() as p:
        # Launch browser to handle JS/Cloudflare
        browser = p.chromium.launch(headless=True)
        context = browser.new_context()
        page = context.new_page()
        
        # Navigate to target airline page
        page.goto("https://www.airlinequality.com/airline-reviews/british-airways/")
        
        # Wait for review articles to appear
        page.wait_for_selector('article[itemprop="review"]')
        
        reviews = page.locator('article[itemprop="review"]').all()
        for review in reviews:
            header = review.locator('.text_header').inner_text()
            text = review.locator('.text_content').inner_text()
            print(f"Processing: {header}")
            
        browser.close()

if __name__ == "__main__":
    scrape_reviews()

import scrapy

class SkytraxSpider(scrapy.Spider):
    name = 'skytrax'
    start_urls = ['https://www.airlinequality.com/airline-reviews/british-airways/?pagesize=100']
    
    def parse(self, response):
        for review in response.css('article.review-stats'):
            yield {
                'title': review.css('h2.text_header::text').get(),
                'rating': review.css('span[itemprop="ratingValue"]::text').get(),
                'text': review.css('div.text_content::text').get(),
                'recommended': review.xpath("//td[contains(@class, 'review-rating-header') and text()='Recommended']/following-sibling::td/text()").get()
            }
        
        next_page = response.css('article.pagination li:last-child a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  await page.goto('https://www.airlinequality.com/airline-reviews/british-airways/');
  
  const reviews = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('article[itemprop="review"]'));
    return items.map(item => ({
      title: item.querySelector('.text_header')?.innerText,
      score: item.querySelector('span[itemprop="ratingValue"]')?.innerText,
      content: item.querySelector('.text_content')?.innerText
    }));
  });

  console.log(reviews);
  await browser.close();
})();