GoAbroadの留学プログラムをスクレイピングする方法

from playwright.sync_api import sync_playwright

def scrape_goabroad():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto('https://www.goabroad.com/study-abroad/search/italy/study-abroad-1')
        page.wait_for_selector('.listing-card')
        
        # Load More ボタンをクリックして、より多くのリスティングを表示させる
        for _ in range(3):
            load_more = page.query_selector('button:has-text("Load More")')
            if load_more:
                load_more.click()
                page.wait_for_timeout(2000)

        programs = page.query_selector_all('.listing-card')
        for prog in programs:
            title = prog.query_selector('h4').inner_text()
            print(f'Program: {title}')
        browser.close()

scrape_goabroad()

import scrapy

class GoAbroadSpider(scrapy.Spider):
    name = 'goabroad'
    start_urls = ['https://www.goabroad.com/study-abroad/search/italy/study-abroad-1']

    def parse(self, response):
        # 初期ページからプログラム情報を抽出
        for program in response.css('.listing-card'):
            yield {
                'title': program.css('h4::text').get(),
                'provider': program.css('.provider-name::text').get(),
                'rating': program.css('.rating-score::text').get()
            }
        
        # ページネーションが存在すればたどる
        next_page = response.css('a.pagination-next::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://www.goabroad.com/study-abroad/search/italy/study-abroad-1');
  await page.waitForSelector('.listing-card');

  const data = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.listing-card')).map(el => ({
      title: el.querySelector('h4')?.innerText,
      provider: el.querySelector('.provider-name')?.innerText
    }));
  });

  console.log(data);
  await browser.close();
})();