Google検索結果をスクレイピングする方法

from playwright.sync_api import sync_playwright

def scrape_google():
    with sync_playwright() as p:
        # ヘッドレスブラウザの起動
        browser = p.chromium.launch(headless=True)
        page = browser.new_page(user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36')
        
        # Google検索へ移動
        page.goto('https://www.google.com/search?q=best+web+scrapers+2025')
        
        # オーガニック結果がロードされるのを待機
        page.wait_for_selector('.tF2Cxc')
        
        # データを抽出
        results = page.query_selector_all('.tF2Cxc')
        for res in results:
            title_el = res.query_selector('h3')
            link_el = res.query_selector('a')
            if title_el and link_el:
                print(f"{title_el.inner_text()}: {link_el.get_attribute('href')}")
        
        browser.close()

scrape_google()

import scrapy

class GoogleSearchSpider(scrapy.Spider):
    name = 'google_spider'
    allowed_domains = ['google.com']
    start_urls = ['https://www.google.com/search?q=python+web+scraping']

    def parse(self, response):
        # オーガニック検索結果のコンテナをループ処理
        for result in response.css('.tF2Cxc'):
            yield {
                'title': result.css('h3::text').get(),
                'link': result.css('a::attr(href)').get(),
                'snippet': result.css('.VwiC3b::text').get()
            }

        # 「次へ」ボタンを探してページネーションを処理
        next_page = response.css('a#pnnext::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  // 重要：実際のUser-Agentを設定する
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36');
  
  await page.goto('https://www.google.com/search?q=scraping+best+practices');
  
  // オーガニック結果の抽出
  const data = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('.tF2Cxc'));
    return items.map(el => ({
      title: el.querySelector('h3')?.innerText,
      link: el.querySelector('a')?.href,
      snippet: el.querySelector('.VwiC3b')?.innerText
    }));
  });

  console.log(data);
  await browser.close();
})();