Pollen.comのスクレイピング方法：地域別アレルギーデータ抽出ガイド

from playwright.sync_api import sync_playwright

def run(playwright):
    browser = playwright.chromium.launch(headless=True)
    page = browser.new_page()
    # 特定のZIPコードの予測ページに移動
    page.goto('https://www.pollen.com/forecast/current/pollen/20001')
    
    # AngularJSが動的な花粉指数をレンダリングするのを待機
    page.wait_for_selector('.forecast-level')
    
    data = {
        'pollen_index': page.inner_text('.forecast-level'),
        'status': page.inner_text('.forecast-level-desc'),
        'allergens': [el.inner_text() for el in page.query_selector_all('.top-allergen-item span')]
    }
    
    print(f'Data for 20001: {data}')
    browser.close()

with sync_playwright() as playwright:
    run(playwright)

import scrapy

class PollenSpider(scrapy.Spider):
    name = 'pollen_spider'
    start_urls = ['https://www.pollen.com/forecast/current/pollen/20001']

    def parse(self, response):
        # 動的コンテンツの場合は、Scrapy-Playwrightなどのミドルウェアを使用してください
        # この標準的なparseメソッドは、見出しのような静的な要素を処理します
        yield {
            'url': response.url,
            'page_title': response.css('title::text').get(),
            'news_headlines': response.css('article h2 a::text').getall()
        }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  // 本物のブラウザを模倣するためにUser-Agentを設定
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64)');
  
  await page.goto('https://www.pollen.com/forecast/current/pollen/20001');
  
  // 動的な予測レベルが表示されるのを待機
  await page.waitForSelector('.forecast-level');
  
  const data = await page.evaluate(() => ({
    pollenIndex: document.querySelector('.forecast-level')?.innerText,
    description: document.querySelector('.forecast-level-desc')?.innerText,
    location: document.querySelector('h1')?.innerText
  }));

  console.log(data);
  await browser.close();
})();