Healthlineのスクレイピング方法：究極の健康・医療データガイド

import asyncio
from playwright.async_api import async_playwright

async def scrape():
    async with async_playwright() as p:
        # ステルス設定でヘッドレスブラウザを起動
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        
        # 疾患ページへ移動
        await page.goto('https://www.healthline.com/health/gerd', wait_until='networkidle')
        
        # JavaScript実行を使用してデータを抽出
        data = await page.evaluate('''() => {
            return {
                title: document.querySelector('h1')?.innerText,
                intro: document.querySelector('p')?.innerText,
                reviewer: document.querySelector('.css-1p2092a')?.innerText
            };
        }''')
        
        print(data)
        await browser.close()

asyncio.run(scrape())

import scrapy

class HealthlineSpider(scrapy.Spider):
    name = 'healthline'
    start_urls = ['https://www.healthline.com/directory/topics']

    def parse(self, response):
        # 疾患記事へのリンクを検索
        for link in response.css('a.css-1m17l36::attr(href)').getall():
            yield response.follow(link, self.parse_article)

    def parse_article(self, response):
        yield {
            'title': response.css('h1::text').get(),
            'author': response.css('.css-1p2092a::text').get(),
            'body': response.css('div.article-body p::text').getall(),
            'last_updated': response.css('time::attr(datetime)').get()
        }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  // 本物のブラウザを模倣するためにUser-Agentを設定
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36');
  
  await page.goto('https://www.healthline.com/health/gerd', { waitUntil: 'networkidle2' });
  
  const data = await page.evaluate(() => {
    return {
      title: document.querySelector('h1')?.innerText,
      headers: Array.from(document.querySelectorAll('h2')).map(h => h.innerText),
      medicalReviewer: document.querySelector('.css-1p2092a')?.innerText
    };
  });

  console.log(data);
  await browser.close();
})();