Century 21 をスクレイピングする方法：不動産データの技術ガイド

from playwright.sync_api import sync_playwright

def scrape_with_playwright():
    with sync_playwright() as p:
        # ステルスに近いパラメータでの起動を推奨
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)')
        page = context.new_page()
        
        # 検索結果ページへ移動
        page.goto('https://www.century21.com/real-estate/los-angeles-ca/LCCALOSANGELES/')
        
        # JavaScriptによる物件グリッドの読み込みを待機
        page.wait_for_selector('.property-card')
        
        cards = page.query_selector_all('.property-card')
        for card in cards:
            price = card.query_selector('.property-card-price').inner_text()
            address = card.query_selector('.property-address').inner_text()
            print({'price': price, 'address': address})
            
        browser.close()

scrape_with_playwright()

import scrapy

class Century21Spider(scrapy.Spider):
    name = 'c21_spider'
    start_urls = ['https://www.century21.com/real-estate/miami-fl/LCCFMIAMI/']

    def parse(self, response):
        # CSSセレクターを使用して物件コンテナをループ処理
        for listing in response.css('.property-card'):
            yield {
                'address': listing.css('.property-address::text').get().strip(),
                'price': listing.css('.property-card-price::text').get().strip(),
                'url': response.urljoin(listing.css('a.card-anchor::attr(href)').get())
            }
        
        # ページネーションバーの次ページリンクをたどる
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

async function scrapeCentury21() {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // 現実的なビューポートを設定
  await page.setViewport({ width: 1280, height: 800 });
  
  await page.goto('https://www.century21.com/real-estate/chicago-il/LCCICHICAGO/');
  
  // Reactコンポーネントがリスティングをレンダリングするのを待機
  await page.waitForSelector('.property-card');
  
  const properties = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.property-card')).map(el => ({
      price: el.querySelector('.property-card-price')?.innerText,
      address: el.querySelector('.property-address')?.innerText,
      beds: el.querySelector('.property-beds')?.innerText
    }));
  });
  
  console.log(properties);
  await browser.close();
}

scrapeCentury21();