Century 21 스크래핑 방법: 기술적인 부동산 데이터 수집 가이드

from playwright.sync_api import sync_playwright

def scrape_with_playwright():
    with sync_playwright() as p:
        # 스텔스 모드와 유사한 파라미터로 실행하는 것을 권장합니다
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)')
        page = context.new_page()
        
        # 검색 결과 페이지로 이동
        page.goto('https://www.century21.com/real-estate/los-angeles-ca/LCCALOSANGELES/')
        
        # JavaScript를 통해 부동산 그리드가 로드될 때까지 대기
        page.wait_for_selector('.property-card')
        
        cards = page.query_selector_all('.property-card')
        for card in cards:
            price = card.query_selector('.property-card-price').inner_text()
            address = card.query_selector('.property-address').inner_text()
            print({'price': price, 'address': address})
            
        browser.close()

scrape_with_playwright()

import scrapy

class Century21Spider(scrapy.Spider):
    name = 'c21_spider'
    start_urls = ['https://www.century21.com/real-estate/miami-fl/LCCFMIAMI/']

    def parse(self, response):
        # CSS selector를 사용하여 매물 컨테이너 반복 처리
        for listing in response.css('.property-card'):
            yield {
                'address': listing.css('.property-address::text').get().strip(),
                'price': listing.css('.property-card-price::text').get().strip(),
                'url': response.urljoin(listing.css('a.card-anchor::attr(href)').get())
            }
        
        # 페이지네이션 바에서 다음 페이지 링크 이동
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

async function scrapeCentury21() {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // 실제적인 뷰포트 설정
  await page.setViewport({ width: 1280, height: 800 });
  
  await page.goto('https://www.century21.com/real-estate/chicago-il/LCCICHICAGO/');
  
  // React 컴포넌트가 리스팅을 렌더링할 때까지 대기
  await page.waitForSelector('.property-card');
  
  const properties = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.property-card')).map(el => ({
      price: el.querySelector('.property-card-price')?.innerText,
      address: el.querySelector('.property-address')?.innerText,
      beds: el.querySelector('.property-beds')?.innerText
    }));
  });
  
  console.log(properties);
  await browser.close();
}

scrapeCentury21();