Trulia 부동산 데이터를 스크래핑하는 방법

from playwright.sync_api import sync_playwright

def scrape_trulia_playwright():
    with sync_playwright() as p:
        # 스텔스 기술이 필요함
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/119.0.0.0 Safari/537.36',
            viewport={'width': 1920, 'height': 1080}
        )
        page = context.new_page()
        
        # 이동 후 동적 부동산 카드가 로드될 때까지 대기
        page.goto('https://www.trulia.com/CA/San_Francisco/', wait_until='networkidle')
        page.wait_for_selector('[data-testid="property-card-details"]')
        
        # DOM에서 데이터 추출
        listings = page.query_selector_all('[data-testid="property-card-details"]')
        for item in listings:
            address = item.query_selector('[data-testid="property-address"]').inner_text()
            price = item.query_selector('[data-testid="property-price"]').inner_text()
            print(f'Address: {address} | Price: {price}')
            
        browser.close()

scrape_trulia_playwright()

import scrapy

class TruliaSpider(scrapy.Spider):
    name = 'trulia_spider'
    # 기본 보호 기능 우회를 위한 사용자 지정 설정
    custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Safari/537.36',
        'CONCURRENT_REQUESTS': 1,
        'DOWNLOAD_DELAY': 5
    }
    start_urls = ['https://www.trulia.com/CA/San_Francisco/']

    def parse(self, response):
        for card in response.css('[data-testid="property-card-details"]'):
            yield {
                'address': card.css('[data-testid="property-address"]::text').get(),
                'price': card.css('[data-testid="property-price"]::text').get(),
                'meta': card.css('[data-testid="property-meta"]::text').getall(),
            }
        
        # "Next" 버튼 링크 팔로우
        next_page = response.css('a[aria-label="Next Page"]::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // 실제 브라우저 header 모방
  await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US,en;q=0.9' });
  
  await page.goto('https://www.trulia.com/CA/San_Francisco/', { waitUntil: 'networkidle2' });
  
  const properties = await page.evaluate(() => {
    const data = [];
    const cards = document.querySelectorAll('[data-testid="property-card-details"]');
    cards.forEach(card => {
      data.push({
        address: card.querySelector('[data-testid="property-address"]')?.innerText,
        price: card.querySelector('[data-testid="property-price"]')?.innerText
      });
    });
    return data;
  });

  console.log(properties);
  await browser.close();
})();