Sacramento Delta Property Management 데이터 스크래핑 방법

import asyncio
from playwright.async_api import async_playwright

async def run():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        context = await browser.new_context(user_agent='Mozilla/5.0')
        page = await context.new_page()
        
        # 가용성 페이지로 이동
        await page.goto('https://www.sacdelt.com/availability', wait_until='networkidle')
        
        # AppFolio iframe 또는 React 컴포넌트 로딩 대기
        await page.wait_for_selector('.listing-item')
        
        listings = await page.query_selector_all('.listing-item')
        for listing in listings:
            title = await listing.query_selector('.listing-title')
            price = await listing.query_selector('.listing-rent')
            print({
                'title': await title.inner_text() if title else 'N/A',
                'price': await price.inner_text() if price else 'N/A'
            })
        
        await browser.close()

asyncio.run(run())

import scrapy
from scrapy_playwright.page import PageMethod

class SacDeltSpider(scrapy.Spider):
    name = 'sacdelt_spider'

    def start_requests(self):
        yield scrapy.Request(
            'https://www.sacdelt.com/availability',
            meta={
                'playwright': True,
                'playwright_page_methods': [
                    PageMethod('wait_for_selector', '.listing-item'),
                ]
            }
        )

    def parse(self, response):
        for listing in response.css('.listing-item'):
            yield {
                'address': listing.css('.listing-address::text').get(),
                'rent': listing.css('.listing-rent::text').get(),
                'beds': listing.css('.listing-beds::text').get(),
                'url': response.urljoin(listing.css('a::attr(href)').get())
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  // 현실적인 user agent 설정
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
  
  await page.goto('https://www.sacdelt.com/availability', { waitUntil: 'networkidle2' });

  // 동적 콘텐츠 렌더링 대기
  await page.waitForSelector('.listing-item');

  const results = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('.listing-item'));
    return items.map(item => ({
      title: item.querySelector('h3')?.innerText,
      price: item.querySelector('.listing-rent')?.innerText,
      address: item.querySelector('.listing-address')?.innerText
    }));
  });

  console.log(results);
  await browser.close();
})();