Brown Real Estate NCをスクレイピングする方法 | フェイエットビル物件スクレイパー

import asyncio
from playwright.async_api import async_playwright

async def scrape_brown():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto('https://www.brownrealestatenc.com/fayetteville-homes-for-rent')
        
        # AppFolioリストウィジェットがコンテンツをレンダリングするのを待機
        await page.wait_for_selector('.listing-item')
        
        listings = await page.query_selector_all('.listing-item')
        for item in listings:
            title = await item.query_selector('.listing-title')
            price = await item.query_selector('.listing-rent')
            print({'title': await title.inner_text(), 'rent': await price.inner_text()})
        
        await browser.close()

asyncio.run(scrape_brown())

import scrapy

class BrownSpider(scrapy.Spider):
    name = 'brown_spider'
    start_urls = ['https://www.brownrealestatenc.com/fayetteville-homes-for-rent']

    def parse(self, response):
        # このサイトにはScrapy JSミドルウェア（scrapy-playwrightなど）が必要です
        for listing in response.css('.listing-item'):
            yield {
                'name': listing.css('.listing-title::text').get(),
                'rent': listing.css('.listing-rent::text').get(),
                'address': listing.css('.listing-address::text').get()
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://www.brownrealestatenc.com/fayetteville-homes-for-rent');
  
  // 動的なリストアイテムが表示されるまで待機
  await page.waitForSelector('.listing-item');
  
  const data = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.listing-item')).map(el => ({
      title: el.querySelector('.listing-title')?.innerText,
      rent: el.querySelector('.listing-rent')?.innerText
    }));
  });

  console.log(data);
  await browser.close();
})();