如何抓取 Apartments.com | Apartments.com 网络爬虫指南

from playwright.sync_api import sync_playwright

def scrape_apartments():
    with sync_playwright() as p:
        # Launching with stealth-like parameters
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/119.0.0.0')
        page = context.new_page()
        
        # Navigate to a listing page
        page.goto('https://www.apartments.com/los-angeles-ca/', wait_until='networkidle')
        
        # Wait for the main listings container to load
        page.wait_for_selector('.placard')
        
        # Extracting property names and prices
        properties = page.query_selector_all('.placard')
        for prop in properties:
            name = prop.query_selector('.property-title').inner_text()
            price = prop.query_selector('.property-pricing').inner_text() if prop.query_selector('.property-pricing') else 'N/A'
            print(f'Property: {name} | Price: {price}')
            
        browser.close()

scrape_apartments()

import scrapy

class ApartmentsSpider(scrapy.Spider):
    name = 'apartments_spider'
    start_urls = ['https://www.apartments.com/chicago-il/']

    custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/119.0.0.0',
        'CONCURRENT_REQUESTS': 1,
        'DOWNLOAD_DELAY': 3
    }

    def parse(self, response):
        for listing in response.css('article.placard'):
            yield {
                'name': listing.css('.property-title::text').get(),
                'address': listing.css('.property-address::text').get(),
                'price': listing.css('.property-pricing::text').get(),
            }

        next_page = response.css('a.next::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // Set a realistic user agent
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/119.0.0.0');

  try {
    await page.goto('https://www.apartments.com/houston-tx/', { waitUntil: 'networkidle2' });
    
    const data = await page.evaluate(() => {
      const items = Array.from(document.querySelectorAll('.placard'));
      return items.map(item => ({
        title: item.querySelector('.property-title')?.innerText,
        price: item.querySelector('.property-pricing')?.innerText,
        link: item.querySelector('a.property-link')?.href
      }));
    });

    console.log(data);
  } catch (err) {
    console.error('Extraction failed:', err);
  } finally {
    await browser.close();
  }
})();