如何爬取 Zillow：房地产数据终极指南 (2025)

from playwright.sync_api import sync_playwright

def scrape_zillow():
    with sync_playwright() as p:
        # 使用真实的 user agent 启动以绕过基础检查
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
        page = context.new_page()
        
        # 导航并等待 React 完全渲染内容
        page.goto('https://www.zillow.com/homes/for_sale/90210_rb/', wait_until='networkidle')
        
        # 等待房产卡片选择器出现
        page.wait_for_selector('[data-test="property-card"]')
        
        # 从渲染后的 DOM 中提取数据
        listings = page.query_selector_all('[data-test="property-card"]')
        for listing in listings:
            price_el = listing.query_selector('[data-test="property-card-price"]')
            address_el = listing.query_selector('address')
            
            price = price_el.inner_text() if price_el else "N/A"
            address = address_el.inner_text() if address_el else "N/A"
            print(f'价格: {price}, 地址: {address}')
            
        browser.close()

scrape_zillow()

import scrapy
import json

class ZillowSpider(scrapy.Spider):
    name = 'zillow'
    start_urls = ['https://www.zillow.com/homes/for_sale/90210_rb/']

    def parse(self, response):
        # Zillow 将数据存储在名为 __NEXT_DATA__ 的 JSON 脚本标签中
        # 这比爬取 HTML 布局更稳定
        json_data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').get()
        if json_data:
            data = json.loads(json_data)
            # 导航嵌套的 JSON 结构以找到列表结果
            results = data.get('props', {}).get('pageProps', {}).get('searchPageState', {}).get('cat1', {}).get('searchResults', {}).get('listResults', [])
            for item in results:
                yield {
                    'price': item.get('price'),
                    'address': item.get('address'),
                    'zpid': item.get('zpid'),
                    'bedrooms': item.get('beds'),
                    'bathrooms': item.get('baths')
                }

const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());

(async () => {
  // 启动带有 stealth 插件的浏览器以避免 DataDome 检测
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // 设置额外的 header 以显得更像真人
  await page.setExtraHTTPHeaders({
    'Accept-Language': 'en-US,en;q=0.9'
  });

  await page.goto('https://www.zillow.com/homes/for_sale/90210_rb/', { waitUntil: 'networkidle2' });

  const properties = await page.evaluate(() => {
    const cards = Array.from(document.querySelectorAll("[data-test='property-card']"));
    return cards.map(card => ({
      price: card.querySelector("[data-test='property-card-price']")?.innerText,
      address: card.querySelector("address")?.innerText
    }));
  });

  console.log(properties);
  await browser.close();
})();