如何爬取 ImmoScout24：房地产数据指南

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        # 使用隐身配置启动
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36',
            locale='de-DE'
        )
        page = context.new_page()
        
        # 导航至搜索结果页
        page.goto('https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten', wait_until='networkidle')
        
        # 等待列表渲染完成
        page.wait_for_selector('.result-list-entry__data')
        
        # 使用定位器提取标题
        titles = page.locator('.result-list-entry__brand-title').all_inner_texts()
        for title in titles:
            print(f'发现房源: {title}')
            
        browser.close()

run()

import scrapy

class ImmoSpider(scrapy.Spider):
    name = 'immoscout'
    start_urls = ['https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten']

    def parse(self, response):
        # 遍历每个房源列表容器
        for listing in response.css('.result-list-entry__data'):
            yield {
                'title': listing.css('.result-list-entry__brand-title::text').get(),
                'price': listing.css('.result-list-entry__primary-criterion:nth-child(1) dd::text').get(),
                'rooms': listing.css('.result-list-entry__primary-criterion:nth-child(3) dd::text').get(),
                'area': listing.css('.result-list-entry__primary-criterion:nth-child(2) dd::text').get(),
            }
            
        # 通过寻找“下一页”按钮处理分页
        next_page = response.css('a[data-is24-test="pagination-next"]::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // 模拟真实的德国用户
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36');
  
  await page.goto('https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten');
  
  // 在浏览器上下文中执行评估
  const results = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('.result-list-entry__brand-title'));
    return items.map(item => item.textContent.trim());
  });
  
  console.log('找到的标题:', results);
  await browser.close();
})();