چگونه داده‌های ImmoScout24 را اسکرپ کنیم: راهنمای جامع داده‌های املاک

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        # Launching with stealth-like configurations
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36',
            locale='de-DE'
        )
        page = context.new_page()
        
        # Navigate to search results
        page.goto('https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten', wait_until='networkidle')
        
        # Wait for listings to render
        page.wait_for_selector('.result-list-entry__data')
        
        # Extract titles using locators
        titles = page.locator('.result-list-entry__brand-title').all_inner_texts()
        for title in titles:
            print(f'Listing found: {title}')
            
        browser.close()

run()

import scrapy

class ImmoSpider(scrapy.Spider):
    name = 'immoscout'
    start_urls = ['https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten']

    def parse(self, response):
        # Loop through each property listing container
        for listing in response.css('.result-list-entry__data'):
            yield {
                'title': listing.css('.result-list-entry__brand-title::text').get(),
                'price': listing.css('.result-list-entry__primary-criterion:nth-child(1) dd::text').get(),
                'rooms': listing.css('.result-list-entry__primary-criterion:nth-child(3) dd::text').get(),
                'area': listing.css('.result-list-entry__primary-criterion:nth-child(2) dd::text').get(),
            }
            
        # Handle pagination by finding the 'Next' button
        next_page = response.css('a[data-is24-test="pagination-next"]::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // Mimic a real German user
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36');
  
  await page.goto('https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten');
  
  // Evaluation in the browser context
  const results = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('.result-list-entry__brand-title'));
    return items.map(item => item.textContent.trim());
  });
  
  console.log('Titles found:', results);
  await browser.close();
})();