SeLoger Bureaux & Commerces 스크래핑 방법

import asyncio
from playwright.async_api import async_playwright

async def scrape_bucom():
    async with async_playwright() as p:
        # headless=False는 기본적인 봇 탐지 트리거를 피하는 데 도움이 됩니다
        browser = await p.chromium.launch(headless=False)
        context = await browser.new_context(
            user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
        )
        page = await context.new_page()
        try:
            # 특정 상업 카테고리 및 도시 타겟팅
            await page.goto('https://www.seloger-bureaux-commerces.com/achat/bureau/lyon', wait_until='networkidle')
            
            # 매물 카드가 렌더링될 때까지 대기
            await page.wait_for_selector('div[data-testid="listing-card"]', timeout=15000)
            
            listings = await page.query_selector_all('div[data-testid="listing-card"]')
            for card in listings:
                title = await card.query_selector('h2')
                price = await card.query_selector('span[class*="Price"]')
                print(f"제목: {await title.inner_text()} | 가격: {await price.inner_text()}")
        except Exception as e:
            print(f'스크래핑 실패: {e}')
        finally:
            await browser.close()

asyncio.run(scrape_bucom())

import scrapy

class SeLogerBucomSpider(scrapy.Spider):
    name = 'bucom_spider'
    allowed_domains = ['seloger-bureaux-commerces.com']
    start_urls = ['https://www.seloger-bureaux-commerces.com/location/boutique']

    custom_settings = {
        'DOWNLOAD_DELAY': 5,
        'RANDOMIZE_DOWNLOAD_DELAY': True,
        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
        'COOKIES_ENABLED': True
    }

    def parse(self, response):
        # 매물 결과 컨테이너에서 데이터 추출
        for listing in response.css('div[class*="Card_container"]'):
            yield {
                'title': listing.css('h2::text').get(),
                'price': listing.css('span[class*="Price"]::text').get(),
                'surface': listing.css('span[class*="Surface"]::text').get(),
                'link': listing.css('a::attr(href)').get()
            }

        # 간단한 페이지네이션 처리
        next_page = response.css('a[class*="PaginationNext"]::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());

(async () => {
    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    
    // 뷰포트와 에이전트 설정을 통해 사람의 행동 모방
    await page.setViewport({ width: 1280, height: 800 });
    
    try {
        await page.goto('https://www.seloger-bureaux-commerces.com/location/bureau/paris', { 
            waitUntil: 'networkidle2' 
        });
        
        const results = await page.evaluate(() => {
            return Array.from(document.querySelectorAll('a[class*="Card_title"]')).map(el => ({
                title: el.innerText,
                url: el.href
            }));
        });
        
        console.log(results);
    } catch (err) {
        console.error('추출 오류:', err);
    } finally {
        await browser.close();
    }
})();