The AA(theaa.com) 스크래핑 방법: 자동차 및 보험 데이터를 위한 기술 가이드

import asyncio
from playwright.async_api import async_playwright

async def scrape_aa():
    async with async_playwright() as p:
        # 디버깅 시에는 headed 브라우저, 그렇지 않으면 headless로 실행
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        
        # listing 페이지로 이동
        await page.goto('https://www.theaa.com/used-cars/bmw/3-series')
        
        # JS를 통해 렌더링되는 listing 아이템을 기다림
        await page.wait_for_selector('.listing-item')
        
        # 페이지 컨텍스트에서 데이터 추출
        cars = await page.eval_on_selector_all('.listing-item', """
            elements => elements.map(el => ({
                title: el.querySelector('h3')?.innerText,
                price: el.querySelector('strong')?.innerText,
                mileage: el.querySelector('.mileage')?.innerText
            }))
        """)
        
        for car in cars:
            print(car)
            
        await browser.close()

asyncio.run(scrape_aa())

import scrapy

class AACarsSpider(scrapy.Spider):
    name = 'aa_spider'
    allowed_domains = ['theaa.com']
    start_urls = ['https://www.theaa.com/used-cars/ford/fiesta']

    def parse(self, response):
        # listing 블록 반복 처리
        for car in response.css('.listing-item'):
            yield {
                'make_model': car.css('h3::text').get(),
                'price': car.css('strong::text').get(),
                'details': car.css('ul.listing-details li::text').getall()
            }
        
        # 페이지네이션 링크 처리
        next_page = response.css('a.next-pagination::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  // 일반적인 User-Agent로 봇 프로필 마스킹
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
  
  await page.goto('https://www.theaa.com/used-cars/brands');
  
  // 크롤링을 위한 자동차 브랜드 링크 추출
  const brands = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('a[href^="/used-cars/"]'))
                .map(a => a.innerText.trim())
                .filter(text => text.length > 0);
  });
  
  console.log('Available Brands:', brands);
  await browser.close();
})();