Cheapflights 스크래핑 방법 | 항공권 데이터 웹 스크래퍼

import asyncio
from playwright.async_api import async_playwright

async def scrape_cheapflights():
    async with async_playwright() as p:
        # Launching with a real-looking browser context
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page(user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/119.0.0.0')
        
        # Navigate to a specific flight search result
        await page.goto('https://www.cheapflights.com/flights/NYC-LON/2026-06-15')
        
        # Wait for flight results to load dynamically
        try:
            await page.wait_for_selector('.resultWrapper', timeout=15000)
            flights = await page.query_selector_all('.resultWrapper')
            for flight in flights[:5]:
                price = await flight.query_selector('.price-text')
                print(f'Found flight price: {await price.inner_text()}')
        except:
            print('Flight results did not load or were blocked.')
            
        await browser.close()

asyncio.run(scrape_cheapflights())

import scrapy

class CheapflightsSpider(scrapy.Spider):
    name = 'cheapflights_spider'
    start_urls = ['https://www.cheapflights.com/flights/']

    def parse(self, response):
        # Scrapy is best for crawling links; for search results, use Scrapy-Playwright
        for item in response.css('.destination-card'):
            yield {
                'destination': item.css('.city-name::text').get(),
                'price': item.css('.price-value::text').get(),
                'route': item.css('.route-info::text').get(),
            }

const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // Navigate to a search result
  await page.goto('https://www.cheapflights.com/flights/SFO-TYO/2026-08-20');
  
  // Wait for the dynamic flight cards to appear
  await page.waitForSelector('.resultWrapper', { timeout: 10000 });
  
  const results = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.resultWrapper')).map(el => ({
      price: el.querySelector('.price-text')?.innerText,
      airline: el.querySelector('.codeshare-airline-name')?.innerText
    }));
  });

  console.log(results);
  await browser.close();
})();