Arc.dev 스크래핑 방법: 원격 채용 데이터 수집 완벽 가이드

from playwright.sync_api import sync_playwright

def scrape_arc():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        # 실제 사용자 프로필 또는 스텔스 설정 사용
        context = browser.new_context(user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36')
        page = context.new_page()
        
        # 탐색 및 콘텐츠 하이드레이션 대기
        page.goto('https://arc.dev/remote-jobs', wait_until='networkidle')
        
        # 채용 카드 요소 대기
        page.wait_for_selector('div[class*="JobCard_container"]')
        
        jobs = page.query_selector_all('div[class*="JobCard_container"]')
        for job in jobs:
            title = job.query_selector('h2').inner_text()
            company = job.query_selector('div[class*="JobCard_company"]').inner_text()
            print(f'스크래핑됨: {title} @ {company}')
        
        browser.close()

scrape_arc()

import scrapy

class ArcSpider(scrapy.Spider):
    name = 'arc_jobs'
    start_urls = ['https://arc.dev/remote-jobs']

    def parse(self, response):
        # Arc.dev를 위해서는 Scrapy에 (scrapy-playwright와 같은) JS 미들웨어가 필요합니다
        for job in response.css('div[class*="JobCard_container"]'):
            yield {
                'title': job.css('h2::text').get(),
                'company': job.css('div[class*="JobCard_company"]::text').get(),
                'salary': job.css('div[class*="JobCard_salary"]::text').get(),
                'tags': job.css('div[class*="JobCard_tags"] span::text').getall()
            }

        next_page = response.css('a[class*="Pagination_next"]::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
  await page.goto('https://arc.dev/remote-jobs', { waitUntil: 'networkidle2' });

  const jobData = await page.evaluate(() => {
    const cards = Array.from(document.querySelectorAll('div[class*="JobCard_container"]'));
    return cards.map(card => ({
      title: card.querySelector('h2')?.innerText,
      company: card.querySelector('div[class*="JobCard_company"]')?.innerText,
      location: card.querySelector('div[class*="JobCard_location"]')?.innerText
    }));
  });

  console.log(jobData);
  await browser.close();
})();