Upworkのスクレイピング方法

from playwright.sync_api import sync_playwright

def scrape_upwork_jobs():
    with sync_playwright() as p:
        # Headless=Falseにすることで基本的なボット検知を回避しやすくなります
        browser = p.chromium.launch(headless=False)
        page = browser.new_page()
        
        # 検索結果ページへ移動
        page.goto("https://www.upwork.com/nx/search/jobs/?q=react")
        
        # DOMへ求人リストが読み込まれるのを待機
        page.wait_for_selector('[data-test="job-tile-list"]')
        
        # 求人カードからデータを抽出
        jobs = page.query_selector_all('[data-test="JobTile"]')
        for job in jobs:
            title = job.query_selector("h3").inner_text()
            print(f"Extracted: {title.strip()}")
        
        browser.close()

scrape_upwork_jobs()

import scrapy

class UpworkSpider(scrapy.Spider):
    name = "upwork_spider"
    start_urls = ["https://www.upwork.com/nx/search/jobs/?q=automation"]

    def parse(self, response):
        # Upworkの場合、Scrapy-Playwrightのようなミドルウェアが必要です
        for job in response.css('[data-test="JobTile"]'):
            yield {
                "title": job.css("h3 a::text").get(),
                "posted": job.css('[data-test="posted-on"]::text').get(),
                "description": job.css('[data-test="job-description"]::text').get(),
            }

        next_page = response.css("button.up-pagination-next::attr(href)").get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({ headless: false });
  const page = await browser.newPage();
  
  await page.goto('https://www.upwork.com/nx/search/jobs/?q=nodejs');

  // 求人結果を含むコンテナを待機
  await page.waitForSelector('[data-test="JobTile"]');

  const results = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('[data-test="JobTile"]'));
    return items.map(item => ({
      title: item.querySelector('h3').innerText.trim(),
      budget: item.querySelector('[data-test="job-type"]').innerText.trim()
    }));
  });

  console.log(results);
  await browser.close();
})();