We Work Remotelyをスクレイピングする方法：究極のガイド

import asyncio
from playwright.async_api import async_playwright

async def run():
    async with async_playwright() as p:
        # ヘッドレスブラウザを起動
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto('https://weworkremotely.com/')
        # メインコンテナが読み込まれるのを待機
        await page.wait_for_selector('.jobs-container')
        jobs = await page.query_selector_all('li.feature')
        for job in jobs:
            title = await job.query_selector('.title')
            if title:
                print(await title.inner_text())
        await browser.close()

asyncio.run(run())

import scrapy

class WwrSpider(scrapy.Spider):
    name = 'wwr_spider'
    start_urls = ['https://weworkremotely.com/']

    def parse(self, response):
        # 求人アイテムをループで処理
        for job in response.css('li.feature'):
            yield {
                'title': job.css('span.title::text').get(),
                'company': job.css('span.company::text').get(),
                'url': response.urljoin(job.css('a::attr(href)').get())
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://weworkremotely.com/');
  // evaluateを使用してデータを抽出
  const jobs = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('li.feature')).map(li => ({
      title: li.querySelector('.title')?.innerText.trim(),
      company: li.querySelector('.company')?.innerText.trim()
    }));
  });
  console.log(jobs);
  await browser.close();
})();