ProxyScrapeをスクレイピングする方法：究極のプロキシデータガイド

import asyncio
from playwright.async_api import async_playwright

async def scrape_proxyscrape_table():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto('https://proxyscrape.com/free-proxy-list')
        
        # JavaScriptによるテーブル行のレンダリングを待機
        await page.wait_for_selector('table tbody tr')
        
        proxies = await page.evaluate('''() => {
            const rows = Array.from(document.querySelectorAll('table tbody tr'));
            return rows.map(row => ({
                ip: row.cells[1]?.innerText.trim(),
                port: row.cells[2]?.innerText.trim(),
                country: row.cells[4]?.innerText.trim()
            }));
        }''')
        
        for proxy in proxies[:5]:
            print(proxy)
            
        await browser.close()

asyncio.run(scrape_proxyscrape_table())

import scrapy

class ProxySpider(scrapy.Spider):
    name = 'proxyscrape'
    start_urls = ['https://proxyscrape.com/free-proxy-list']

    def parse(self, response):
        # 注：テーブルは動的なことが多いため、Scrapyの場合はAPIミドルウェアを使用する方が良いですが、
        # ここでは静的要素のパースを試みます。
        for row in response.css('table tr'):
            yield {
                'ip': row.css('td:nth-child(2)::text').get(),
                'port': row.css('td:nth-child(3)::text').get(),
                'protocol': row.css('td:nth-child(1)::text').get(),
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://proxyscrape.com/free-proxy-list');

  // 動的テーブルの読み込みを待機
  await page.waitForSelector('table');

  const data = await page.evaluate(() => {
    const rows = Array.from(document.querySelectorAll('table tbody tr'));
    return rows.map(row => ({
      ip: row.querySelector('td:nth-child(2)')?.innerText,
      port: row.querySelector('td:nth-child(3)')?.innerText
    }));
  });

  console.log(data.slice(0, 10));
  await browser.close();
})();