Exploit-DBをスクレイピングする方法 | Exploit Databaseウェブスクレイパー

from playwright.sync_api import sync_playwright
def scrape_exploit_db():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto('https://www.exploit-db.com/')
        # AJAX経由でDataTablesにデータが入るのを待機
        page.wait_for_selector('table#exploits-table')
        rows = page.query_selector_all('table#exploits-table tbody tr')
        for row in rows[:5]:
            print(row.inner_text())
        browser.close()
scrape_exploit_db()

import scrapy
class ExploitSpider(scrapy.Spider):
    name = 'exploit_spider'
    start_urls = ['https://www.exploit-db.com/']
    def parse(self, response):
        # このサイトでは、Scrapyにscrapy-playwrightのようなJSミドルウェアが必要です
        for exploit in response.css('table#exploits-table tbody tr'):
            yield {
                'title': exploit.css('td.title a::text').get(),
                'id': exploit.css('td.id::text').get(),
                'cve': exploit.css('td.cve a::text').get()
            }

const puppeteer = require('puppeteer');
(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://www.exploit-db.com/', { waitUntil: 'networkidle2' });
  const results = await page.evaluate(() => {
    const rows = Array.from(document.querySelectorAll('table#exploits-table tbody tr'));
    return rows.map(row => row.innerText);
  });
  console.log(results.slice(0, 5));
  await browser.close();
})();