如何抓取 NoCodeList：完整的网页抓取指南

from playwright.sync_api import sync_playwright

def run():
    with sync_playwright() as p:
        # 启动浏览器以处理 JavaScript
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto("https://nocodelist.co/software/nocode-api")
        
        # 等待动态内容渲染完成
        page.wait_for_selector("h1")
        
        # 从 DOM 中提取渲染后的数据
        data = {
            "name": page.inner_text("h1"),
            "pricing": page.inner_text("div:has-text('Pricing:')"),
            "description": page.inner_text("div.blog")
        }
        
        print(data)
        browser.close()

run()

import scrapy
from scrapy_playwright.page import PageMethod

class NoCodeSpider(scrapy.Spider):
    name = 'nocodelist'
    
    def start_requests(self):
        yield scrapy.Request(
            "https://nocodelist.co/",
            meta={
                "playwright": True,
                "playwright_page_methods": [
                    # 等待 SPA 中的可点击卡片出现
                    PageMethod("wait_for_selector", ".clickable-element")
                ]
            }
        )

    def parse(self, response):
        # Scrapy-Playwright 返回完整渲染的 HTML
        for item in response.css('.clickable-element'):
            yield {
                'tool_name': item.css('div::text').get(),
                'link': item.attrib.get('href')
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  // 使用 networkidle2 确保所有动态组件加载完毕
  await page.goto('https://nocodelist.co/software/nocode-api', { waitUntil: 'networkidle2' });

  const results = await page.evaluate(() => {
    return {
      title: document.querySelector('h1')?.innerText,
      pricing: document.body.innerText.match(/Pricing: .+/)?.[0],
      customer_types: Array.from(document.querySelectorAll('h3')).find(el => el.innerText.includes('Typical Customers'))?.nextElementSibling?.innerText
    };
  });

  console.log(results);
  await browser.close();
})();