如何爬取 Good Books | Good Books 网页抓取工具

from playwright.sync_api import sync_playwright

def run(playwright):
    # 启动浏览器
    browser = playwright.chromium.launch(headless=True)
    page = browser.new_page()
    
    # 导航至 Good Books 列表页
    page.goto('https://goodbooks.io/books')
    
    # 等待书籍项加载
    page.wait_for_selector('.book-item')
    
    # 从页面提取书籍数据
    books = page.query_selector_all('.book-item')
    for book in books:
        title = book.query_selector('h5').inner_text()
        author = book.query_selector('h6').inner_text()
        print(f'已抓取: {title} 作者: {author}')
    
    # 关闭连接
    browser.close()

with sync_playwright() as playwright:
    run(playwright)

import scrapy

class GoodbooksSpider(scrapy.Spider):
    name = 'goodbooks'
    allowed_domains = ['goodbooks.io']
    start_urls = ['https://goodbooks.io/books']

    def parse(self, response):
        # 提取每本书项的详细信息
        for book in response.css('.book-item-class'):
            yield {
                'title': book.css('h5::text').get(),
                'author': book.css('h6::text').get(),
                'url': response.urljoin(book.css('a::attr(href)').get()),
            }

        # 处理简单的分页链接
        next_page = response.css('a.next-page-selector::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  await page.goto('https://goodbooks.io/top-100/all-books');
  
  // 确保卡片已渲染
  await page.waitForSelector('.book-card');

  const data = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('.book-card'));
    return items.map(item => ({
      title: item.querySelector('h5') ? item.querySelector('h5').innerText : '无',
      author: item.querySelector('h6') ? item.querySelector('h6').innerText : '无'
    }));
  });

  console.log(data);
  await browser.close();
})();