Good Booksをスクレイピングする方法 | Good Books Web Scraper

from playwright.sync_api import sync_playwright

def run(playwright):
    # Launch browser
    browser = playwright.chromium.launch(headless=True)
    page = browser.new_page()
    
    # Navigate to Good Books listings
    page.goto('https://goodbooks.io/books')
    
    # Wait for the book items to load
    page.wait_for_selector('.book-item')
    
    # Extract book data from the page
    books = page.query_selector_all('.book-item')
    for book in books:
        title = book.query_selector('h5').inner_text()
        author = book.query_selector('h6').inner_text()
        print(f'Scraped: {title} by {author}')
    
    # Close connection
    browser.close()

with sync_playwright() as playwright:
    run(playwright)

import scrapy

class GoodbooksSpider(scrapy.Spider):
    name = 'goodbooks'
    allowed_domains = ['goodbooks.io']
    start_urls = ['https://goodbooks.io/books']

    def parse(self, response):
        # Extract details for each book item
        for book in response.css('.book-item-class'):
            yield {
                'title': book.css('h5::text').get(),
                'author': book.css('h6::text').get(),
                'url': response.urljoin(book.css('a::attr(href)').get()),
            }

        # Handle simple pagination link
        next_page = response.css('a.next-page-selector::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  await page.goto('https://goodbooks.io/top-100/all-books');
  
  // Ensure cards are rendered
  await page.waitForSelector('.book-card');

  const data = await page.evaluate(() => {
    const items = Array.from(document.querySelectorAll('.book-card'));
    return items.map(item => ({
      title: item.querySelector('h5') ? item.querySelector('h5').innerText : 'N/A',
      author: item.querySelector('h6') ? item.querySelector('h6').innerText : 'N/A'
    }));
  });

  console.log(data);
  await browser.close();
})();