如何抓取 Substack 通讯和文章

import asyncio
from playwright.async_api import async_playwright

async def scrape_substack():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto('https://example.substack.com/archive')
        await page.wait_for_selector('.post-preview')
        for _ in range(3):
            await page.mouse.wheel(0, 1000)
            await asyncio.sleep(2)
        posts = await page.query_selector_all('.post-preview')
        for post in posts:
            title = await post.inner_text('.post-preview-title')
            print({'title': title})
        await browser.close()

asyncio.run(scrape_substack())

import scrapy

class SubstackSpider(scrapy.Spider):
    name = 'substack'
    start_urls = ['https://example.substack.com/archive']

    def parse(self, response):
        for post in response.css('div.post-preview'):
            yield {
                'title': post.css('a.post-preview-title::text').get(),
                'url': post.css('a.post-preview-title::attr(href)').get(),
                'date': post.css('time::attr(datetime)').get()
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.substack.com/archive');
  await page.waitForSelector('.post-preview');
  const posts = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.post-preview')).map(item => ({
      title: item.querySelector('.post-preview-title')?.innerText,
      link: item.querySelector('.post-preview-title')?.href
    }));
  });
  console.log(posts);
  await browser.close();
})();