如何抓取 Bento.me | Bento.me Web Scraper

from playwright.sync_api import sync_playwright

def run(playwright):
    # 启动 headless 浏览器
    browser = playwright.chromium.launch(headless=True)
    page = browser.new_page()
    # 导航到 Bento profile
    page.goto('https://bento.me/alex')
    # 等待主 profile 标题加载
    page.wait_for_selector('h1')
    
    # 从渲染后的页面提取内容
    name = page.inner_text('h1')
    links = [a.get_attribute('href') for a in page.query_selector_all('a')]
    
    print(f'Profile Name: {name}')
    print(f'Links found: {len(links)}')
    
    browser.close()

with sync_playwright() as playwright:
    run(playwright)

import scrapy
import json

class BentoSpider(scrapy.Spider):
    name = 'bento'
    start_urls = ['https://bento.me/alex']

    def parse(self, response):
        # 定位包含 profile JSON 状态的 Next.js 数据脚本
        raw_data = response.xpath('//script[@id="__NEXT_DATA__"]/text()').get()
        if raw_data:
            data = json.loads(raw_data)
            profile = data['props']['pageProps']['initialState']['user']
            yield {
                'name': profile.get('name'),
                'about': profile.get('about'),
                'links': [tile.get('url') for tile in profile.get('tiles', []) if tile.get('url')],
                'socials': profile.get('socials'),
                'verified': profile.get('isVerified')
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  // 使用 networkidle2 确保所有 widgets 加载完成
  await page.goto('https://bento.me/alex', { waitUntil: 'networkidle2' });

  const profileData = await page.evaluate(() => {
    // 直接从 DOM 访问内部状态
    const dataElement = document.getElementById('__NEXT_DATA__');
    if (dataElement) {
      const nextData = JSON.parse(dataElement.innerText);
      return nextData.props.pageProps.initialState.user;
    }
    return null;
  });

  console.log(profileData);
  await browser.close();
})();