pump.funをスクレイピングする方法：Solanaのリアルタイムデータ取得に関する技術ガイド

import asyncio
from playwright.async_api import async_playwright

async def scrape_pump_fun():
    async with async_playwright() as p:
        # ステルス設定での起動を推奨
        browser = await p.chromium.launch(headless=True)
        context = await browser.new_context(
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
        )
        page = await context.new_page()
        
        # ナビゲートし、JSがtokenボードをレンダリングするのを待機
        await page.goto('https://pump.fun/board', wait_until='networkidle')
        await page.wait_for_selector('.token-card')
        
        tokens = await page.query_selector_all('.token-card')
        for token in tokens[:5]:
            name = await (await token.query_selector('.name')).inner_text()
            mcap = await (await token.query_selector('.market-cap')).inner_text()
            print(f'Token: {name} | MCAP: {mcap}')
            
        await browser.close()

asyncio.run(scrape_pump_fun())

import scrapy
from scrapy_playwright.page import PageMethod

class PumpFunSpider(scrapy.Spider):
    name = 'pump_spider'
    
    def start_requests(self):
        yield scrapy.Request(
            url='https://pump.fun/board',
            meta={
                'playwright': True,
                'playwright_page_methods': [
                    PageMethod('wait_for_selector', '.token-card')
                ]
            }
        )

    def parse(self, response):
        for token in response.css('.token-card'):
            yield {
                'name': token.css('.token-name::text').get(),
                'ticker': token.css('.ticker::text').get(),
                'mcap': token.css('.mcap-value::text').get(),
            }

const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  await page.goto('https://pump.fun/board', { waitUntil: 'networkidle2' });
  
  // tokenカードが動的に読み込まれるのを待機
  await page.waitForSelector('.token-card');
  
  const data = await page.evaluate(() => {
    const cards = Array.from(document.querySelectorAll('.token-card'));
    return cards.map(card => ({
      name: card.querySelector('.name')?.innerText,
      ticker: card.querySelector('.symbol')?.innerText,
      mcap: card.querySelector('.market-cap')?.innerText
    }));
  });
  
  console.log(data);
  await browser.close();
})();