如何抓取 GitHub | 2025 终极技术指南

from playwright.sync_api import sync_playwright

def run(query):
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        context = browser.new_context()
        page = context.new_page()
        # Search for repositories
        page.goto(f'https://github.com/search?q={query}&type=repositories')
        # Wait for dynamic results to render
        page.wait_for_selector('div[data-testid="results-list"]')
        # Extract names
        repos = page.query_selector_all('a.Link__StyledLink-sc-14289xe-0')
        for repo in repos[:10]:
            print(f'Repo found: {repo.inner_text()}')
        browser.close()

run('web-scraping')

import scrapy

class GithubTrendingSpider(scrapy.Spider):
    name = 'github_trending'
    start_urls = ['https://github.com/trending']

    def parse(self, response):
        for repo in response.css('article.Box-row'):
            yield {
                'name': repo.css('h2 a::text').getall()[-1].strip(),
                'language': repo.css('span[itemprop="programmingLanguage"]::text').get(),
                'stars': repo.css('a.Link--muted::text').get().strip()
            }
        # Pagination logic for next trending pages if applicable
        next_page = response.css('a.next_page::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  // Set user agent to avoid basic bot detection
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36');
  
  await page.goto('https://github.com/psf/requests');
  
  const data = await page.evaluate(() => {
    return {
      title: document.querySelector('strong.mr-2 > a').innerText,
      stars: document.querySelector('#repo-stars-counter-star').innerText,
      forks: document.querySelector('#repo-network-counter').innerText
    };
  });

  console.log(data);
  await browser.close();
})();