如何抓取 Behance：创意数据提取分步指南

from playwright.sync_api import sync_playwright

def scrape_behance():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto("https://www.behance.net/search/projects?field=architecture")
        # 等待动态内容加载
        page.wait_for_selector(".ProjectCover-root-167")
        # 向下滚动以触发延迟加载
        page.mouse.wheel(0, 5000)
        page.wait_for_timeout(2000)
        projects = page.query_selector_all(".ProjectCover-root-167")
        data = []
        for p_elem in projects:
            title = p_elem.query_selector(".ProjectCover-title-3_1").inner_text()
            owner = p_elem.query_selector(".ProjectCover-username-28M").inner_text()
            data.append({"title": title, "owner": owner})
        print(data)
        browser.close()

scrape_behance()

import scrapy
from scrapy_playwright.page import PageMethod

class BehanceSpider(scrapy.Spider):
    name = "behance"
    start_urls = ["https://www.behance.net/search/projects?field=interaction"]

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(
                url,
                meta={"playwright": True, "playwright_page_methods": [
                    PageMethod("wait_for_selector", ".ProjectCover-root-167"),
                ]},
            )

    def parse(self, response):
        for project in response.css(".ProjectCover-root-167"):
            yield {
                "title": project.css(".ProjectCover-title-3_1::text").get(),
                "url": project.css("a::attr(href)").get(),
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://www.behance.net/search/projects?field=branding');
  // 确保内容已加载
  await page.waitForSelector('.ProjectCover-content-3Z_');
  const projects = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.ProjectCover-root-167')).map(el => ({
      title: el.querySelector('.ProjectCover-title-3_1')?.innerText,
      owner: el.querySelector('.ProjectCover-username-28M')?.innerText
    }));
  });
  console.log(projects);
  await browser.close();
})();