ResearchGateのスクレイピング方法：論文と研究者データの抽出

import asyncio
from playwright.async_api import async_playwright

async def scrape_researchgate_search(query):
    async with async_playwright() as p:
        # Launching with stealth-like settings
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page(user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
        
        search_url = f'https://www.researchgate.net/search/publication?q={query}'
        await page.goto(search_url)
        
        # Wait for dynamic results to load
        await page.wait_for_selector('.nova-legacy-v-publication-item__title')
        
        # Extract titles
        titles = await page.eval_on_selector_all('.nova-legacy-v-publication-item__title a', 'nodes => nodes.map(n => n.innerText)')
        
        for i, title in enumerate(titles[:10]):
            print(f'{i+1}. {title}')
            
        await browser.close()

asyncio.run(scrape_researchgate_search('machine learning'))

import scrapy

class ResearchGateSpider(scrapy.Spider):
    name = 'rg_spider'
    allowed_domains = ['researchgate.net']
    
    # Use a custom settings dictionary for bot avoidance
    custom_settings = {
        'DOWNLOAD_DELAY': 3,
        'CONCURRENT_REQUESTS': 1,
        'USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/110.0.0.0 Safari/537.36'
    }

    def start_requests(self):
        urls = ['https://www.researchgate.net/search/publication?q=bioinformatics']
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        for item in response.css('.nova-legacy-v-publication-item__body'):
            yield {
                'title': item.css('.nova-legacy-v-publication-item__title a::text').get(),
                'link': response.urljoin(item.css('.nova-legacy-v-publication-item__title a::attr(href)').get()),
            }

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36');
  
  // Navigate to ResearchGate search
  await page.goto('https://www.researchgate.net/search/publication?q=neuroscience');
  
  // Wait for the specific container of results
  await page.waitForSelector('.nova-legacy-v-publication-item__title');

  const results = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.nova-legacy-v-publication-item__title a')).map(a => ({
      title: a.innerText.trim(),
      link: a.href
    }));
  });

  console.log(results);
  await browser.close();
})();