Imgurをスクレイピングする方法：画像データ抽出の包括的ガイド

import asyncio
from playwright.async_api import async_playwright

async def run():
    async with async_playwright() as p:
        # Launching browser with a standard viewport
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        
        # Navigate to Imgur
        await page.goto('https://imgur.com/gallery/hot')
        
        # Wait for the gallery items to load (JS rendered)
        await page.wait_for_selector('.Post-item')
        
        # Extract data from the first few items
        titles = await page.eval_on_selector_all('.Post-item-title', 'elements => elements.map(e => e.innerText)')
        for title in titles[:5]:
            print(f'Post Title: {title}')
            
        await browser.close()

asyncio.run(run())

import scrapy

class ImgurSpider(scrapy.Spider):
    name = 'imgur'
    start_urls = ['https://imgur.com/gallery/hot']
    
    def parse(self, response):
        # Scrapy extracts from the initial HTML; note that Imgur loads most content via JS
        for post in response.css('.Post-item'):
            yield {
                'title': post.css('.Post-item-title::text').get(),
                'link': post.css('a::attr(href)').get(),
            }
            
        # Example logic for finding the next page or API endpoint
        # Imgur often uses JSON API endpoints for pagination

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  // Mimic a desktop browser to reduce blocking risk
  await page.setViewport({ width: 1280, height: 800 });
  
  await page.goto('https://imgur.com/gallery/hot', { waitUntil: 'networkidle2' });
  
  // Extract post titles from the gallery
  const titles = await page.evaluate(() => {
    const elements = document.querySelectorAll('.Post-item-title');
    return Array.from(elements).map(el => el.innerText);
  });
  
  console.log('Found Titles:', titles.slice(0, 5));
  
  await browser.close();
})();