PHP Web スクレイピング完全ガイド — cURL/DOMDocument/Goutte/Headless Chrome

方式一覧

1. file_get_contents + DOMDocument

loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8'));
libxml_clear_errors();

// XPath で抽出
$xpath = new DOMXPath($dom);
$titles = $xpath->query('//h2[@class="article-title"]/a');

foreach ($titles as $node) {
    echo $node->textContent . " => " . $node->getAttribute('href') . "\n";
}

// 全 img タグの src
$imgs = $xpath->query('//img');
foreach ($imgs as $img) {
    echo $img->getAttribute('src') . "\n";
}

2. cURL（Cookie・認証・User-Agent）

 $url,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_TIMEOUT        => 30,
        CURLOPT_CONNECTTIMEOUT => 10,
        CURLOPT_USERAGENT      => 'Mozilla/5.0 (compatible; MyBot/1.0; +https://example.com/bot)',
        CURLOPT_COOKIEJAR      => '/tmp/cookies.txt',
        CURLOPT_COOKIEFILE     => '/tmp/cookies.txt',
        CURLOPT_HTTPHEADER     => array_merge([
            'Accept: text/html,application/xhtml+xml',
            'Accept-Language: ja,en;q=0.9',
        ], $headers),
        CURLOPT_SSL_VERIFYPEER => true,
        CURLOPT_ENCODING       => '',           // gzip 等自動展開
    ]);

    $body = curl_exec($ch);
    $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
    $err  = curl_error($ch);
    curl_close($ch);

    if ($body === false) throw new RuntimeException("cURL: $err");
    if ($code >= 400)    throw new RuntimeException("HTTP $code");
    return $body;
}

// 使用
$html = fetch('https://example.com/login');

3. Guzzle（モダン HTTP クライアント）

 $jar,
    'timeout' => 30,
    'headers' => [
        'User-Agent' => 'MyBot/1.0',
    ],
]);

// GET
$response = $client->get('https://example.com/');
$html = (string) $response->getBody();

// POST フォーム
$response = $client->post('https://example.com/login', [
    'form_params' => [
        'username' => 'taro',
        'password' => 'secret',
    ],
]);

// JSON API
$response = $client->get('https://api.example.com/items', [
    'headers' => ['Authorization' => 'Bearer xxx'],
]);
$data = json_decode((string) $response->getBody(), true);

4. Symfony DomCrawler (CSS セレクタ)

filter('h2.article-title a')->each(function (Crawler $node) {
    echo $node->text() . " => " . $node->attr('href') . "\n";
});

// XPath
$crawler->filterXPath('//meta[@property="og:image"]')->each(function ($node) {
    echo $node->attr('content') . "\n";
});

// 属性・テキスト
$title = $crawler->filter('title')->text();
$canonical = $crawler->filter('link[rel="canonical"]')->attr('href');

// 子要素
$crawler->filter('.product')->each(function (Crawler $product) {
    $name  = $product->filter('.name')->text();
    $price = $product->filter('.price')->text();
    echo "$name : $price\n";
});

5. Goutte / BrowserKit（高レベル API）

request('GET', 'https://example.com/login');

// フォーム
$form = $crawler->selectButton('ログイン')->form([
    'username' => 'taro',
    'password' => 'secret',
]);
$crawler = $browser->submit($form);

// リンククリック
$link = $crawler->selectLink('マイページ')->link();
$crawler = $browser->click($link);

// 結果のパース
$crawler->filter('.user-info')->each(function ($node) {
    echo $node->text() . "\n";
});

6. 動的サイト: Headless Chrome

JavaScript で動的にレンダリングされる SPA や React/Vue サイトには Headless Chrome が必要です:

createBrowser([
    'headless' => true,
    'noSandbox' => true,
    'windowSize' => [1920, 1080],
]);

try {
    $page = $browser->createPage();
    $page->navigate('https://spa.example.com')->waitForNavigation();

    // JS 実行完了を待つ
    $page->waitUntilContainsElement('.product-list');

    // HTML 取得
    $html = $page->getHtml();

    // JS 実行で値取得
    $title = $page->evaluate('document.title')->getReturnValue();

    // スクリーンショット
    $page->screenshot()->saveToFile('screenshot.png');
} finally {
    $browser->close();
}

7. Puppeteer 連携（Node.js 経由）

// scraper.js (Node.js)
const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({ headless: 'new' });
  const page = await browser.newPage();
  await page.goto(process.argv[2], { waitUntil: 'networkidle0' });

  const data = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.product')).map(el => ({
      name:  el.querySelector('.name').innerText,
      price: el.querySelector('.price').innerText,
    }));
  });

  console.log(JSON.stringify(data));
  await browser.close();
})();

minInterval = $minIntervalSec;
    }

    public function wait(): void {
        if ($this->lastRequest !== null) {
            $elapsed = microtime(true) - $this->lastRequest;
            if ($elapsed < $this->minInterval) {
                usleep((int)(($this->minInterval - $elapsed) * 1_000_000));
            }
        }
        $this->lastRequest = microtime(true);
    }
}

$rl = new RateLimiter(2.0);
foreach ($urls as $url) {
    $rl->wait();
    $html = fetch($url);
    // ...
}