PHP Web Scraping

<?php
$ch = curl_init('https://books.toscrape.com/');
curl_setopt_array($ch, [
    CURLOPT_RETURNTRANSFER => true,          // return the body instead of printing it
    CURLOPT_FOLLOWLOCATION => true,
    CURLOPT_TIMEOUT        => 30,
    CURLOPT_USERAGENT      => 'Mozilla/5.0 (compatible; books-demo/1.0)',
]);
$html = curl_exec($ch);
if ($html === false) {
    exit('cURL error: ' . curl_error($ch) . PHP_EOL);
}
$status = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);

libxml_use_internal_errors(true);            // real-world HTML is rarely valid
$dom = new DOMDocument();
$dom->loadHTML($html);
$xpath = new DOMXPath($dom);

foreach ($xpath->query('//article[contains(@class, "product_pod")]') as $book) {
    $title = $xpath->query('.//h3/a', $book)->item(0)->getAttribute('title');
    $price = trim($xpath->query('.//p[contains(@class, "price_color")]', $book)->item(0)->textContent);
    echo "$title: $price\n";
}

composer require guzzlehttp/guzzle symfony/dom-crawler symfony/css-selector

<?php
require 'vendor/autoload.php';

use GuzzleHttp\Client;
use Symfony\Component\DomCrawler\Crawler;

$client = new Client([
    'timeout' => 30,
    'headers' => ['User-Agent' => 'Mozilla/5.0 (compatible; books-demo/1.0)'],
]);

$books = [];
$url = 'https://books.toscrape.com/catalogue/page-1.html';
while ($url) {
    $html = (string) $client->get($url)->getBody();
    $page = new Crawler($html, $url);                    // the URL lets links resolve

    $page->filter('article.product_pod')->each(function (Crawler $book) use (&$books) {
        $books[] = [
            'title'   => $book->filter('h3 a')->attr('title'),
            'price'   => $book->filter('.price_color')->text(),
            'inStock' => $book->filter('.instock')->count() > 0,
        ];
    });

    $next = $page->filter('li.next a');
    $url = $next->count() ? $next->link()->getUri() : null;   // absolute URL or null
    sleep(1);
}

file_put_contents('books.json', json_encode($books, JSON_PRETTY_PRINT));
echo count($books) . " books saved\n";

// app/Console/Commands/ScrapeBooks.php  ->  php artisan scrape:books
namespace App\Console\Commands;

use Illuminate\Console\Command;
use Illuminate\Support\Facades\Http;
use Symfony\Component\DomCrawler\Crawler;

class ScrapeBooks extends Command
{
    protected $signature = 'scrape:books';

    public function handle(): void
    {
        $response = Http::timeout(30)
            ->retry(3, 1000)                                  // 3 tries, 1 s apart
            ->withUserAgent('Mozilla/5.0 (compatible; books-demo/1.0)')
            ->get('https://books.toscrape.com/');

        $crawler = new Crawler($response->body());
        $crawler->filter('article.product_pod')->each(function (Crawler $book) {
            $this->line($book->filter('h3 a')->attr('title') . ' ' . $book->filter('.price_color')->text());
        });
    }
}

<?php
require 'vendor/autoload.php';
use Symfony\Component\Panther\Client;

$client = Client::createChromeClient();                  // needs Chrome and chromedriver
$client->request('GET', 'https://quotes.toscrape.com/js/');
$crawler = $client->waitFor('.quote');                  // wait until the quotes are rendered
$quotes = $crawler->filter('.quote .text')->each(fn ($node) => $node->text());
print_r($quotes);
$client->quit();