C# Web Scraping

dotnet new console -n BookScraper
cd BookScraper
dotnet add package HtmlAgilityPack
dotnet run

// Program.cs
using HtmlAgilityPack;

using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(30) };
http.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; books-demo/1.0)");

var url = new Uri("https://books.toscrape.com/catalogue/page-1.html");
var books = new List<(string Title, string Price)>();

while (url != null)
{
    var html = await http.GetStringAsync(url);
    var doc = new HtmlDocument();
    doc.LoadHtml(html);

    var cards = doc.DocumentNode.SelectNodes("//article[contains(@class,'product_pod')]");
    foreach (var card in cards ?? Enumerable.Empty<HtmlNode>())        // SelectNodes returns null on no match
    {
        var title = card.SelectSingleNode(".//h3/a").GetAttributeValue("title", "");
        var price = card.SelectSingleNode(".//p[contains(@class,'price_color')]").InnerText.Trim();
        books.Add((HtmlEntity.DeEntitize(title), price));
    }

    var next = doc.DocumentNode.SelectSingleNode("//li[@class='next']/a");
    url = next == null ? null : new Uri(url, next.GetAttributeValue("href", ""));   // resolve relative link
    await Task.Delay(1000);
}

Console.WriteLine($"{books.Count} books");
foreach (var b in books.Take(5)) Console.WriteLine($"{b.Title}: {b.Price}");

// dotnet add package AngleSharp
using AngleSharp.Html.Parser;

var html = await http.GetStringAsync("https://books.toscrape.com/");
var document = await new HtmlParser().ParseDocumentAsync(html);
foreach (var card in document.QuerySelectorAll("article.product_pod"))
{
    var title = card.QuerySelector("h3 a")?.GetAttribute("title");
    var price = card.QuerySelector(".price_color")?.TextContent.Trim();
    Console.WriteLine($"{title}: {price}");
}

// dotnet add package PuppeteerSharp
using PuppeteerSharp;

await new BrowserFetcher().DownloadAsync();                  // downloads a compatible browser once
await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true });
await using var page = await browser.NewPageAsync();

await page.GoToAsync("https://quotes.toscrape.com/js/", WaitUntilNavigation.DOMContentLoaded);
await page.WaitForSelectorAsync(".quote");
var quotes = await page.EvaluateExpressionAsync<string[]>(
    "[...document.querySelectorAll('.quote .text')].map(e => e.textContent)");
Console.WriteLine(string.Join("\n", quotes));

// gcc scrape.c -o scrape $(curl-config --cflags --libs) $(xml2-config --cflags --libs)
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
#include <libxml/HTMLparser.h>
#include <libxml/xpath.h>

struct buf { char *data; size_t len; };

static size_t on_data(char *ptr, size_t size, size_t nmemb, void *userdata) {
    struct buf *b = userdata;
    size_t n = size * nmemb;
    char *p = realloc(b->data, b->len + n + 1);
    if (!p) return 0;
    b->data = p;
    memcpy(b->data + b->len, ptr, n);
    b->len += n;
    b->data[b->len] = '\0';
    return n;
}

int main(void) {
    struct buf b = {0};
    CURL *curl = curl_easy_init();
    curl_easy_setopt(curl, CURLOPT_URL, "https://books.toscrape.com/");
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, on_data);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &b);
    curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; c-demo/1.0)");
    if (curl_easy_perform(curl) != CURLE_OK) return 1;
    curl_easy_cleanup(curl);

    htmlDocPtr doc = htmlReadMemory(b.data, (int)b.len, NULL, NULL, HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING);
    xmlXPathContextPtr ctx = xmlXPathNewContext(doc);
    xmlXPathObjectPtr res = xmlXPathEvalExpression((const xmlChar *)"//article//h3/a/@title", ctx);
    for (int i = 0; res->nodesetval && i < res->nodesetval->nodeNr; i++) {
        xmlChar *title = xmlNodeGetContent(res->nodesetval->nodeTab[i]);
        printf("%s\n", (char *)title);
        xmlFree(title);
    }
    xmlXPathFreeObject(res);
    xmlXPathFreeContext(ctx);
    xmlFreeDoc(doc);
    free(b.data);
    return 0;
}