Asynchronous Web Scraping in Python

import asyncio

async def fetch(n):
    await asyncio.sleep(1)          # stands in for a network request
    return n

async def main():
    return await asyncio.gather(*(fetch(n) for n in range(10)))

print(asyncio.run(main()))          # about 1 second, not 10

pip install aiohttp beautifulsoup4 lxml

import asyncio

import aiohttp
from bs4 import BeautifulSoup

BASE = "https://www.scrapethissite.com/pages/forms/?page_num={}"
CONCURRENCY = 4


def parse(html):
    soup = BeautifulSoup(html, "lxml")
    return [td.get_text(strip=True) for td in soup.select("tr.team td.name")]


async def fetch(session, sem, url):
    async with sem:                                   # at most CONCURRENCY requests at once
        async with session.get(url) as resp:
            resp.raise_for_status()
            return await resp.text()


async def main(pages):
    sem = asyncio.Semaphore(CONCURRENCY)
    timeout = aiohttp.ClientTimeout(total=30)
    headers = {"User-Agent": "Mozilla/5.0"}
    async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
        tasks = [fetch(session, sem, BASE.format(n)) for n in range(1, pages + 1)]
        htmls = await asyncio.gather(*tasks)
    return [name for html in htmls for name in parse(html)]


teams = asyncio.run(main(8))
print(len(teams), "teams")          # 200 teams (8 pages of 25)

import asyncio

import httpx
from bs4 import BeautifulSoup

BASE = "https://www.scrapethissite.com/pages/forms/?page_num={}"


async def fetch(client, sem, url):
    async with sem:
        r = await client.get(url)
        r.raise_for_status()
        return r.text


async def main(pages):
    sem = asyncio.Semaphore(4)
    limits = httpx.Limits(max_connections=4)
    async with httpx.AsyncClient(timeout=30, limits=limits, headers={"User-Agent": "Mozilla/5.0"}) as client:
        async with asyncio.TaskGroup() as tg:
            tasks = [tg.create_task(fetch(client, sem, BASE.format(n))) for n in range(1, pages + 1)]
    names = []
    for t in tasks:
        names += [td.get_text(strip=True) for td in BeautifulSoup(t.result(), "lxml").select("tr.team td.name")]
    return names


print(len(asyncio.run(main(8))), "teams")

import random

async def fetch_with_retry(session, sem, url, tries=4):
    for attempt in range(tries):
        try:
            async with sem, session.get(url) as resp:
                if resp.status in (429, 500, 502, 503, 504):
                    raise aiohttp.ClientResponseError(resp.request_info, resp.history, status=resp.status)
                resp.raise_for_status()
                return await resp.text()
        except (aiohttp.ClientError, asyncio.TimeoutError):
            if attempt == tries - 1:
                raise
            await asyncio.sleep(2 ** attempt + random.random())   # 1, 2, 4 s plus jitter

# sketch: define find_links() and parse() first; this block does not run as printed
async def worker(session, queue, seen, results):
    while True:
        url = await queue.get()
        try:
            async with session.get(url) as resp:
                html = await resp.text()
            results.append(parse(html))
            for link in find_links(html, url):          # your own link extractor
                if link not in seen:
                    seen.add(link)
                    queue.put_nowait(link)
        finally:
            queue.task_done()

async def crawl(start, n_workers=5):
    queue, seen, results = asyncio.Queue(), {start}, []
    queue.put_nowait(start)
    async with aiohttp.ClientSession() as session:
        workers = [asyncio.create_task(worker(session, queue, seen, results)) for _ in range(n_workers)]
        await queue.join()                              # wait until every queued URL is done
        for w in workers:
            w.cancel()
    return results

rows = await asyncio.to_thread(parse, html)     # runs parse() in a worker thread

from playwright.async_api import async_playwright

async def render(browser, sem, url):
    async with sem:
        page = await browser.new_page()
        try:
            await page.goto(url)
            return await page.content()
        finally:
            await page.close()

async def main(urls):
    sem = asyncio.Semaphore(3)                       # browser pages are heavy: keep this low
    async with async_playwright() as p:
        browser = await p.chromium.launch()
        return await asyncio.gather(*(render(browser, sem, u) for u in urls))