Google Flights Scraper: Daily Fares for a List of Routes

# daily_fares.py: the request the Web Scraping API takes (access through Contact sales)
import csv, pathlib, re, requests
from datetime import datetime, timezone
from urllib.parse import quote

API_KEY = "<api_key>"
MARKET = "us"  # point of sale: exit IP country and gl
ROUTES = [("JFK", "LHR", "2026-12-10"), ("LAX", "NRT", "2026-12-15"), ("ORD", "CDG", "2027-01-08")]
ROW = '[aria-label^="From "]'  # one flight row; check on a live page
LABEL = re.compile(r"From ([\d,]+) US dollars.*?(Nonstop|\d+ stops?) flight with ([^.]+)\.")

now = datetime.now(timezone.utc)
pages = pathlib.Path("pages", now.strftime("%Y-%m-%d"))
pages.mkdir(parents=True, exist_ok=True)

with open("fares.csv", "a", newline="") as f:
    out = csv.writer(f)
    for origin, dest, day in ROUTES:
        trip = f"Flights to {dest} from {origin} on {day} oneway"
        url = f"https://www.google.com/travel/flights?q={quote(trip)}&hl=en&gl={MARKET}&curr=USD"
        r = requests.post("https://api.proxyfield.io/v1/scrape",
                          headers={"Authorization": f"Bearer {API_KEY}"},
                          json={"url": url, "render": True, "country": MARKET, "output": "json",
                                "actions": [
                                    {"type": "wait_for", "selector": ROW, "timeout_ms": 30000},
                                    {"type": "evaluate", "script":
                                     f"Array.from(document.querySelectorAll('{ROW}'), e => e.getAttribute('aria-label'))"}]},
                          timeout=180)
        if not r.ok:  # error responses are not billed
            out.writerow([now.isoformat(timespec="seconds"), origin, dest, day, "", "", "", "", MARKET, 0, "error " + str(r.status_code)])
            continue
        data = r.json()
        (pages / f"{origin}-{dest}-{day}.html").write_text(data["content"] or "")
        labels = data["actions_log"][-1].get("result") or []
        flights = [(int(m[1].replace(",", "")), m[3].strip(), m[2]) for m in map(LABEL.search, labels) if m]
        fare, airline, stops = min(flights) if flights else ("", "", "")
        out.writerow([now.isoformat(timespec="seconds"), origin, dest, day, fare, "USD", airline, stops,
                      MARKET, len(flights), data["metadata"]["billed"]])

0 6 * * * cd /opt/fares && /usr/bin/python3 daily_fares.py

fares.csv:  2026-12-01T06:00:41+00:00,JFK,LHR,2026-12-10,512,USD,Example Air,Nonstop,us,14,True
pages/2026-12-01/JFK-LHR-2026-12-10.html

# reparse.py: read the fares again from saved pages
import html, pathlib, re, sys

LABEL = re.compile(r"From ([\d,]+) US dollars.*?(Nonstop|\d+ stops?) flight with ([^.]+)\.")
for page in sorted(pathlib.Path("pages", sys.argv[1]).glob("*.html")):
    labels = [html.unescape(a) for a in re.findall(r'aria-label="(From [^"]+)"', page.read_text())]
    flights = [(int(m[1].replace(",", "")), m[3].strip(), m[2]) for m in map(LABEL.search, labels) if m]
    print(page.stem, min(flights) if flights else "no flights read: adjust LABEL")