Travel Review Scraper for Hotels, Restaurants and Attractions

import csv, hashlib, json, random, time
from datetime import datetime
import requests
from bs4 import BeautifulSoup

# Your list: place -> the URL of its first review page (placeholders: use pages you may collect)
PLACES = {
    "Example Hotel": "https://reviews.example.com/hotel/12345/example-hotel",
    "Other Hotel":   "https://reviews.example.com/hotel/67890/other-hotel",
}
PROXY = "http://<login>-country-us:<password>@gate.proxyfield.io:7000"  # new IP on every request
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", "Accept-Language": "en-US,en;q=0.9"}

def page_url(url, n):
    # Copy the site's own pattern from page 2 in your browser, e.g. "?page=2" or "?offset=10".
    return url if n == 1 else f"{url}?page={n}"

def fetch(url):
    for attempt in range(3):
        r = requests.get(url, proxies={"http": PROXY, "https": PROXY}, headers=HEADERS, timeout=30)
        if r.status_code == 200:
            return r.text
        time.sleep(10 * (attempt + 1))  # wait; the next try gets a new IP
    return None

def objects(html):
    """Every schema.org object in the page's application/ld+json blocks, nested ones included."""
    out = []
    for block in BeautifulSoup(html, "html.parser").select('script[type="application/ld+json"]'):
        try:
            queue = [json.loads(block.string or "")]
        except ValueError:
            continue
        for x in queue:  # the list grows while the loop walks it, so nested objects follow in page order
            if isinstance(x, list):
                queue.extend(x)
            elif isinstance(x, dict):
                out.append(x)
                queue.extend(v for v in x.values() if isinstance(v, (dict, list)))
    return out

def is_type(x, name):
    t = x.get("@type")
    return name in (t if isinstance(t, list) else [t])

def review_count(objs):
    for x in objs:
        agg = x.get("aggregateRating")
        if isinstance(agg, dict):
            count = str(agg.get("reviewCount") or agg.get("ratingCount") or "").replace(",", "")
            if count.isdigit():
                return int(count)
    return None

def iso(value):
    value = str(value or "")
    if len(value) >= 10 and value[4] == "-" and value[7] == "-":
        return value[:10]  # already ISO, with or without a time
    for fmt in ("%B %d, %Y", "%b %d, %Y", "%d %B %Y"):
        try:
            return datetime.strptime(value, fmt).strftime("%Y-%m-%d")
        except ValueError:
            pass
    return value  # unknown format: keep the raw text

def reviews(objs):
    for x in objs:
        if not is_type(x, "Review"):
            continue
        rating = x.get("reviewRating") if isinstance(x.get("reviewRating"), dict) else {}
        row = {"rating": rating.get("ratingValue", ""), "date": iso(x.get("datePublished")),
               "title": x.get("name", ""), "text": x.get("reviewBody", "")}
        rid = x.get("@id") or x.get("url") or hashlib.sha1(json.dumps(row, sort_keys=True).encode()).hexdigest()[:16]
        yield {"review_id": rid, **row}

with open("travel_reviews.csv", "w", newline="", encoding="utf-8") as f:
    out = csv.DictWriter(f, ["place", "review_id", "rating", "date", "title", "text"])
    out.writeheader()
    for place, url in PLACES.items():
        seen, n, total = set(), 1, None
        while True:
            html = fetch(page_url(url, n))
            if html is None:
                print(f"{place}: stopped at page {n}, page not fetched")
                break
            objs = objects(html)
            if n == 1:
                total = review_count(objs)
            rows = [r for r in reviews(objs) if r["review_id"] not in seen]
            if not rows:
                break  # no reviews, or only reviews we already have: last page passed
            for r in rows:
                seen.add(r["review_id"])
                out.writerow({"place": place, **r})
            n += 1
            time.sleep(random.uniform(3, 6))
        note = "" if total in (None, len(seen)) else "  <- mismatch: check the language filter and the stop point"
        print(f"{place}: {len(seen)} reviews collected, page states {total}{note}")

API = "https://api.proxyfield.io/v1/scrape"
KEY = "<api_key>"

def fetch(url):
    r = requests.post(API, headers={"Authorization": f"Bearer {KEY}"},
                      json={"url": url, "render": False, "country": "us", "output": "html"},
                      timeout=130)
    if r.status_code != 200 or r.headers.get("X-Target-Status") != "200":
        return None  # not delivered: not billed (a target's 404 or 410 is a billed result)
    return r.text