Bing Scraper: Bing Results Pages via API

{
  "search": {"keyword": "residential proxies", "market": "en-GB", "device": "desktop", "date": "2026-10-27"},
  "ads": [
    {"position": 1, "block": "top", "title": "…", "displayed_url": "…", "link": "https://www.bing.com/aclick?…"}
  ],
  "organic_results": [
    {"position": 1, "title": "…", "url": "https://…", "page": 1, "snippet": "…"},
    {"position": 11, "title": "…", "url": "https://…", "page": 2, "snippet": "…"}
  ],
  "related_searches": [
    {"position": 1, "query": "…"}
  ]
}

1 1 <title of result 1> https://<result 1 URL>
1 2 <title of result 2> https://<result 2 URL>
...
related: ['<related query 1>', '<related query 2>', '<related query 3>', ...]
2 11 <title of result 11> https://<result 11 URL>
2 12 <title of result 12> https://<result 12 URL>
...

import csv, json, time
import requests
from bs4 import BeautifulSoup

KEYWORD, MARKET, DEVICE = "residential proxies", "en-GB", "desktop"   # en-US, de-DE ...; or "mobile"
PROXY = "http://<login>-country-gb:<password>@gate.proxyfield.io:7000"  # exit IP in the same country
UA = {"desktop": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",           # the device is set by the
      "mobile": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X)"}  # User-Agent header alone
HEADERS = {"User-Agent": UA[DEVICE], "Accept-Language": "en-GB,en;q=0.9"}

organic, ads, related, seen = [], [], [], set()
for page in range(1, 6):   # top 50: five pages
    first = (page - 1) * 10 + 1   # 1, 11, 21, 31, 41
    params = {"q": KEYWORD, "first": first, "mkt": MARKET, "setlang": MARKET}   # mkt without cc
    resp = requests.get("https://www.bing.com/search", params=params, headers=HEADERS,
                        proxies={"http": PROXY, "https": PROXY}, timeout=30)
    soup = BeautifulSoup(resp.text, "html.parser")
    # Bing changes its markup: check the selectors before you rely on them
    items = soup.select("li.b_algo")
    if not items:   # a challenge page, or the end of the results: never store it as "no results"
        if page == 1:
            raise SystemExit(f"no organic results on page 1 (status {resp.status_code}): nothing stored")
        break
    for item in items:
        link = item.select_one("h2 a")
        if not link or link.get("href") in seen:
            continue
        seen.add(link.get("href"))
        snippet = item.select_one(".b_caption p") or item.select_one("p")
        organic.append({"position": len(organic) + 1, "title": link.get_text(strip=True),
                        "url": link.get("href"), "page": page,
                        "snippet": snippet.get_text(" ", strip=True) if snippet else ""})
    if page == 1:   # ads (top and bottom blocks) and related searches, from the first page
        for block in soup.select("li.b_ad"):
            where = "bottom" if "b_adBottom" in block.get("class", []) else "top"
            for ad in block.select("h2 a"):
                cite = ad.find_parent("li").select_one("cite") if ad.find_parent("li") else None
                ads.append({"position": len(ads) + 1, "block": where, "title": ad.get_text(strip=True),
                            "displayed_url": cite.get_text(strip=True) if cite else "", "link": ad.get("href")})
        related = [{"position": i, "query": a.get_text(strip=True)}
                   for i, a in enumerate(soup.select(".b_rs a"), start=1)]
        if not related:   # not every plain-HTTP page carries them
            print("no related searches in this page: re-fetch it with JS rendering and recheck the selector")
    time.sleep(5)

meta = {"keyword": KEYWORD, "market": MARKET, "device": DEVICE, "date": time.strftime("%Y-%m-%d")}
with open("bing.json", "w") as f:
    json.dump({"search": meta, "ads": ads, "organic_results": organic, "related_searches": related}, f, indent=2)
with open("bing.csv", "w", newline="") as f:
    writer = csv.DictWriter(f, fieldnames=[*meta, "position", "title", "url", "snippet", "page"])
    writer.writeheader()
    writer.writerows({**meta, **row} for row in organic)
print(len(organic), "results,", len(ads), "ads,", len(related), "related searches")

50 results, 2 ads, 8 related searches