Indeed Scraper: Daily Job Listings from Every Indeed Site

# indeed_daily.py: yesterday's new jobs for one search -> jobs.db and new_jobs_<date>.jsonl
import json, random, re, secrets, sqlite3, time
from datetime import date
from urllib.parse import quote_plus
from urllib.robotparser import RobotFileParser
from playwright.sync_api import sync_playwright

COUNTRY = "us"            # one key per Indeed site: "gb", "ca", "de" ...
SITES = {"us": ("www.indeed.com", "en-US"), "gb": ("uk.indeed.com", "en-GB"),
         "ca": ("ca.indeed.com", "en-CA"), "de": ("de.indeed.com", "de-DE"),
         "fr": ("fr.indeed.com", "fr-FR"), "au": ("au.indeed.com", "en-AU"),
         "es": ("es.indeed.com", "es-ES"), "it": ("it.indeed.com", "it-IT"),
         "nl": ("nl.indeed.com", "nl-NL")}
SITE, LOCALE = SITES[COUNTRY]       # the Indeed domain and the browser locale
SESSION = secrets.token_hex(4)      # one sticky IP for the whole run
PROXY = {"server": "http://gate.proxyfield.io:7000",   # an IP in the same country, kept 30 min:
         # e.g. <login>-country-us-session-3f9a1c2e-ttl-1800 (add -city-new_york for one city)
         "username": f"<login>-country-{COUNTRY}-session-{SESSION}-ttl-1800",
         "password": "<password>"}
QUERY, PLACE = "data analyst", "Chicago, IL"           # "London", "Berlin" ...
SEARCH = (f"https://{SITE}/jobs?q={quote_plus(QUERY)}&l={quote_plus(PLACE)}"
          "&sort=date&fromage=1&start={start}")
MAX_PAGES = 10            # this script's own cap: start=0 to 90; split a search that needs more

def parse_pay(text):
    """'$70,000 - $85,000 a year' -> (70000.0, 85000.0, '$', 'year'); NULLs when no pay is shown.
    Reads US and UK number formats; adapt it for sites that write 45.000 EUR or 'pro Jahr'."""
    cur = re.search(r"[$£€]", text or "")
    nums = [float(n.replace(",", "")) for n in re.findall(r"\d[\d,]*(?:\.\d+)?", text or "")]
    if not cur or not nums:   # no currency sign: a job type such as "Full-time", not pay
        return None, None, None, None
    per = next((w for w in ("hour", "day", "week", "month", "year") if w in text.lower()), None)
    return min(nums), max(nums), cur.group(0), per

def text(card, selector):
    el = card.query_selector(selector)
    return el.inner_text().strip() if el else None

robots = RobotFileParser(f"https://{SITE}/robots.txt")
robots.read()
db = sqlite3.connect("jobs.db")
db.row_factory = sqlite3.Row
db.execute("""CREATE TABLE IF NOT EXISTS jobs (site TEXT, jk TEXT, title TEXT, company TEXT,
    location TEXT, salary_text TEXT, salary_min REAL, salary_max REAL, currency TEXT,
    period TEXT, first_seen TEXT, last_seen TEXT, PRIMARY KEY (site, jk))""")
today = date.today().isoformat()

with sync_playwright() as p:
    browser = p.chromium.launch(proxy=PROXY)
    page = browser.new_page(locale=LOCALE)
    page.route("**/*", lambda route: route.abort()        # images, media and fonts are traffic you pay for
               if route.request.resource_type in ("image", "media", "font") else route.continue_())
    for start in range(0, MAX_PAGES * 10, 10):
        url = SEARCH.format(start=start)
        if not robots.can_fetch("*", url):
            print(f"robots.txt disallows {url}: stop")
            break
        resp = page.goto(url, wait_until="domcontentloaded", timeout=60000)
        if resp is None or resp.status in (403, 429) or "Security Check" in page.title():
            print(f"refused at start={start}: stop, retry this search later at a slower pace")
            break
        # Indeed changes its markup: check these selectors on a live page first
        cards = page.query_selector_all("div.job_seen_beacon")
        if not cards:
            break             # end of the results, or the markup changed
        for card in cards:
            link = card.query_selector("a[data-jk]")
            if not link:
                continue
            pay = text(card, "[data-testid='attribute_snippet_testid']")  # pay when shown; may hold the job type
            db.execute("""INSERT INTO jobs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                ON CONFLICT(site, jk) DO UPDATE SET last_seen = excluded.last_seen""",
                (SITE, link.get_attribute("data-jk"), link.inner_text().strip(),
                 text(card, "[data-testid='company-name']"),
                 text(card, "[data-testid='text-location']"),
                 pay, *parse_pay(pay), today, today))
        db.commit()
        time.sleep(random.uniform(5, 10))
    browser.close()

with open(f"new_jobs_{today}.jsonl", "w") as out:   # today's new jobs, one JSON object per line
    for row in db.execute("SELECT * FROM jobs WHERE site = ? AND first_seen = ?", (SITE, today)):
        out.write(json.dumps(dict(row)) + "\n")

sqlite> SELECT jk, title, company, location, salary_text, salary_min, salary_max, period
   ...> FROM jobs WHERE first_seen = date('now') ORDER BY salary_max DESC;