YouTube Scraper: Public YouTube Pages via API

# YT_API_KEY=<your key> python3 channels_api.py
import csv, os, requests

API, KEY, units = "https://www.googleapis.com/youtube/v3/", os.environ["YT_API_KEY"], 0

def call(method, **params):
    global units
    units += 1                     # channels, playlistItems and videos .list cost 1 unit each
    r = requests.get(API + method, params={**params, "key": KEY}, timeout=30)
    r.raise_for_status()
    return r.json()

with open("videos.csv", "w", newline="", encoding="utf-8") as f:
    w = csv.writer(f)
    w.writerow(["channel", "video_id", "published", "views", "likes", "comments", "title"])
    for url in [l.strip() for l in open("channels.txt") if l.strip()]:
        handle = url.rstrip("/").rsplit("/", 1)[-1]                 # "@YouTube"
        found = call("channels", part="contentDetails", forHandle=handle).get("items") or []
        if not found:
            print("not found:", url)
            continue
        uploads = found[0]["contentDetails"]["relatedPlaylists"]["uploads"]
        ids, token = [], None
        while True:
            page = call("playlistItems", part="contentDetails", playlistId=uploads, maxResults=50,
                        **({"pageToken": token} if token else {}))
            ids += [i["contentDetails"]["videoId"] for i in page["items"]]
            token = page.get("nextPageToken")
            if not token:
                break
        for i in range(0, len(ids), 50):
            for v in call("videos", part="snippet,statistics", id=",".join(ids[i:i + 50]))["items"]:
                s = v["statistics"]
                w.writerow([url, v["id"], v["snippet"]["publishedAt"], s.get("viewCount"),
                            s.get("likeCount"), s.get("commentCount"), v["snippet"]["title"]])
print("quota units used:", units)  # about 21 for a 500-video channel

PROXY="http://<login>-country-us:<password>@gate.proxyfield.io:7000"
# channels.txt: one channel URL per line, e.g. https://www.youtube.com/@YouTube
while read -r url; do
  yt-dlp --proxy "$PROXY" --flat-playlist \
    --print "$url %(id)s %(view_count)s %(title)s" "$url/videos" >> videos.txt
done < channels.txt

import csv
from yt_dlp import YoutubeDL

opts = {"proxy": "http://<login>-country-us:<password>@gate.proxyfield.io:7000",
        "extract_flat": True, "quiet": True}
channels = [line.strip() for line in open("channels.txt") if line.strip()]
with YoutubeDL(opts) as ydl, open("videos.csv", "w", newline="", encoding="utf-8") as f:
    w = csv.writer(f)
    w.writerow(["channel", "video_id", "views", "title"])
    for url in channels:
        tab = ydl.extract_info(url.rstrip("/") + "/videos", download=False)
        for v in tab.get("entries") or []:
            w.writerow([url, v.get("id"), v.get("view_count"), v.get("title")])  # views can be empty in flat mode

channel,video_id,views,title
https://www.youtube.com/@YouTube,VIDEO_ID_1,1834467,First video title
https://www.youtube.com/@YouTube,VIDEO_ID_2,,Live stream title

# run once a day; appends only videos not seen before to new-videos.csv
import csv, datetime, os
from yt_dlp import YoutubeDL

opts = {"proxy": "http://<login>-country-us:<password>@gate.proxyfield.io:7000",
        "extract_flat": True, "playlist_items": "1:30", "quiet": True}
seen = set(open("seen.txt").read().split()) if os.path.exists("seen.txt") else set()
today = datetime.date.today().isoformat()
with YoutubeDL(opts) as ydl, open("new-videos.csv", "a", newline="", encoding="utf-8") as f, \
        open("seen.txt", "a") as log:
    w = csv.writer(f)
    for url in [l.strip() for l in open("channels.txt") if l.strip()]:
        for v in ydl.extract_info(url.rstrip("/") + "/videos", download=False).get("entries") or []:
            if v["id"] not in seen:
                seen.add(v["id"])
                log.write(v["id"] + "\n")
                w.writerow([today, url, v["id"], v.get("view_count"), v.get("title")])