import html
import json
import os
from datetime import datetime, timezone
from pathlib import Path
import requests
api_key = os.environ.get("YOUTUBE_API_KEY")
if not api_key:
raise SystemExit("Set YOUTUBE_API_KEY before running this example")
query = os.environ.get("YOUTUBE_QUERY", "python data pipelines")
filters = {
"part": "snippet", "type": "video", "maxResults": 25,
"regionCode": "US", "relevanceLanguage": "en",
"order": "relevance", "safeSearch": "moderate",
}
records, seen_ids, seen_tokens = [], set(), set()
page_token = None
position = 0
stop_reason = "page_limit"
with requests.Session() as session:
for page_number in range(1, 3):
params = {**filters, "q": query, "key": api_key}
if page_token:
params["pageToken"] = page_token
try:
response = session.get(
"https://www.googleapis.com/youtube/v3/search",
params=params, timeout=(10, 30),
)
payload = response.json()
except (requests.RequestException, ValueError):
raise SystemExit("Search request failed; inspect transport safely") from None
if response.status_code != 200 or "error" in payload:
reasons = payload.get("error", {}).get("errors", [])
reason = reasons[0].get("reason", "unknown") if reasons else "unknown"
raise SystemExit(
f"YouTube API error: HTTP {response.status_code}, reason={reason}"
)
items = payload.get("items")
if not isinstance(items, list):
raise SystemExit("Unexpected search response: items is not a list")
observed_at = datetime.now(timezone.utc).isoformat()
for item in items:
position += 1
video_id = item.get("id", {}).get("videoId")
snippet = item.get("snippet", {})
if not video_id or not snippet.get("title"):
raise SystemExit("Unexpected video search item")
if video_id in seen_ids:
continue
seen_ids.add(video_id)
records.append({
"video_id": video_id,
"url": "https://www.youtube.com/watch?v=" + video_id,
"title": html.unescape(snippet["title"]),
"channel_id": snippet.get("channelId"),
"channel_title": snippet.get("channelTitle"),
"published_at": snippet.get("publishedAt"),
"observed_at": observed_at,
"api_position": position,
"page_number": page_number,
})
next_token = payload.get("nextPageToken")
if not next_token:
stop_reason = "end_of_results"
break
if next_token in seen_tokens:
raise SystemExit("Repeated pagination token; refusing a loop")
seen_tokens.add(next_token)
page_token = next_token
result = {
"query": query, "filters": filters,
"raw_items_seen": position, "accepted_unique_videos": len(records),
"stop_reason": stop_reason, "items": records,
}
Path("youtube-search.json.tmp").write_text(
json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8"
)
os.replace("youtube-search.json.tmp", "youtube-search.json")
print(f"Saved {len(records)} unique videos; stop_reason={stop_reason}")
curl --fail-with-body --request POST \
'https://api.nstdata.io/api/v1/crawl/scrape/submit-sync' \
--header "x-api-key: ${NSTDATA_API_KEY:?Set NSTDATA_API_KEY}" \
--header 'Content-Type: application/json' \
--data '{"url":"https://www.python.org/doc/","formats":["markdown"],"onlyMainContent":true,"timeout":60000}'