CREATE EXTERNAL STREAM hn_latest
(
    id uint64,
    type string,
    by string,
    title string,
    url string,
    received_at datetime64(3),
    raw_json string
)
AS $$
import json
import time
import urllib.request
from datetime import datetime, timezone

MAX_RETRIES = 3
BACKOFF_SEC = 0.5
LOOKBACK = 50
BATCH_SIZE = 20
POLL_SEC = 2.0
def get_json(url, timeout=5):
    last_err = None
    for attempt in range(MAX_RETRIES):
        try:
            with urllib.request.urlopen(url, timeout=timeout) as resp:
                return json.loads(resp.read().decode("utf-8"))
        except Exception as e:
            last_err = e
            time.sleep(BACKOFF_SEC * (2 ** attempt))
    raise last_err
def read_hn_latest():
    base = "https://hacker-news.firebaseio.com/v0"
    next_id = 0
    while True:
        try:
            max_id = int(get_json(f"{base}/maxitem.json"))
            if next_id == 0:
                next_id = max(0, max_id - LOOKBACK)
            end_id = min(max_id, next_id + BATCH_SIZE)
            rows = []
            for item_id in range(next_id + 1, end_id + 1):
                item = get_json(f"{base}/item/{item_id}.json") or {}
                rows.append((
                    int(item.get("id") or item_id),
                    str(item.get("type") or ""),
                    str(item.get("by") or ""),
                    str(item.get("title") or ""),
                    str(item.get("url") or ""),
                    datetime.now(timezone.utc),
                    json.dumps(item, ensure_ascii=False),
                ))
            next_id = end_id
            if rows:
                yield rows
            else:
                time.sleep(POLL_SEC)
        except Exception as e:
            # Keep the stream alive; surface error as a synthetic row
            yield [(0, "error", "", str(e), "", datetime.now(timezone.utc), json.dumps({"error": str(e)}))]
            time.sleep(POLL_SEC)
$$
SETTINGS type='python', mode='streaming', read_function_name='read_hn_latest';

SELECT * FROM hn_latest LIMIT 10;