# Real Narrative News — crawler policy
#
# Goal: point Googlebot's crawl budget at content pages (/, categories,
# /read, /editorial, /source, /where, /keyword-reader, /topic-reader,
# /happened-today, /hot-now, /bias-analysis, /propaganda-analysis) and
# keep it out of authenticated routes, internal APIs, and Next.js
# build artefacts that were burning 69 % of the budget on "Page
# resource load" without any indexing value.

User-agent: *

# Authenticated / user-only routes — Google was hitting these and
# getting 401/407, which accounted for ~10 % of crawl responses and
# signalled to Google that the site is unhealthy.
Disallow: /login
Disallow: /join
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /bookmarks
Disallow: /preferences
Disallow: /onboard

# Internal API — reverse-proxied to microservices. Never crawlable.
Disallow: /api/

# Next.js server actions & internal endpoints.
Disallow: /_next/data/
Disallow: /_vercel/

# Error / compact routes — noindex-eligible but no reason to crawl.
Disallow: /403
Disallow: /429
Disallow: /500

# Static build assets & images are fine to fetch (Google needs them to
# render pages) but explicitly allow them here so subsequent Disallow
# lines never accidentally cover them.
Allow: /_next/static/
Allow: /favicon/
Allow: /logo/
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.css$
Allow: /*.js$

# Sitemaps — main static sitemap plus the dedicated editorial sitemap
# so Google can crawl editorials on their own schedule.
Sitemap: https://realnarrativenews.com/sitemap.xml
Sitemap: https://realnarrativenews.com/sitemap-editorials.xml
Sitemap: https://realnarrativenews.com/sitemap-keywords.xml
Sitemap: https://realnarrativenews.com/sitemap-sources.xml