# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
User-agent: *
Allow: /
# Private / authenticated surfaces — not for indexing.
Disallow: /app
Disallow: /admin
Disallow: /developers/dashboard
Disallow: /checkout
Disallow: /f/

# AUTH FORMS LINKED FROM 1.4M PAGES (2026-10-15, from production logs). Every public bill page
# links to /signup with `return_to` and `bill_ids`, and the gated search page links to /signup and
# /login with `return_to: request.fullpath` — so the corpus advertises a DISTINCT auth URL per
# bill and per filter combination, each one the same form. Crawlers were walking them.
#
# ONLY THE QUERY-STRING VARIANTS ARE BLOCKED. Bare /signup and /login stay crawlable ON PURPOSE:
# they carry `X-Robots-Tag: noindex` (AuthPagesNotIndexable), and a crawler has to be allowed to
# FETCH a page to see that header. Disallowing them outright would hide the noindex and freeze
# whatever is already indexed instead of removing it.
Disallow: /signup?
Disallow: /login?

# FILTER COMBINATIONS (2026-10-21, from production logs). Every public bill list links to itself
# with each state added and each page number, so the state × page space is effectively infinite —
# a crawl was walking "states=9,18,20,24,32,46&page=3" at hundreds of requests a second. The
# unfiltered /bills and every /bills/:id page stay crawlable; the filtered views also say noindex.
Disallow: /bills?*states=
Disallow: /bills?*fed=
Disallow: /bills?*page=

Sitemap: https://www.oneclickpolitics.com/sitemap.xml