User-agent: *
Allow: /

# ── Verified SEO crawlers (explicit allow; prevents edge WAF misclassification) ──
User-agent: SemrushBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: AhrefsSiteAudit
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

User-agent: rogerbot
Allow: /

User-agent: BLEXBot
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /

User-agent: SiteAuditBot
Allow: /

User-agent: DataForSeoBot
Allow: /

User-agent: Mediatoolkitbot
Allow: /


# ── Public SEO landing pages (explicit allows for clarity) ──
Allow: /username-search
Allow: /reverse-image-search
Allow: /reverse-username-search
Allow: /find-dating-profiles
Allow: /email-breach-check
Allow: /phone-lookup
Allow: /profile-picture-search
Allow: /find-people-online
Allow: /face-intelligence
Allow: /scan-my-online-presence
Allow: /search-engines-to-find-people
Allow: /compare/
Allow: /developers
Allow: /intelligence-graph
Allow: /account-finder
Allow: /exposure-intelligence
Allow: /safe-osint-for-ai
Allow: /learn
Allow: /learn/

# ── Public dynamic SEO prefixes (curated; thin variants noindex via X-Robots-Tag) ──
Allow: /risk/
Allow: /for/
Allow: /atlas/

# /username/<slug> entity pages — all canonical to /username-search and noindex.
# Disallowed (was Allow) on 2026-06-26: Google had 330k of these stuck in
# "Crawled - currently not indexed" because Allow + noindex created a recrawl
# loop. They're already de-indexed; blocking crawl drops them from the report
# over ~4-6 weeks and stops dragging whole-site quality signals. Internal links
# are wrapped in <CrawlSafeLink> so they render as <button> (no <a href>).
# Keep /username-search, /username-directory, /usernames crawlable (different paths).
Disallow: /username/

# Block query-string spam variants under those prefixes
Disallow: /risk/*?
Disallow: /for/*?
Disallow: /atlas/*?

# ── Global tracking / spam query-string blocks ──
Disallow: /*?utm_
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?session=
Disallow: /*?token=
Disallow: /*?preview=
Disallow: /*?debug=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=

# ── Cache-buster spam variants (?_r=<timestamp>&_cb=<random>) ──
# 637 of these were indexed by Bing as duplicate pages (6 Sep 2026). A
# Cloudflare Redirect Rule also 301s them to the clean path.
Disallow: /*?_r=
Disallow: /*&_r=
Disallow: /*?_cb=
Disallow: /*&_cb=

# ── User-specific / authenticated pages ──
Disallow: /dashboard
Disallow: /admin
Disallow: /settings
Disallow: /auth
Disallow: /account
Disallow: /onboarding
Disallow: /preferences
Disallow: /workspace
Disallow: /workspaces
Disallow: /workspace-management
Disallow: /r/
Disallow: /organization
Disallow: /teams

# ── Dynamic prefixes WITHOUT public canonicals (kept fully blocked) ──
Disallow: /people/
Disallow: /identity/
Disallow: /email-intelligence/

# ── Username Directory pagination (page 1 only is indexable) ──
# Stops Semrush/Ahrefs from following grid pagination into 30k+ /username/<u>
# entity URLs that all canonical to /username-search.
Disallow: /username-directory/page/
Disallow: /username-directory/*/page/

# ── Query-string spam on lookup/scan pages ──
Disallow: /free-scan?
Disallow: /scan?
Disallow: /username-search?
Disallow: /reverse-username-search?
Disallow: /find-dating-profiles?
Disallow: /email-breach-check?
Disallow: /phone-lookup?

# ── Scan results & internal data ──
Disallow: /scan/usernames/
Disallow: /scan/username/
Disallow: /scan/emails/
Disallow: /scan/phones/
Disallow: /scan-results
Disallow: /results/
Disallow: /result
Disallow: /report/
Disallow: /exposure/

# ── Investigation tools (auth-gated) ──
Disallow: /analyst
Disallow: /ai-analyst
Disallow: /assistant
Disallow: /cases
Disallow: /graph
Disallow: /workflows
Disallow: /monitoring
Disallow: /watchlists
Disallow: /anomaly-history
Disallow: /intelligence/
Disallow: /ops
Disallow: /persona-resolver
Disallow: /threat-forecast
Disallow: /threat-intel
Disallow: /predictive-analytics
Disallow: /context-enrichment

# ── Billing & commerce ──
Disallow: /subscription
Disallow: /billing
Disallow: /buy-credits
Disallow: /checkout

# ── Reports & exports ──
Disallow: /reports
Disallow: /my-tickets
Disallow: /evidence-pack
Disallow: /share/

# ── Embeds & widgets ──
Disallow: /embed
Disallow: /embed-widget
Disallow: /embed/widget

# ── API & developer (internal tools only) ──
Disallow: /api-keys
Disallow: /api-docs
Disallow: /agents
Disallow: /automation

# ── Client management ──
Disallow: /clients/

# ── Internal / debug ──
Disallow: /seo-debug
Disallow: /seo/
Disallow: /maigret/results/
Disallow: /maigret/debug
Disallow: /maigret/self-test
Disallow: /404
Disallow: /insights

# ── Misc auth-gated ──
Disallow: /referrals
Disallow: /timeline
Disallow: /analytics/ai
Disallow: /analytics/scans
Disallow: /analytics/executive
Disallow: /analytics/advanced
Disallow: /sovereignty
Disallow: /compliance
Disallow: /dark-web-monitoring
Disallow: /security
Disallow: /executive-dashboard
Disallow: /executive
Disallow: /marketplace-submit
Disallow: /help-center

Sitemap: https://footprintiq.app/sitemap.xml

# ── Googlebot ──
User-agent: Googlebot
Allow: /

# ── Bingbot — respect crawl budget warning ──
User-agent: Bingbot
Crawl-delay: 1
Allow: /
Allow: /risk/
Allow: /for/
Allow: /atlas/
Disallow: /username/
Disallow: /risk/*?
Disallow: /for/*?
Disallow: /atlas/*?
Disallow: /*?utm_
Disallow: /*?session=
Disallow: /*?token=
Disallow: /*?preview=
Disallow: /*?debug=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?_r=
Disallow: /*&_r=
Disallow: /*?_cb=
Disallow: /*&_cb=
# Query-driven search result variants — thin/duplicate content.
# Match anywhere in the path so both `/path?q=` and `/path?foo=&q=` are caught.
Disallow: /*?q=
Disallow: /*?search=
Disallow: /*&q=
Disallow: /*&search=
Disallow: /dashboard
Disallow: /admin
Disallow: /settings
Disallow: /auth
Disallow: /account
Disallow: /onboarding
Disallow: /workspace
Disallow: /organization
Disallow: /teams
Disallow: /people/
Disallow: /identity/
Disallow: /email-intelligence/
Disallow: /free-scan?
Disallow: /scan?
Disallow: /scan/usernames/
Disallow: /scan/username/
Disallow: /scan/emails/
Disallow: /scan/phones/
Disallow: /scan-results
Disallow: /results/
Disallow: /result
Disallow: /report/
Disallow: /exposure/
Disallow: /analyst
Disallow: /ai-analyst
Disallow: /assistant
Disallow: /cases
Disallow: /graph
Disallow: /workflows
Disallow: /monitoring
Disallow: /watchlists
Disallow: /intelligence/
Disallow: /ops
Disallow: /subscription
Disallow: /billing
Disallow: /buy-credits
Disallow: /checkout
Disallow: /reports
Disallow: /my-tickets
Disallow: /evidence-pack
Disallow: /share/
Disallow: /embed
Disallow: /api-keys
Disallow: /api-docs
Disallow: /agents
Disallow: /automation
Disallow: /clients/
Disallow: /seo-debug
Disallow: /seo/
Disallow: /maigret/
Disallow: /404
Disallow: /insights
Disallow: /referrals
Disallow: /timeline
Disallow: /analytics/
Disallow: /sovereignty
Disallow: /compliance
Disallow: /dark-web-monitoring
Disallow: /security
Disallow: /executive
Disallow: /marketplace-submit
Disallow: /help-center
Disallow: /preferences
Disallow: /username-search?
Disallow: /reverse-username-search?
Disallow: /find-dating-profiles?
Disallow: /email-breach-check?
Disallow: /phone-lookup?
Disallow: /anomaly-history
Disallow: /persona-resolver
Disallow: /threat-forecast
Disallow: /threat-intel
Disallow: /predictive-analytics
Disallow: /context-enrichment

User-agent: Slurp
Allow: /

# Social media crawlers for link previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# AI and LLM discovery — explicit policy (rev 2026-08-29)
# Assistants and their crawlers may read all public content (guides,
# comparisons, llms.txt/llms-full.txt). Private and auth surfaces stay
# blocked. Groups below are explicit: Allow-only groups are ambiguous
# and skip the * group's protections entirely.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: CCBot
User-agent: meta-externalagent
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /settings
Disallow: /auth
Disallow: /account
Disallow: /onboarding
Disallow: /preferences
Disallow: /workspace
Disallow: /workspaces
Disallow: /workspace-management
Disallow: /organization
Disallow: /teams
Disallow: /people/
Disallow: /identity/
Disallow: /email-intelligence/
Disallow: /username/
Disallow: /scan/usernames/
Disallow: /scan/username/
Disallow: /scan/emails/
Disallow: /scan/phones/
Disallow: /scan-results
Disallow: /results/
Disallow: /result
Disallow: /report/
Disallow: /exposure/
Disallow: /subscription
Disallow: /billing
Disallow: /buy-credits
Disallow: /checkout
Disallow: /reports
Disallow: /my-tickets
Disallow: /evidence-pack
Disallow: /share/
Disallow: /r/
Disallow: /seo-debug
Disallow: /seo/