# BidSmith Robots.txt

# Disallow legacy authentication subdomains & endpoints
User-agent: *
Disallow: /__clerk/
Disallow: /clerk/
Disallow: /api/

# Training and SEO audit crawlers do not need to enumerate the data-backed directories.
User-agent: Amazonbot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Disallow: /api/

User-agent: SemrushBot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Disallow: /api/

# Amazon's search crawler ships a distinct token from Amazonbot.
User-agent: Amzn-SearchBot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Disallow: /api/

# SEO backlink/audit crawlers index the directory for their own link graphs
# without sending us traffic.
User-agent: IbouBot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Disallow: /api/

User-agent: DotBot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Disallow: /api/

# Applebot honors Crawl-delay. Throttle it hard while still
# letting Apple index the catalog over time for Siri/Spotlight/Apple Intelligence.
User-agent: Applebot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Crawl-delay: 30

# Googlebot, Googlebot-Image, and Googlebot-Favicon explicit access rules
User-agent: Googlebot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /favicon.ico
Allow: /favicon-16x16.png
Allow: /favicon-32x32.png
Allow: /favicon-48x48.png
Allow: /favicon-96x96.png
Allow: /favicon-192x192.png
Allow: /favicon-512x512.png
Allow: /apple-touch-icon.png
Allow: /aris-logo.png
Allow: /logo.jpg
Allow: /og-image.png
Allow: /assets/
Allow: /

User-agent: Googlebot-Favicon
Allow: /favicon.ico
Allow: /favicon-48x48.png
Allow: /favicon-96x96.png
Allow: /favicon-192x192.png
Allow: /favicon-512x512.png
Allow: /apple-touch-icon.png
Allow: /

User-agent: Googlebot-Image
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /favicon.ico
Allow: /aris-logo.png
Allow: /logo.jpg
Allow: /og-image.png
Allow: /apple-touch-icon.png
Allow: /assets/
Allow: /

# General Catch-All
User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /dashboard/
Disallow: /admin/

# Sitemaps
Sitemap: https://www.bidsmith.pro/sitemap-index.xml
Sitemap: https://www.bidsmith.pro/sitemap.xml

# LLM Directives & Context Files
# Reference: https://llmstxt.org/
llms: https://www.bidsmith.pro/llms.txt
llms-full: https://www.bidsmith.pro/llms-full.txt