# robots.txt - Shehnai Restaurant
# Indian restaurant & daily lunch buffet, 447 Main St W, Hamilton, ON
# https://shehnaihamilton.ca/

# ------------------------------------------------------------
# All crawlers (Googlebot, Bingbot, etc.)
# ------------------------------------------------------------
User-agent: *
Allow: /
# /components/ (shared header & footer) must stay crawlable so pages render fully
Allow: /components/
Disallow: /.github/
Disallow: /.vscode/
Disallow: /update-menu.js

# llms.txt stays crawlable for AI agents but should not appear in search results.
# Note: Google does not honour "Noindex" inside robots.txt; for guaranteed removal
# also serve /llms.txt with the HTTP header "X-Robots-Tag: noindex" at the host.
Noindex: /llms.txt

# ------------------------------------------------------------
# AI crawlers and assistants - allowed, with llms.txt as the guide
# LLM guide: https://shehnaihamilton.ca/llms.txt
# ------------------------------------------------------------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: CCBot
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /components/
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html
Disallow: /.github/
Disallow: /.vscode/
Disallow: /update-menu.js

# ------------------------------------------------------------
# Google Ads landing-page checks
# ------------------------------------------------------------
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Allow: /

# ------------------------------------------------------------
# Sitemap
# ------------------------------------------------------------
Sitemap: https://shehnaihamilton.ca/sitemap.xml