# Default — allow all search-index crawlers, with low-value paths blocked.
User-agent: *
Allow: /

# Allow important assets explicitly
Allow: /_next/static/
Allow: /images/

# Block APIs (used client-side, nothing indexable)
Disallow: /api/

# Block WordPress API endpoints (headless WP lives on resources.* — these
# paths don't exist on this domain, but block them defensively)
Disallow: /blog/wp-json/
Disallow: /wp-json/

# Block search / filter / sort / tracking query-parameter URLs — crawl-budget
# waste with no unique indexable content (canonical strips query params anyway)
Disallow: /*?s=
Disallow: /*?search=
Disallow: /*?filter=
Disallow: /*?sort=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=

# Block preview URLs
Disallow: /*preview=

# AI-training crawlers. Blocked because they generate the vast majority of
# bot traffic (ClaudeBot alone was 99% of all requests in a 2026-05 audit)
# without driving any user-facing search referrals, and their volume was
# pushing Edge Middleware + ISR Writes past the Vercel free-tier limits.
# Search-index crawlers (Googlebot, Bingbot, Applebot, etc.) are unaffected.
User-agent: ClaudeBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Amazonbot feeds Alexa + Amazon's AI training, not a consumer search engine,
# so it drives no user-facing referrals. In a 2026-07-13 log window it was the
# single heaviest crawler (~36% of requests), pushing Edge Middleware + ISR
# writes toward the Vercel free-tier limits. Blocked for the same reason as the
# AI-training crawlers above. Search-index crawlers remain unaffected.
User-agent: Amazonbot
Disallow: /

# SEO-tool / data-reseller / AI crawlers. A 2026-10-16 log sample showed
# SemrushBot, Amzn-SearchBot and DataForSeoBot causing ~74% of uncached ISR
# regenerations (Googlebot + Bingbot together were ~1%), driving Vercel usage
# from ~$0.20/day to $1–2.60/day after 11 Oct. None send search referrals.
# Amzn-SearchBot is a separate token from Amazonbot, so it needs its own block.
User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Full-volume follow-up (2026-10-16): MJ12bot (Majestic backlink index, ~4% of
# regenerations) and meta-webindexer (Meta AI indexer, ~8% and rising).
# facebookexternalhit — Meta's link-preview fetcher — stays allowed so shared
# links still render previews.
User-agent: MJ12bot
Disallow: /

User-agent: meta-webindexer
Disallow: /

# Sitemaps
Sitemap: https://islamicprayerstimes.online/sitemap-index.xml