# ============================================================
# WiseLoansFinancing.com — robots.txt
# Last updated: June 2026
# ============================================================

# Default rules for all crawlers
User-agent: *
Allow: /
Disallow: /uploads/
Disallow: /admin/
Disallow: /tmp/
Disallow: /*?*ref=
Disallow: /*?*utm_

# Block known SEO scrapers (they don't add value, only consume bandwidth)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Block AI training crawlers (optional — you can remove these if you want AI indexing)
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

# Google's primary crawler — allowed everywhere
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Bing crawler — allowed everywhere
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Sitemap location
Sitemap: https://wiseloansfinancing.com/sitemap.xml

# Host preference (helps with www vs non-www canonicalization)
Host: https://wiseloansfinancing.com