# ============================================================
# robots.txt — VerixID (verixid.com)
# Strategy: Allow all legitimate SEO + AI crawlers
#            Block sensitive internal & API paths only
# Last updated: 2026-03
# ============================================================


# ── GENERAL (semua crawler) ──────────────────────────────────
User-agent: *
Allow: /

# Internal / API — tidak perlu diindex
Disallow: /api/
Disallow: /price_list
Disallow: /submit
Disallow: /welcome-email
Disallow: /assets/
Disallow: /cdn-cgi/

# Admin & billing paths
Disallow: /admin/
Disallow: /billing/
Disallow: /invoice/
Disallow: /dashboard/

# Utility & error pages
Disallow: /404
Disallow: /500
Disallow: /maintenance


# ── GOOGLE ──────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /price_list
Disallow: /submit
Disallow: /assets/
Disallow: /cdn-cgi/
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /
Disallow: /assets/private/

User-agent: AdsBot-Google
Disallow: /


# ── BING / MICROSOFT ────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /price_list
Disallow: /submit
Disallow: /assets/
Disallow: /cdn-cgi/
Crawl-delay: 0

User-agent: MicrosoftPreview
Allow: /


# ── AI CRAWLERS — LEGAL & RESMI ─────────────────────────────

# OpenAI — ChatGPT search + training
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /price_list
Disallow: /submit
Disallow: /cdn-cgi/

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /


# Anthropic — Claude
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /


# Google AI — Gemini / AI Overviews
User-agent: Google-Extended
Allow: /


# Meta — Llama / Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /


# Apple — Siri, Apple Intelligence
User-agent: Applebot
Allow: /
Crawl-delay: 0

User-agent: Applebot-Extended
Allow: /


# Perplexity AI
User-agent: PerplexityBot
Allow: /


# You.com
User-agent: YouBot
Allow: /


# Brave Search
User-agent: BraveBot
Allow: /


# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /cdn-cgi/


# Yandex
User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /cdn-cgi/
Crawl-delay: 2


# ── CONTENT SCRAPERS — BLOKIR ───────────────────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /


# ── SITEMAP & AI DISCOVERY ───────────────────────────────────
Sitemap: https://verixid.com/sitemap.xml