# startup.ph — robots.txt
# Governs crawler access, including AI Content-Signal directives (search/
# ai-input/ai-train/use). Site policy: full permissive access for search,
# AI input (RAG/grounding), and AI training. Corrected/verified 2026-08-08 —
# see project notes for which crawler names were freshly confirmed against
# each company's own current documentation vs. carried forward unverified.

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=full
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# ── Anthropic — verified against Anthropic's current official crawler
#    documentation (support.claude.com). Only these three names are
#    real/current: ClaudeBot, Claude-User, Claude-SearchBot. The previous
#    version of this file used "anthropic-ai" and "Claude-Web", which do
#    NOT appear in current official documentation — replaced below. ──
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# ── OpenAI — verified against OpenAI's official bots documentation ──
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ── Google — verified against Google's official crawler documentation ──
User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

User-agent: GoogleOther
Allow: /

# ── Apple ──
User-agent: Applebot-Extended
Allow: /

# ── Microsoft / Bing ──
User-agent: Bingbot
Allow: /

# ── Perplexity — verified against Perplexity's official bots documentation ──
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ── xAI — NOT independently verified this session (no official xAI crawler
#    documentation page could be reached). Both entries kept as best-effort,
#    same reasoning as the DeepSeek/Qwen block below. ──
User-agent: Grok
Allow: /

User-agent: xAI-Bot
Allow: /

# ── ByteDance ──
User-agent: Bytespider
Allow: /

# ── Huawei ──
User-agent: PetalBot
Allow: /

# ── Amazon ──
User-agent: Amazonbot
Allow: /

# ── Meta — verified against Meta's official crawler documentation ──
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ── Mistral — verified via third-party crawler directory (no first-party
#    Mistral crawler-documentation page found) ──
User-agent: MistralAI-User
Allow: /

# ── Best-effort entries — exact agent strings unverified/inconsistently
#    reported across sources. Harmless no-ops if a name doesn't match a
#    real bot, since the wildcard Allow: / above already covers everything
#    not explicitly matched. DeepSeek does not publish a public crawler
#    user-agent at all (its fetches reportedly look like ordinary browser
#    traffic) — DeepSeekBot below is not a confirmed real identifier, kept
#    only because it's a harmless no-op either way. Qwen/Alibaba naming is
#    inconsistently reported (QwenBot / Tongyi-Crawler / Alibaba-AIBot /
#    Qwen-User) — all listed variants kept for the same reason. ──
User-agent: DeepSeekBot
Allow: /

User-agent: QwenBot
Allow: /

User-agent: Qwen-User
Allow: /

User-agent: Alibaba-AIBot
Allow: /

User-agent: Moonshotbot
Allow: /

# ── SEO/scraper bots — blocked ──
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

Sitemap: https://www.startup.ph/wp-sitemap.xml