# ============================================================================ # BeChallenge Robots.txt # https://bechallenge.io # Last updated: 2026-01-27 # ============================================================================ # ============================================================================ # DEFAULT RULES (All Crawlers) # ============================================================================ User-agent: * # Public pages - ALLOW Allow: / Allow: /challenges Allow: /library Allow: /about Allow: /contact Allow: /terms Allow: /privacy Allow: /help Allow: /faq Allow: /blog # Administrative areas - DISALLOW Disallow: /admin Disallow: /admin/ Disallow: /nova Disallow: /nova/ # API endpoints - DISALLOW Disallow: /api Disallow: /api/ Disallow: /graphql Disallow: /webhook Disallow: /webhooks # Development/Debug tools - DISALLOW Disallow: /_debugbar Disallow: /_debugbar/ Disallow: /telescope Disallow: /telescope/ Disallow: /horizon Disallow: /horizon/ Disallow: /_ignition Disallow: /clockwork # Storage and system files - DISALLOW Disallow: /storage Disallow: /storage/ Disallow: /vendor Disallow: /vendor/ Disallow: /node_modules Disallow: /.git Disallow: /.env # Authentication pages - DISALLOW # NOTE: /login/institution/* is intentionally ALLOWED so crawlers can see the # noindex meta tag and deindex those URLs (blocked URLs can't be deindexed). Allow: /login/institution/ Disallow: /login Disallow: /register Disallow: /password Disallow: /password/ Disallow: /email/verify Disallow: /email/ Disallow: /two-factor Disallow: /logout Disallow: /sanctum Disallow: /oauth # User private areas - DISALLOW Disallow: /dashboard Disallow: /profile Disallow: /settings Disallow: /notifications Disallow: /my-challenges Disallow: /my-courses Disallow: /my-teams Disallow: /inbox Disallow: /messages # Challenge internal pages (require auth) - DISALLOW Disallow: /challenge/*/edit Disallow: /challenge/*/settings Disallow: /challenge/*/participants Disallow: /challenge/*/analytics Disallow: /challenge/*/submissions Disallow: /course/*/edit # LTI integration endpoints - DISALLOW Disallow: /lti Disallow: /lti/ # Temporary and cache files - DISALLOW Disallow: /tmp Disallow: /temp Disallow: /cache Disallow: /*.log # Prevent indexing of search results and filtered pages Disallow: /search Disallow: /*?sort= Disallow: /*?filter= Disallow: /*?page= Disallow: /*&sort= Disallow: /*&filter= # ============================================================================ # GOOGLE (Primary Search Engine) # ============================================================================ User-agent: Googlebot Allow: / Allow: /challenges Allow: /library Allow: /blog Crawl-delay: 1 User-agent: Googlebot-Image Allow: /images/ Allow: /storage/public/ Disallow: /storage/private/ User-agent: Googlebot-Video Allow: /videos/ # ============================================================================ # BING (Microsoft Search Engine) # ============================================================================ User-agent: Bingbot Allow: / Crawl-delay: 2 User-agent: MSNBot Allow: / Crawl-delay: 2 # ============================================================================ # OTHER SEARCH ENGINES # ============================================================================ User-agent: DuckDuckBot Allow: / Crawl-delay: 2 User-agent: Slurp Allow: / Crawl-delay: 2 User-agent: Baiduspider Disallow: / User-agent: YandexBot Allow: / Crawl-delay: 3 # ============================================================================ # AI/LLM CRAWLERS - CONTROLLED ACCESS # We allow AI crawlers to index public educational content # to improve discoverability and AI-assisted learning # ============================================================================ # OpenAI User-agent: GPTBot Allow: / Allow: /challenges Allow: /library Allow: /blog Allow: /about Allow: /llms.txt Allow: /llms-full.txt Disallow: /api/ Disallow: /admin/ Disallow: /dashboard Disallow: /profile Disallow: /storage/private/ User-agent: ChatGPT-User Allow: / Disallow: /api/ Disallow: /admin/ Disallow: /dashboard # Google AI User-agent: Google-Extended Allow: / Allow: /challenges Allow: /library Disallow: /api/ Disallow: /admin/ Disallow: /dashboard # Anthropic (Claude) User-agent: anthropic-ai Allow: / Allow: /challenges Allow: /library Allow: /llms.txt Allow: /llms-full.txt Disallow: /api/ Disallow: /admin/ Disallow: /dashboard User-agent: Claude-Web Allow: / Disallow: /api/ Disallow: /admin/ # Common Crawl (used for AI training datasets) User-agent: CCBot Allow: / Allow: /challenges Allow: /library Disallow: /api/ Disallow: /admin/ Disallow: /dashboard Disallow: /profile # Perplexity AI User-agent: PerplexityBot Allow: / Allow: /challenges Allow: /library Disallow: /api/ Disallow: /admin/ # Cohere AI User-agent: cohere-ai Allow: / Disallow: /api/ Disallow: /admin/ # Meta AI User-agent: FacebookBot Allow: / Disallow: /api/ Disallow: /admin/ User-agent: Meta-ExternalAgent Allow: / Disallow: /api/ Disallow: /admin/ # ============================================================================ # SOCIAL MEDIA CRAWLERS # ============================================================================ User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Pinterest Allow: / User-agent: WhatsApp Allow: / User-agent: TelegramBot Allow: / # ============================================================================ # MALICIOUS/UNWANTED BOTS - BLOCK # ============================================================================ User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: SearchmetricsBot Disallow: / User-agent: MegaIndex Disallow: / User-agent: Bytespider Disallow: / User-agent: PetalBot Disallow: / # ============================================================================ # SITEMAP & HOST # ============================================================================ Sitemap: https://bechallenge.io/sitemap.xml Host: https://bechallenge.io # ============================================================================ # LLMS.TXT REFERENCE (following llmstxt.org specification) # Concise version: https://bechallenge.io/llms.txt # Full documentation: https://bechallenge.io/llms-full.txt # ============================================================================