# ═══════════════════════════════════════════════════════════════════
# robots.txt — DermaPouf (dermapouf.com)
# World's First ISO-Certified Antibacterial Shower Mesh Pouf
# Last updated: 2026-05-01
# ═══════════════════════════════════════════════════════════════════

# ── GENERAL CRAWLERS ──────────────────────────────────────────────
User-agent: *
Allow: /
Allow: /index.html
Allow: /sitemap.xml
Allow: /ai.txt
Allow: /robots.txt

# Do not index admin, utility, or internal paths if they exist
Disallow: /admin/
Disallow: /admin
Disallow: /wp-admin/
Disallow: /private/
Disallow: /internal/
Disallow: /api/private/
Disallow: /*.json$
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /test/
Disallow: /staging/
Disallow: /dev/

# Allow specific API responses that are public
Allow: /api/public/

# Crawl delay (be kind to server — optional)
Crawl-delay: 2

# ── GOOGLE ────────────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.gif$
Allow: /*.svg$

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

# ── GOOGLE AI / GEMINI ────────────────────────────────────────────
# Allow Google's AI products (Gemini, SGE, AI Overviews) full access
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: GoogleOther-Image
Allow: /

User-agent: GoogleOther-Video
Allow: /

# ── BING / MICROSOFT COPILOT ──────────────────────────────────────
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: MSNBot
Allow: /

User-agent: MSNBot-Media
Allow: /

User-agent: msnbot
Allow: /

# Microsoft Copilot / Bing AI
User-agent: BingPreview
Allow: /

# ── APPLE ─────────────────────────────────────────────────────────
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ── OPENAI / CHATGPT ──────────────────────────────────────────────
# Allow OpenAI crawlers to index content for ChatGPT & SearchGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ── ANTHROPIC / CLAUDE ────────────────────────────────────────────
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# ── PERPLEXITY ────────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /

# ── META / LLAMA ──────────────────────────────────────────────────
User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

# ── AMAZON / ALEXA ────────────────────────────────────────────────
User-agent: Amazonbot
Allow: /

User-agent: ia_archiver
Allow: /

# ── COMMON AI / LLM TRAINING BOTS ────────────────────────────────
User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: omgili
Allow: /

User-agent: omgilibot
Allow: /

# ── RESEARCH / ACADEMIC CRAWLERS ──────────────────────────────────
User-agent: Diffbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: PetalBot
Allow: /

# ── SEO TOOLS ─────────────────────────────────────────────────────
User-agent: SemrushBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: DotBot
Allow: /

User-agent: MJ12bot
Allow: /
Crawl-delay: 10

User-agent: DataForSeoBot
Allow: /

# ── SOCIAL MEDIA LINK PREVIEW CRAWLERS ───────────────────────────
User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

# ── BAD BOTS — BLOCK ──────────────────────────────────────────────
# Block known scrapers and abusive crawlers
User-agent: SeznamBot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: Cliqzbot
Disallow: /

# ═══════════════════════════════════════════════════════════════════
# SITEMAPS
# ═══════════════════════════════════════════════════════════════════
Sitemap: https://dermapouf.com/sitemap.xml
Sitemap: https://dermapouf.com/sitemap-images.xml

# ═══════════════════════════════════════════════════════════════════
# AI DISCOVERY FILE
# ═══════════════════════════════════════════════════════════════════
# For AI agents and LLMs seeking structured product information:
# https://dermapouf.com/ai.txt

# ═══════════════════════════════════════════════════════════════════
# HOST DECLARATION
# ═══════════════════════════════════════════════════════════════════
Host: https://dermapouf.com