# ─────────────────────────────────────────────────────────────
# DNSai robots.txt
# Canonical URL: https://dnsai.com/robots.txt
# Version: 2026-11-01 (Content-Signal added)
#
# Policy: OPEN CRAWL
# All major search engines and AI/LLM systems are welcome to crawl,
# index, retrieve, and use DNSai content (including for training).
# See also: https://dnsai.com/llms.txt (bot-facing manifest)
#           https://dnsai.com/.well-known/agents.json (agent/API manifest)
#           https://dnsai.com/openapi.json (OpenAPI 3.1 public tool APIs)
# Contact:  abuse@dnsai.com
# ─────────────────────────────────────────────────────────────

# Default rules — apply to every crawler, including AI/search bots
# that do not declare a specific user-agent.
User-agent: *
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /cgi-bin/
Disallow: /secure/
Disallow: /account-management/secure/
Disallow: /listen/
Disallow: /profile/company/eporner.com.html
Disallow: /profile/company/extremetube.com.html
Disallow: /profile/company/beeg.com.html
Disallow: /profile/company/z_disabled_aa.com.html
Disallow: /profile/company/googleapps.com.html
Disallow: /profile/company/performancefoodgroup.com.html
Disallow: /profile/company/nickiminaj.com.html

# ─────────────────────────────────────────────────────────────
# Explicit per-bot welcome blocks.
# Each group re-states the full Disallow set so bots that look up
# their own user-agent get an authoritative answer (per the robots
# spec, a user-agent-specific group overrides the `*` group and does
# NOT inherit rules from it).
# ─────────────────────────────────────────────────────────────

# Search engines
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: Bingbot
User-agent: msnbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Yeti
User-agent: SeznamBot
User-agent: MojeekBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /cgi-bin/
Disallow: /secure/
Disallow: /account-management/secure/
Disallow: /listen/
Disallow: /profile/company/eporner.com.html
Disallow: /profile/company/extremetube.com.html
Disallow: /profile/company/beeg.com.html
Disallow: /profile/company/z_disabled_aa.com.html
Disallow: /profile/company/googleapps.com.html
Disallow: /profile/company/performancefoodgroup.com.html
Disallow: /profile/company/nickiminaj.com.html

# LLM / AI training + retrieval crawlers
# ChatGPT, Claude, Gemini, Perplexity, Mistral, Cohere, Meta, Apple,
# Amazon, DuckDuckGo-AI, You.com, Common Crawl, ByteDance, Diffbot,
# and every other major AI system that respects robots.txt.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: facebookexternalhit
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: meta-webindexer
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: CCBot
User-agent: Bytespider
User-agent: ByteSpider
User-agent: YouBot
User-agent: Diffbot
User-agent: Omgilibot
User-agent: OmgiliBot
User-agent: ImagesiftBot
User-agent: PanguBot
User-agent: PetalBot
User-agent: TimpiBot
User-agent: NeevaBot
User-agent: Scrapy
User-agent: AndiBot
User-agent: Allen-AI
User-agent: Kagibot
User-agent: iaskspider
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /cgi-bin/
Disallow: /secure/
Disallow: /account-management/secure/
Disallow: /listen/
Disallow: /profile/company/eporner.com.html
Disallow: /profile/company/extremetube.com.html
Disallow: /profile/company/beeg.com.html
Disallow: /profile/company/z_disabled_aa.com.html
Disallow: /profile/company/googleapps.com.html
Disallow: /profile/company/performancefoodgroup.com.html
Disallow: /profile/company/nickiminaj.com.html

# SEO / site-audit crawlers — allowed (our audit vendor + industry norm)
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: MJ12bot
User-agent: DataForSeoBot
User-agent: rogerbot
User-agent: dotbot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /cgi-bin/
Disallow: /secure/
Disallow: /account-management/secure/
Disallow: /listen/
Disallow: /profile/company/eporner.com.html
Disallow: /profile/company/extremetube.com.html
Disallow: /profile/company/beeg.com.html
Disallow: /profile/company/z_disabled_aa.com.html
Disallow: /profile/company/googleapps.com.html
Disallow: /profile/company/performancefoodgroup.com.html
Disallow: /profile/company/nickiminaj.com.html

# ─────────────────────────────────────────────────────────────
# Sitemaps
# Primary sitemap index — child sitemaps enumerated for the benefit
# of bots that do not follow sitemap-index files.
# ─────────────────────────────────────────────────────────────
Sitemap: https://dnsai.com/sitemap.xml
Sitemap: https://dnsai.com/sitemap-main.xml
Sitemap: https://dnsai.com/sitemap-tools.xml
Sitemap: https://dnsai.com/sitemap-blog.xml
Sitemap: https://dnsai.com/sitemap-vendor-directory.xml
Sitemap: https://dnsai.com/profile/sitemap-profiles.xml
Sitemap: https://dnsai.com/sitemap-video.xml