mirror of
https://github.com/x1xhlol/system-prompts-and-models-of-ai-tools.git
synced 2026-06-17 23:09:35 +00:00
32 lines
501 B
Plaintext
32 lines
501 B
Plaintext
User-agent: *
|
|
Allow: /
|
|
Allow: /marketers
|
|
Allow: /pricing
|
|
Allow: /partners
|
|
Allow: /privacy
|
|
Allow: /terms
|
|
|
|
Disallow: /admin/
|
|
Disallow: /api/
|
|
Disallow: /internal/
|
|
Disallow: /*.json$
|
|
|
|
# Dealix — مندوب مبيعات AI بالعربي الخليجي
|
|
Sitemap: https://dealix.ai/sitemap.xml
|
|
|
|
# Crawl-delay for aggressive bots
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: AhrefsBot
|
|
Crawl-delay: 10
|
|
|
|
User-agent: SemrushBot
|
|
Crawl-delay: 10
|