# Allow all major search engines and AI bots

User-agent: *
Disallow: /admin/
Disallow: /login/
Disallow: /register/
Disallow: /search-results/
Disallow:	/webmail
Disallow:	/database
Disallow:	/webftp
Disallow:	/private
Disallow:	/backup
Disallow:	/downloads
Disallow:	/missing.php
Disallow:	/status.php
Disallow:	/old
Disallow:	/vnc
Disallow:	/quotes.txt
Disallow:	/url.php
Disallow:	/url

Disallow:	/beta
Disallow:	/blog
Disallow:	/blog_old
Disallow:	/downloads
Disallow:	/private
Disallow:	/missing.php
Disallow:	/old
Disallow:	/vnc



# Sitemap location (important for search engines)
#Sitemap: https://skyriser.com/sitemap.xml

# Additional configuration for specific crawlers
User-agent: Googlebot
Disallow: /private/
Allow: /

User-agent: Bingbot
Disallow: /private/
Allow: /

User-agent: Applebot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ChatGPT
Allow: /

User-agent: YandexBot
Disallow: /

# Restrict known unwanted or aggressive bots
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: DotBot
Disallow: /