User-agent: *
Disallow: /dashboard.html
Disallow: /thank-you.html
Disallow: /sendform.php
Disallow: /logs/
Disallow: /assets/js/
Disallow: /assets/css/
User-agent: Googlebot
Allow: /
Disallow: /dashboard.html
Disallow: /thank-you.html
Disallow: /sendform.php
Disallow: /logs/
Sitemap: https://prozonc.cc/sitemap.xml
Host: prozonc.cc
# Crawl-delay settings
Crawl-delay: 10
# Specific directives for search bots
User-agent: Bingbot
Crawl-delay: 10
User-agent: Slurp
Crawl-delay: 10
# Directories
Disallow: /admin/
Disallow: /includes/
Disallow: /config/
Disallow: /private/
# Files
Disallow: *.pdf$
Disallow: *.zip$
# Allow access to important pages
Allow: /index.html
Allow: /about.html
Allow: /articles.html
Allow: /article*.html
Allow: /faq.html
Allow: /contacts.html
Allow: /login.html
Allow: /register.html
Allow: /privacy-policy.html
Allow: /cookies-policy.html
Allow: /terms-of-service.html
# Image crawling
User-agent: Googlebot-Image
Disallow:
User-agent: YandexImages
Disallow:
# Video crawling
User-agent: Googlebot-Video
Disallow:
# Mobile crawlers
User-agent: Googlebot-Mobile
Allow: /
User-agent: bingbot-mobile
Allow: /
# Social media crawlers
User-agent: facebookexternalhit
Allow: /
User-agent: Twitterbot
Allow: /
# Analytics and monitoring
User-agent: AhrefsBot
Crawl-delay: 10
User-agent: SemrushBot
Crawl-delay: 10
User-agent: MJ12bot
Crawl-delay: 10
# Host directive
Host: prozonc.cc