User-agent: *
Disallow: /dashboard.html
Disallow: /thank-you.html
Disallow: /sendform.php
Disallow: /logs/
Disallow: /assets/js/
Disallow: /assets/css/

User-agent: Googlebot
Allow: /
Disallow: /dashboard.html
Disallow: /thank-you.html
Disallow: /sendform.php
Disallow: /logs/

Sitemap: https://prozonc.cc/sitemap.xml

Host: prozonc.cc

# Crawl-delay settings
Crawl-delay: 10

# Specific directives for search bots
User-agent: Bingbot
Crawl-delay: 10

User-agent: Slurp
Crawl-delay: 10

# Directories
Disallow: /admin/
Disallow: /includes/
Disallow: /config/
Disallow: /private/

# Files
Disallow: *.pdf$
Disallow: *.zip$

# Allow access to important pages
Allow: /index.html
Allow: /about.html
Allow: /articles.html
Allow: /article*.html
Allow: /faq.html
Allow: /contacts.html
Allow: /login.html
Allow: /register.html
Allow: /privacy-policy.html
Allow: /cookies-policy.html
Allow: /terms-of-service.html

# Image crawling
User-agent: Googlebot-Image
Disallow:

User-agent: YandexImages
Disallow:

# Video crawling
User-agent: Googlebot-Video
Disallow:

# Mobile crawlers
User-agent: Googlebot-Mobile
Allow: /

User-agent: bingbot-mobile
Allow: /

# Social media crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# Analytics and monitoring
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

# Host directive
Host: prozonc.cc