# ============================================================
# robots.txt — VideoIndex
# ============================================================
# Generated for: VideoIndex PHP/MySQL Video Indexing Website
# ============================================================

User-agent: *

# Allow all crawlers to index key pages
Allow: /index.php
Allow: /sitemap.php
Allow: /robots.txt

# Block admin & internal actions from crawlers
Disallow: /index.php?action=add
Disallow: /index.php?action=delete
Disallow: /index.php?action=edit
Disallow: /?action=delete
Disallow: /?action=add
Disallow: /?action=edit

# Block duplicate/pagination from crawlers (optional – remove if you want paginated pages indexed)
Disallow: /index.php?p=

# Block search result pages (avoid duplicate content)
Disallow: /index.php?q=

# Specific bots — generous crawl-delay for heavy bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 15

User-agent: DotBot
Disallow: /

User-agent: Baiduspider
Crawl-delay: 5

# Allow Google, Bing, and major search engines — no restrictions
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# ============================================================
# Sitemap Location — CHANGE to your actual domain!
# ============================================================
Sitemap: https://hdvideos.learnwithtest.pro/sitemap.xml