# Robots.txt for Clothing Digest
# Enhanced configuration for fashion and sewing content site

# Allow all search engines access to public content
User-agent: *

# WordPress core directories to block
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/*/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/uploads/wpforms/

# Allow important WordPress functionality
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*/css/
Allow: /wp-content/themes/*/js/
Allow: /wp-content/themes/*/images/
Allow: /wp-content/themes/*/fonts/

# Block sensitive and unnecessary pages
Disallow: /cgi-bin/
Disallow: /wp-config.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-trackback.php
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /xmlrpc.php

# Block search and filtering pages to prevent duplicate content
Disallow: /?s=
Disallow: /search/
Disallow: /*?s=*
Disallow: /*&s=*

# Block pagination and sorting parameters
Disallow: /*?page=*
Disallow: /*&page=*
Disallow: /*?sort=*
Disallow: /*&sort=*
Disallow: /*?filter=*
Disallow: /*&filter=*

# Block private and user-specific content
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /my-account/

# Block development and testing areas
Disallow: /test/
Disallow: /staging/
Disallow: /dev/
Disallow: /demo/

# Block comment and reply pages
Disallow: /*/comment-page-*
Disallow: /*?replytocom=*

# Allow important crawlers specific access
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Block problematic bots (adjust based on your server logs)
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# Set crawl delay for aggressive crawlers
User-agent: *
Crawl-delay: 1

# Sitemap locations
Sitemap: https://clothingdigest.com/sitemap_index.xml
Sitemap: https://clothingdigest.com/post-sitemap.xml
Sitemap: https://clothingdigest.com/page-sitemap.xml
Sitemap: https://clothingdigest.com/category-sitemap.xml