# robots.txt for fashiontrendyshop.com
# Last updated: 2025-09-07
# Optimized for women's fashion e-commerce

# Default rules for all crawlers
User-agent: *
Allow: /

# Crawl delay to prevent server overload
# Note: Google ignores crawl-delay, use Google Search Console for rate limiting
Crawl-delay: 1

# === SECURITY & PRIVACY BLOCKS ===
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-content/uploads/wpo/
Disallow: /cart/
Disallow: /checkout/
Disallow: /customer/
Disallow: /account/
Disallow: /my-account/
Disallow: /login/
Disallow: /register/
Disallow: /password-reset/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /track-order/
Disallow: /order-tracking/
Disallow: /returns/
Disallow: /refunds/
Disallow: /ajax/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /private/
Disallow: /tmp/
Disallow: /temp/
Disallow: /backup/
Disallow: /error/
Disallow: /404/
Disallow: /500/

# === CONVERSION & THANK YOU PAGES ===
Disallow: /thank-you/
Disallow: /order-received/
Disallow: /order-confirmation/
Disallow: /payment-success/
Disallow: /payment-failed/
Disallow: /checkout-complete/

# === SEARCH & FILTER PARAMETERS ===
Disallow: /search
Disallow: /search/
Disallow: /*?q=*
Disallow: /*?search=*
Disallow: /*?s=*
Disallow: /*?query=*
Disallow: /*?keyword=*

# === DUPLICATE CONTENT PREVENTION ===
# Sorting and filtering parameters
Disallow: /*?sort=*
Disallow: /*?order=*
Disallow: /*?orderby=*
Disallow: /*?filter=*
Disallow: /*?price=*
Disallow: /*?color=*
Disallow: /*?size=*
Disallow: /*?style=*
Disallow: /*?brand=*
Disallow: /*?material=*
Disallow: /*?category=*

# Pagination parameters
Disallow: /*?page=*
Disallow: /*?p=*
Disallow: /*?paged=*
Disallow: /*&page=*
Disallow: /*&p=*

# View and display parameters
Disallow: /*?view=*
Disallow: /*?display=*
Disallow: /*?layout=*
Disallow: /*?grid=*
Disallow: /*?list=*

# Tracking parameters
Disallow: /*?utm_*
Disallow: /*&utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Disallow: /*?ref=*
Disallow: /*?source=*
Disallow: /*?campaign=*
Disallow: /*?affiliate=*
Disallow: /*?promo=*

# Session and cache parameters
Disallow: /*?session=*
Disallow: /*?sid=*
Disallow: /*?cache=*
Disallow: /*?v=*
Disallow: /*?ver=*
Disallow: /*?version=*
Disallow: /*?timestamp=*

# Fashion-specific parameters
Disallow: /*?collection=*
Disallow: /*?trend=*
Disallow: /*?season=*
Disallow: /*?occasion=*

# === FILE TYPE RESTRICTIONS ===
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.xls$
Disallow: /*.xlsx$
Disallow: /*.ppt$
Disallow: /*.pptx$
Disallow: /*.zip$
Disallow: /*.rar$
Disallow: /*.tar$
Disallow: /*.gz$
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.bak$

# === EXPLICITLY ALLOW IMPORTANT CONTENT ===
# Essential assets
Allow: /css/
Allow: /js/
Allow: /javascript/
Allow: /images/
Allow: /img/
Allow: /media/
Allow: /fonts/
Allow: /static/
Allow: /assets/
Allow: /uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /wp-includes/

# Important pages and sections
Allow: /products/
Allow: /product/
Allow: /collections/
Allow: /collection/
Allow: /categories/
Allow: /category/
Allow: /clothing/
Allow: /dresses/
Allow: /accessories/
Allow: /bodycon-dresses/
Allow: /maxi-dresses/
Allow: /mini-dresses/
Allow: /summer-dresses/
Allow: /leggings/
Allow: /tops/
Allow: /bottoms/
Allow: /sale/
Allow: /deals/
Allow: /offers/
Allow: /clearance/
Allow: /new-arrivals/
Allow: /trending/
Allow: /bestsellers/
Allow: /blog/
Allow: /style-guide/
Allow: /fashion-tips/
Allow: /about/
Allow: /contact/
Allow: /shipping/
Allow: /delivery/
Allow: /size-guide/
Allow: /size-chart/
Allow: /faq/
Allow: /privacy-policy/
Allow: /terms-conditions/
Allow: /cookie-policy/
Allow: /return-policy/

# === SPECIFIC BOT CONFIGURATIONS ===

# Google bots - most permissive
# Note: Google ignores crawl-delay directive, use Google Search Console to control crawl rate
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
Allow: /

# Bing bot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Facebook bot (for social sharing - important for fashion)
User-agent: facebookexternalhit
Allow: /
Allow: /product/
Allow: /products/
Allow: /collections/
Allow: /images/
Allow: /media/

# Twitter bot (for social sharing)
User-agent: Twitterbot
Allow: /
Allow: /product/
Allow: /products/
Allow: /collections/

# Pinterest bot (very important for fashion e-commerce)
User-agent: Pinterest
Allow: /
Allow: /product/
Allow: /products/
Allow: /collections/
Allow: /images/
Allow: /media/
Crawl-delay: 3

# Instagram bot (fashion-focused)
User-agent: InstagramBot
Allow: /
Allow: /product/
Allow: /products/
Allow: /images/
Allow: /media/

# LinkedIn bot
User-agent: LinkedInBot
Allow: /
Allow: /about/
Allow: /blog/

# SEO tool bots - allow but with slower crawling
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: MozBot
Allow: /
Crawl-delay: 10

User-agent: YandexBot
Allow: /
Crawl-delay: 5

# Block genuinely aggressive or low-value bots
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

# === SITEMAP DECLARATIONS ===
Sitemap: https://www.fashiontrendyshop.com/sitemap.xml
Sitemap: https://www.fashiontrendyshop.com/sitemap_index.xml
Sitemap: https://www.fashiontrendyshop.com/product-sitemap.xml
Sitemap: https://www.fashiontrendyshop.com/category-sitemap.xml
Sitemap: https://www.fashiontrendyshop.com/collection-sitemap.xml
Sitemap: https://www.fashiontrendyshop.com/post-sitemap.xml
Sitemap: https://www.fashiontrendyshop.com/page-sitemap.xml

# === ADDITIONAL NOTES ===
# This robots.txt is optimized for fashion e-commerce SEO
# Regular review and updates recommended
# Monitor crawl budget usage in Google Search Console
# Note: Google ignores crawl-delay - use Google Search Console > Settings > Crawl Rate to control Google's crawling speed
# For other search engines, crawl-delay is still respected
# Social media bots are prioritized due to fashion industry's visual nature