# robots.txt for printonline.ae
# Magento 1.9 — Updated April 2026

# ============================================
# Default rules for all crawlers
# (Googlebot, Bingbot, GPTBot, ClaudeBot,
#  PerplexityBot, Google-Extended, Applebot-Extended,
#  Amazonbot, Meta-ExternalAgent, etc.)
# ============================================

User-agent: *

# PRINTO-NFC-V1
# NFC/QR card endpoints. Technical redirects, not content: they carry no
# customer data and every response already sends X-Robots-Tag: noindex.
Disallow: /google-reviews/
Disallow: /my-nfc-cards/

# Magento system directories
Disallow: /app/
Disallow: /cgi-bin/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /magento/
Disallow: /pkginfo/
Disallow: /shell/
Disallow: /var/
Disallow: /report/
Disallow: /stats/
Disallow: /404/

# Magento dynamic / account / checkout / search paths
Disallow: /index.php/
Disallow: /catalog/product_compare/
# Disallow: /catalog/category/view/   <- unblocked 2026: canonical consolidates these
# Disallow: /catalog/product/view/    <- unblocked 2026: canonical consolidates these
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /control/
# Disallow: /contacts/                <- unblocked 2026: legitimate indexable page
Disallow: /customer/
Disallow: /customize/
Disallow: /newsletter/
Disallow: /poll/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/
Disallow: /productalert/
Disallow: /directory/currency/switch/
Disallow: /*/filter/
Disallow: /filter/
Disallow: /archive/

# Magento system files
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /mage
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /LICENSE_EE.html
Disallow: /LICENSE_EE.txt
Disallow: /RELEASE_NOTES.txt
Disallow: /STATUS.txt

# Session IDs and noisy URL parameters
Disallow: /*?SID=
Disallow: /*?q=
Disallow: /*?dir=
Disallow: /*?mode=
Disallow: /*?limit=

# Legacy / osCommerce / junk paths
Disallow: /product_info
Disallow: /popup_image
Disallow: /*cPath=

# Site-specific blocks (security / privacy)
Disallow: /mcoptions/
Disallow: /whatsapp/

# Crawler traps (added 2026-09-07 after 7,140 rate-limited requests in one day)
# Seller-list sort/page permutations: 299 distinct URLs in a day, 274 of 275
# hits from Amazonbot alone. The clean /csmarketplace/vshops/ landing page and
# the individual /csmarketplace/vshops/view/ shop pages stay crawlable.
Disallow: /csmarketplace/vshops/index/
Disallow: /csmarketplace/account/
# Tracking pixel endpoint, no page behind it.
Disallow: /templates_gallery/track_view.php
# Self-referencing links that nest ?return= inside itself without limit.
Disallow: /*?*return=

# Explicit allows
Allow: /*?p=
Allow: /index.php/blog/
Allow: /catalog/seo_sitemap/category/
# AI agent discovery & catalogue endpoints (Printo AiVisibility)
Allow: /llms.txt
Allow: /agents.md
Allow: /.well-known/ucp
Allow: /agent/
Allow: /agent/products.json?q=*

# Web fonts (do not block — required for rendering)
Allow: /*.ttf$
Allow: /*.woff$
Allow: /*.woff2$

# Polite crawl rate (Googlebot ignores this; others honor it)
Crawl-delay: 5

# ============================================
# Aggressive / unwanted crawlers — blocked
# ============================================

User-agent: PetalBot
Disallow: /

# ============================================
# Sitemap
# ============================================

Sitemap: https://www.printonline.ae/sitemap.xml