# ====================================================== # Robots.txt for https://www.bookshub.pk/ # BooksHub PK - Search & AI Friendly # ====================================================== # ====================================================== # DEFAULT RULES # ====================================================== User-agent: * # ----------------------------- # WordPress # ----------------------------- Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /xmlrpc.php # ----------------------------- # WooCommerce / Customer Areas # ----------------------------- Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /wishlist/ Disallow: /order-received/ # ----------------------------- # Internal Search # ----------------------------- Disallow: /?s= Disallow: /search/ # ----------------------------- # Dynamic / Parameter URLs # ----------------------------- Disallow: /*?add-to-cart= Disallow: /*?orderby= Disallow: /*?filter= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?rating_filter= Disallow: /*?wc-ajax= Disallow: /*?replytocom= # Tracking Parameters Disallow: /*?utm_ Disallow: /*?fbclid= Disallow: /*?gclid= Disallow: /*?mc_cid= Disallow: /*?mc_eid= # ----------------------------- # Feeds # ----------------------------- Disallow: /feed/ Disallow: /*/feed/ # ----------------------------- # Allow Important SEO Content # ----------------------------- Allow: /product/ Allow: /product-category/ Allow: /wp-content/uploads/ # ====================================================== # GOOGLE # ====================================================== User-agent: Googlebot Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /wishlist/ Disallow: /order-received/ Disallow: /wp-admin/ Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /?s= Disallow: /search/ Disallow: /*?add-to-cart= Disallow: /*?orderby= Disallow: /*?filter= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?rating_filter= Disallow: /*?wc-ajax= Disallow: /*?replytocom= Disallow: /*?utm_ Disallow: /*?fbclid= Disallow: /*?gclid= Allow: /wp-content/uploads/ # ====================================================== # GOOGLE IMAGE # ====================================================== User-agent: Googlebot-Image Allow: /wp-content/uploads/ # ====================================================== # BING # ====================================================== User-agent: Bingbot Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /wishlist/ Disallow: /order-received/ Disallow: /wp-admin/ Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /?s= Disallow: /search/ Disallow: /*?add-to-cart= Disallow: /*?orderby= Disallow: /*?filter= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?rating_filter= Disallow: /*?wc-ajax= Allow: /wp-content/uploads/ # Optional crawl pacing for Bing # Crawl-delay: 5 # ====================================================== # OTHER MAJOR SEARCH CRAWLERS # ====================================================== User-agent: Applebot Allow: / User-agent: MSNBot Allow: / # ====================================================== # AI SEARCH CRAWLERS # ====================================================== User-agent: OAI-SearchBot Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /wishlist/ Disallow: /order-received/ Disallow: /wp-admin/ Disallow: /wp-login.php/ Disallow: /xmlrpc.php Disallow: /?s= Disallow: /search/ Disallow: /*?add-to-cart= Disallow: /*?orderby= Disallow: /*?filter= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?wc-ajax= Allow: /product/ Allow: /product-category/ Allow: /wp-content/uploads/ User-agent: GPTBot Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /wishlist/ Disallow: /order-received/ Disallow: /?s= Disallow: /search/ Disallow: /*?add-to-cart= Disallow: /*?orderby= Disallow: /*?filter= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?wc-ajax= Allow: /product/ Allow: /product-category/ Allow: /wp-content/uploads/ User-agent: ClaudeBot Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /wishlist/ Disallow: /order-received/ Disallow: /?s= Disallow: /search/ Disallow: /*?add-to-cart= Disallow: /*?orderby= Disallow: /*?filter= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?wc-ajax= Allow: /product/ Allow: /product-category/ Allow: /wp-content/uploads/ User-agent: Claude-SearchBot Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /wishlist/ Disallow: /order-received/ Disallow: /*?add-to-cart= Disallow: /*?orderby= Disallow: /*?filter= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?wc-ajax= Allow: /product/ Allow: /product-category/ Allow: /wp-content/uploads/ User-agent: PerplexityBot Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /wishlist/ Disallow: /order-received/ Disallow: /?s= Disallow: /search/ Disallow: /*?add-to-cart= Disallow: /*?orderby= Disallow: /*?filter= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?wc-ajax= Allow: /product/ Allow: /product-category/ Allow: /wp-content/uploads/ User-agent: Google-Extended Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Allow: /product/ Allow: /product-category/ Allow: /wp-content/uploads/ User-agent: Amazonbot Allow: / Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Allow: /product/ Allow: /product-category/ Allow: /wp-content/uploads/ # ====================================================== # HEAVY SEO / DATA SCRAPING CRAWLERS # ====================================================== User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: Barkrowler Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: SeekportBot Disallow: / User-agent: CCBot Disallow: / User-agent: Bytespider Disallow: / # ====================================================== # SITEMAP # ====================================================== Sitemap: https://www.bookshub.pk/sitemap_index.xml