# This virtual robots.txt file was created by the Virtual Robots.txt WordPress plugin: https://www.wordpress.org/plugins/pc-robotstxt/ # robots.txt for https://memorablepress.com # Disallow the Internet Archive User-agent: ia_archiver Disallow: / User-agent: GPTBot Disallow: / User-agent: AhrefsBot Disallow: / # For all other agents User-agent: * # Include the full sitemap url for discovery # Note, this may do such on its own as it can sometimes detect one # So check the rendered file before addit it manually here Sitemap: https://memorablepress.com/sitemap.xml # Yandex complains without this Host: memorablepress.com # Disallow specific files or directories # Disallow: /filenames* # Disallow: /directory-and-contents/* # Disallow and allow some wordpress stuff Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-includes/ Allow: /wp-includes/js/ Allow: /wp-includes/images/ Allow: /wp-includes/css/ Disallow: /trackback/ Disallow: /feed/ Disallow: /xmlrpc.php Disallow: /wp-login.php Disallow: /wp-register.php # End of wordpress stuff # Some internal blocks Disallow: /order-cancelled* Disallow: /order-completed* Disallow: /estore-action* Disallow: /account* Disallow: /index-az* Disallow: /checkout* Disallow: /contact-submitted* # End of internal blocks # Note: any blocked pages should also have a meta tag # Put this in the Raw Data section of HeadSpace #