# Allow all bots to crawl public content User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /xmlrpc.php Disallow: /wp-login.php Disallow: /wp-register.php # Optional: Block readme/license/changelog files for security Disallow: /readme.html Disallow: /license.txt Disallow: /changelog.txt # Allow crawling of all other resources (including CSS, JS, images)