{"id":252378,"date":"2025-09-16T14:25:09","date_gmt":"2025-09-16T14:25:09","guid":{"rendered":"https:\/\/wordpress.org\/plugins\/crawler-record\/"},"modified":"2026-01-07T16:00:01","modified_gmt":"2026-01-07T16:00:01","slug":"crawler-record","status":"publish","type":"plugin","link":"https:\/\/tah.wordpress.org\/plugins\/crawler-record\/","author":13881536,"comment_status":"closed","ping_status":"closed","template":"","meta":{"_crdt_document":"","version":"0.9.1","stable_tag":"0.9.1","tested":"6.9.4","requires":"6.0","requires_php":"7.4","requires_plugins":null,"header_name":"Crawler Record","header_author":"dizzysoft","header_description":"Are the most common search and chat (AI\/LLM) bots able to access the pages on your website? Crawler Record can tell you the last time each of the most common search\/chat bots visited \u2014and which pages they looked at.","assets_banners_color":"6e787d","last_updated":"2026-01-07 16:00:01","external_support_url":"","external_repository_url":"","donate_link":"","header_plugin_uri":"https:\/\/www.dizzysoft.com\/crawler-record-plugin-for-wordpress\/","header_author_uri":"https:\/\/www.dizzysoft.com\/","rating":5,"author_block_rating":0,"active_installs":60,"downloads":608,"num_ratings":1,"support_threads":0,"support_threads_resolved":0,"author_block_count":0,"sections":["description","installation","faq","changelog"],"tags":{"0.5.0":{"tag":"0.5.0","author":"dizzysoft","date":"2025-09-16 14:27:42"},"0.6.0":{"tag":"0.6.0","author":"dizzysoft","date":"2025-09-23 14:43:53"},"0.7.0":{"tag":"0.7.0","author":"dizzysoft","date":"2025-09-29 18:15:21"},"0.8.0":{"tag":"0.8.0","author":"dizzysoft","date":"2025-11-21 18:11:29"},"0.9.0":{"tag":"0.9.0","author":"dizzysoft","date":"2025-12-19 18:24:55"},"0.9.1":{"tag":"0.9.1","author":"dizzysoft","date":"2026-01-07 16:00:01"}},"upgrade_notice":[],"ratings":{"1":0,"2":0,"3":0,"4":0,"5":1},"assets_icons":{"icon-128x128.png":{"filename":"icon-128x128.png","revision":3362637,"resolution":"128x128","location":"assets","locale":""},"icon-256x256.png":{"filename":"icon-256x256.png","revision":3362637,"resolution":"256x256","location":"assets","locale":""}},"assets_banners":{"banner-772x250.png":{"filename":"banner-772x250.png","revision":3362637,"resolution":"772x250","location":"assets","locale":""}},"assets_blueprints":{},"all_blocks":[],"tagged_versions":["0.5.0","0.6.0","0.7.0","0.8.0","0.9.0","0.9.1"],"block_files":[],"assets_screenshots":[],"screenshots":[],"jetpack_post_was_ever_published":false},"plugin_section":[],"plugin_tags":[241430,13864,247966,6499,186],"plugin_category":[55],"plugin_contributors":[247967],"plugin_business_model":[],"class_list":["post-252378","plugin","type-plugin","status-publish","hentry","plugin_tags-bingbot","plugin_tags-googlebot","plugin_tags-gptbot","plugin_tags-robots","plugin_tags-seo","plugin_category-seo-and-marketing","plugin_contributors-dizzysoft","plugin_committers-dizzysoft"],"banners":{"banner":"https:\/\/ps.w.org\/crawler-record\/assets\/banner-772x250.png?rev=3362637","banner_2x":false,"banner_rtl":false,"banner_2x_rtl":false},"icons":{"svg":false,"icon":"https:\/\/ps.w.org\/crawler-record\/assets\/icon-128x128.png?rev=3362637","icon_2x":"https:\/\/ps.w.org\/crawler-record\/assets\/icon-256x256.png?rev=3362637","generated":false},"screenshots":[],"raw_content":"<!--section=description-->\n<p><strong>Crawler Record<\/strong> logs the last time specific user agents (for search and AI chat\/LLMs) accessed your content including:<\/p>\n\n<ul>\n<li>Google<\/li>\n<li>Bing<\/li>\n<li>ChatGPT (OpenAI)<\/li>\n<li>Claude (Anthropic)<\/li>\n<li>Perplexity<\/li>\n<li>DuckDuckGo<\/li>\n<li>Meta<\/li>\n<li>Apple<\/li>\n<\/ul>\n\n<p>You can view this information:<\/p>\n\n<ul>\n<li><strong>Admin page<\/strong>: a grouped list of all crawlers, with last seen time, last page, and robots status for the crawler.  <\/li>\n<li><strong>Admin Bar (front-end)<\/strong>: quick-glance status for the current singular or URL context (no dropdowns in wp-admin).<\/li>\n<li><strong>Per Page\/Post<\/strong>: From the edit screen on any page or post, you can see that last time each crawler visited that page.<\/li>\n<\/ul>\n\n<p><strong>Robots-aware:<\/strong> The plugin checks  your <strong>robots.txt<\/strong> and evaluates <strong>Allow\/Disallow<\/strong> rules for a given path. If <strong>Settings \u2192 Reading \u2192 \u201cDiscourage search engines\u201d<\/strong> is enabled, all agents are shown as blocked with a prominent warning.<\/p>\n\n<p><strong>Performance-friendly by design:<\/strong> Write-throttling (default 10 minutes) and an auxiliary \u201clast post ID per agent\u201d record avoids heavy admin queries on large sites.<\/p>\n\n<p><strong>Privacy-friendly:<\/strong> Saves only bot visit timestamps and last URLs crawled \u2014 no personal data.<\/p>\n\n<p>Learn <a href=\"https:\/\/www.dizzysoft.com\/crawler-record-plugin-for-wordpress\/\"><strong>how to use this plugin<\/strong><\/a>.<\/p>\n\n<h4>Highlights<\/h4>\n\n<ul>\n<li>Supports common user-agent variants for Google, Bing, OpenAI (ChatGPT), Anthropic (Claude), Perplexity, Meta, Apple, and DuckDuckGo.<\/li>\n<li>Robots status computed from local robots.txt (physical or virtual) without outbound requests.<\/li>\n<li>Clear UI with grouped sections, microsecond timestamps, and a small diagnostics toggle showing the matched robots group and rule.<\/li>\n<\/ul>\n\n<h4>Known limitations<\/h4>\n\n<ul>\n<li>robots.txt <strong>wildcards<\/strong> (<code>*<\/code>) and end-of-line marker (<code>$<\/code>) are <strong>not<\/strong> interpreted; matching is prefix-based only. Future versions may add full spec support.<\/li>\n<\/ul>\n\n<h3>Privacy<\/h3>\n\n<p>This plugin stores:\n- <strong>Timestamps<\/strong> of crawler visits (float, with microseconds)\n- <strong>Last URL<\/strong> seen per crawler (per-URL records)\n- <strong>Last post ID<\/strong> per crawler (for admin performance)<\/p>\n\n<p>It does <strong>not<\/strong> collect or store personal data about site visitors. No data is transmitted to third parties.<\/p>\n\n<h3>License<\/h3>\n\n<p>GPLv2 or later. See LICENSE file.<\/p>\n\n<!--section=installation-->\n<ol>\n<li>Upload the plugin folder to <code>\/wp-content\/plugins\/<\/code>, or install via the admin Plugins screen.<\/li>\n<li>Activate the plugin.<\/li>\n<li>Visit <strong>Crawler Record<\/strong> under <strong>Admin \u2192 Crawler Record<\/strong> to review crawler activity.<\/li>\n<\/ol>\n\n<!--section=faq-->\n<dl>\n<dt id='why%20are%20there%20no%20crawler%20visits%20recorded%3F'><h3>Why are there no crawler visits recorded?<\/h3><\/dt>\n<dd><p>This plugin can only begin to track crawler\/bot visits from the time you have installed the plugun- not before. It may take several days or weeks (depending upon the popularity of your website) before any come for a visit.<\/p><\/dd>\n<dt id='why%20do%20i%20see%20%E2%80%9Cblocked%20by%20wordpress%20setting%E2%80%9D%3F'><h3>Why do I see \u201cBlocked by WordPress setting\u201d?<\/h3><\/dt>\n<dd><p>If <strong>Settings \u2192 Reading \u2192 Discourage search engines<\/strong> is enabled, so none of these systems are allowed to read the pages on your site.<\/p><\/dd>\n\n<\/dl>\n\n<!--section=changelog-->\n<h4>0.9.1<\/h4>\n\n<ul>\n<li>Fixed error on any WP archive pages (pages that list posts).<\/li>\n<\/ul>\n\n<h4>0.9.0<\/h4>\n\n<ul>\n<li>Updated for WordPress 6.9<\/li>\n<li>Now monitoring for Meta and Apple User Agents<\/li>\n<li>More accurate site-wide UA reporting.<\/li>\n<li>Ensured video tutorial appears on all admin screens.<\/li>\n<li>Fixed small code errors.<\/li>\n<\/ul>\n\n<h4>0.8.0<\/h4>\n\n<ul>\n<li>Google updated its useragents so I updated the matching strings to account for these changes.<\/li>\n<\/ul>\n\n<h4>0.7.0<\/h4>\n\n<ul>\n<li>The robots.txt checker wasn't actually working. It does now.<\/li>\n<li>When you look at the report in the admin section, the robots checker is looking for a site-wide rule; when you look at the back-end of a page, the robots checker is looking at that particular page.<\/li>\n<li>If a page is blocked by the robots.txt file, a link appears sending you to the robots.txt file.<\/li>\n<li>Added a video to explain how to use this plugin.<\/li>\n<\/ul>\n\n<h4>0.6.0<\/h4>\n\n<ul>\n<li>Clarified distinction between Googlebots.<\/li>\n<li>Better distinguishes Bingbots.<\/li>\n<li>Cosmetic changes to page in admin section.<\/li>\n<li>More clear documentation.<\/li>\n<\/ul>\n\n<h4>0.5.0<\/h4>\n\n<ul>\n<li>First public release<\/li>\n<\/ul>","raw_excerpt":"Crawler Record tells you the last time each of the most common search\/chat bots visited (Google, ChatGPT, etc)- and the pages at which they looked.","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin\/252378","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin"}],"about":[{"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/types\/plugin"}],"replies":[{"embeddable":true,"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/comments?post=252378"}],"author":[{"embeddable":true,"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wporg\/v1\/users\/dizzysoft"}],"wp:attachment":[{"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/media?parent=252378"}],"wp:term":[{"taxonomy":"plugin_section","embeddable":true,"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_section?post=252378"},{"taxonomy":"plugin_tags","embeddable":true,"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_tags?post=252378"},{"taxonomy":"plugin_category","embeddable":true,"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_category?post=252378"},{"taxonomy":"plugin_contributors","embeddable":true,"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_contributors?post=252378"},{"taxonomy":"plugin_business_model","embeddable":true,"href":"https:\/\/tah.wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_business_model?post=252378"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}