{
  "format": "scrubnet-page",
  "version": 1,
  "language": "en-GB",
  "title": "Crawler Observatory & Live Data | Scrubnet",
  "description": "Explore Scrubnet's site-wide crawler logs, freshness data and public feeds: an open research environment supporting technical SEO and web development insight.",
  "source": "https://scrubnet.org/observatory.html",
  "representations": {
    "html": "https://scrubnet.org/observatory.html",
    "markdown": "https://scrubnet.org/observatory.md",
    "text": "https://scrubnet.org/observatory.txt",
    "json": "https://scrubnet.org/observatory.json"
  },
  "languageAlternates": {
    "en-GB": "https://scrubnet.org/observatory.html",
    "fr-FR": "https://scrubnet.org/fr/observatoire.html"
  },
  "content": [
    {
      "type": "section",
      "label": "Scrubnet crawler observatory",
      "content": [
        {
          "type": "heading",
          "level": 1,
          "content": [
            {
              "type": "text",
              "text": "Crawler Observatory"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "Site-wide request data and public feeds for observing search and AI crawler behaviour"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "link",
              "url": "https://scrubnet.org/dashboard.php",
              "content": [
                {
                  "type": "text",
                  "text": "Explore Live Logs"
                }
              ]
            }
          ]
        }
      ]
    },
    {
      "type": "section",
      "id": "about-observatory",
      "content": [
        {
          "type": "heading",
          "level": 2,
          "content": [
            {
              "type": "text",
              "text": "A research environment within Scrubnet"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "The crawler observatory is Scrubnet's open environment for examining how verified search and AI crawlers discover, fetch and revisit resources across the Scrubnet domain, including public machine-readable feeds. It complements our tools and articles with first-party request data that technical SEO specialists, developers and researchers can inspect directly."
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "Observatory findings inform practical guidance and new questions for investigation. They remain one part of Scrubnet's wider work to create better ways of inspecting, understanding and working with the technical web."
            }
          ]
        }
      ]
    },
    {
      "type": "section",
      "content": [
        {
          "type": "heading",
          "level": 2,
          "id": "observatory-resources",
          "content": [
            {
              "type": "text",
              "text": "Explore the observatory"
            }
          ]
        },
        {
          "type": "linkGroup",
          "url": "https://scrubnet.org/dashboard.php",
          "label": "Open the live crawler logs",
          "content": [
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "image",
                  "url": "https://scrubnet.org/crawler-log-dashboard.webp",
                  "alt": "Live Crawler Logs dashboard showing crawler request charts"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Live data"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Live Crawler Logs"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Filter verified requests by crawler, path, format, response and date."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Updated throughout the day"
                }
              ]
            }
          ]
        },
        {
          "type": "linkGroup",
          "url": "https://scrubnet.org/freshness-lab.php",
          "label": "Open the Freshness Observatory",
          "content": [
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "image",
                  "url": "https://scrubnet.org/freshness-observatory.webp",
                  "alt": "Freshness Observatory dashboard showing crawler request data by content age"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Interactive analysis"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Freshness Observatory"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Explore request timing, content age, formats and crawler distributions."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Interactive crawler data"
                }
              ]
            }
          ]
        },
        {
          "type": "linkGroup",
          "url": "https://scrubnet.org/llms.html",
          "label": "Explore Scrubnet public crawler feeds",
          "content": [
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Open infrastructure"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Public Crawler Feeds"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Inspect the consistent machine-readable surface used by the observatory."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "HTML, JSON, TXT and Markdown"
                }
              ]
            }
          ]
        },
        {
          "type": "linkGroup",
          "url": "https://scrubnet.org/brands.html",
          "label": "Add a site to the Scrubnet observatory",
          "content": [
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Participate"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Add a Site"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Contribute authorised public content and broaden the observable dataset."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Free for eligible sites"
                }
              ]
            }
          ]
        }
      ]
    },
    {
      "type": "section",
      "id": "practice",
      "content": [
        {
          "type": "heading",
          "level": 2,
          "content": [
            {
              "type": "text",
              "text": "From observation to practical insight"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "Requests across Scrubnet are examined alongside resource formats, timestamps, status codes and content changes. Useful patterns become documented observations, technical articles and questions that can improve audits, publishing systems and crawler-aware development workflows."
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "link",
              "url": "https://scrubnet.org/discoveries.html",
              "content": [
                {
                  "type": "text",
                  "text": "Read Articles & Research"
                }
              ]
            },
            {
              "type": "text",
              "text": " "
            },
            {
              "type": "link",
              "url": "https://scrubnet.org/partnerships.html",
              "content": [
                {
                  "type": "text",
                  "text": "Collaborate With Scrubnet"
                }
              ]
            }
          ]
        }
      ]
    },
    {
      "type": "section",
      "id": "scrubberduck",
      "content": [
        {
          "type": "heading",
          "level": 2,
          "content": [
            {
              "type": "text",
              "text": "Meet ScrubberDuck"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "strong",
              "content": [
                {
                  "type": "text",
                  "text": "ScrubberDuck"
                }
              ]
            },
            {
              "type": "text",
              "text": " is the lightweight, robots-aware feed compiler behind the observatory. It collects authorised public content and creates consistent feeds for observing discovery, formats, freshness signals and recrawl behaviour while minimising unnecessary requests."
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "image",
              "url": "https://scrubnet.org/scrubberduck-200.webp",
              "alt": "Illustration of ScrubberDuck, the feed compiler used by Scrubnet"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "strong",
              "content": [
                {
                  "type": "text",
                  "text": "User-Agent:"
                }
              ]
            },
            {
              "type": "code",
              "text": "ScrubberDuck/1.0 (+https://scrubnet.org)"
            }
          ]
        }
      ]
    },
    {
      "type": "section",
      "id": "limits",
      "content": [
        {
          "type": "heading",
          "level": 2,
          "content": [
            {
              "type": "text",
              "text": "Interpreting the data"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "A request confirms that a resource was fetched from Scrubnet. It does not by itself prove indexing, ranking, model training, retrieval, citation or a crawler's reason for visiting. Scrubnet reports descriptive findings, documents relevant limitations and keeps observable evidence separate from assumptions."
            }
          ]
        }
      ]
    }
  ],
  "structuredData": [
    {
      "@context": "https://schema.org",
      "@graph": [
        {
          "@type": "CollectionPage",
          "@id": "https://scrubnet.org/observatory.html#page",
          "url": "https://scrubnet.org/observatory.html",
          "name": "Scrubnet Crawler Observatory",
          "description": "Site-wide crawler logs, freshness data and public feeds supporting evidence-led technical SEO and web development insight.",
          "isPartOf": {
            "@id": "https://scrubnet.org/#website"
          },
          "about": {
            "@id": "https://scrubnet.org/#org"
          },
          "mainEntity": {
            "@id": "https://scrubnet.org/observatory.html#resources"
          },
          "inLanguage": "en-GB"
        },
        {
          "@type": "ItemList",
          "@id": "https://scrubnet.org/observatory.html#resources",
          "numberOfItems": 4,
          "itemListElement": [
            {
              "@type": "ListItem",
              "position": 1,
              "url": "https://scrubnet.org/dashboard.php",
              "name": "Live Crawler Logs"
            },
            {
              "@type": "ListItem",
              "position": 2,
              "url": "https://scrubnet.org/freshness-lab.php",
              "name": "Freshness Observatory"
            },
            {
              "@type": "ListItem",
              "position": 3,
              "url": "https://scrubnet.org/llms.html",
              "name": "Public Crawler Feeds"
            },
            {
              "@type": "ListItem",
              "position": 4,
              "url": "https://scrubnet.org/brands.html",
              "name": "Add a Site"
            }
          ]
        },
        {
          "@type": "WebSite",
          "@id": "https://scrubnet.org/#website",
          "url": "https://scrubnet.org/",
          "name": "Scrubnet",
          "publisher": {
            "@id": "https://scrubnet.org/#org"
          },
          "inLanguage": "en-GB"
        },
        {
          "@type": "Organization",
          "@id": "https://scrubnet.org/#org",
          "name": "Scrubnet Ltd",
          "url": "https://scrubnet.org/",
          "description": "An independent toolkit and knowledge hub for technical SEO specialists and web developers.",
          "logo": {
            "@type": "ImageObject",
            "url": "https://scrubnet.org/scrubberduck-72.webp"
          }
        }
      ]
    },
    {
      "@context": "https://schema.org",
      "@type": "BreadcrumbList",
      "@id": "https://scrubnet.org/observatory.html#breadcrumbs",
      "itemListElement": [
        {
          "@type": "ListItem",
          "position": 1,
          "name": "Home",
          "item": "https://scrubnet.org/"
        },
        {
          "@type": "ListItem",
          "position": 2,
          "name": "Observatory",
          "item": "https://scrubnet.org/observatory.html"
        }
      ]
    }
  ]
}