{
  "format": "scrubnet-page",
  "version": 1,
  "language": "en-GB",
  "title": "The Ultimate Technical AI Search Optimisation Guide",
  "description": "Technical AI search guidance covering crawler access, rendering, structured data, freshness, measurement, governance and implementation.",
  "source": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html",
  "representations": {
    "html": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html",
    "markdown": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.md",
    "text": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.txt",
    "json": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.json"
  },
  "languageAlternates": {
    "en-GB": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html",
    "fr-FR": "https://scrubnet.org/fr/articles/guide-optimisation-technique-recherche-ia.html"
  },
  "content": [
    {
      "type": "section",
      "label": "Article hero",
      "content": [
        {
          "type": "heading",
          "level": 1,
          "content": [
            {
              "type": "text",
              "text": "The Ultimate Technical AI Search Optimisation Guide"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "A practical technical framework for AI crawlers, retrieval systems, answer engines, and agentic browsing."
            }
          ]
        }
      ]
    },
    {
      "type": "aside",
      "label": "Article author",
      "content": [
        {
          "type": "paragraph",
          "content": [
            {
              "type": "image",
              "url": "https://scrubnet.org/charles-vayssie.webp",
              "alt": "Charles Vayssié, founder of Scrubnet"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "By "
            },
            {
              "type": "link",
              "url": "https://scrubnet.org/about-us.html",
              "content": [
                {
                  "type": "text",
                  "text": "Charles Vayssié"
                }
              ]
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "Founder, Technical SEO & Web Developer"
            }
          ]
        },
        {
          "type": "paragraph",
          "content": [
            {
              "type": "text",
              "text": "Technical SEO specialist and web developer leading strategy and delivery across complex e-commerce and international websites, with expertise in migrations, rendering, crawl management, structured data and Core Web Vitals."
            }
          ]
        }
      ]
    },
    {
      "type": "article",
      "content": [
        {
          "type": "section",
          "id": "guide-intro",
          "content": [
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Last updated: "
                },
                {
                  "type": "time",
                  "datetime": "2026-06-07",
                  "content": [
                    {
                      "type": "text",
                      "text": "7 June 2026"
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI search optimisation extends technical SEO for systems that crawl, retrieve, chunk, embed, rank, reason, summarise and cite. This guide collects the technical foundations that make content easier to access, extract, understand, refresh, verify and use as evidence."
                }
              ]
            }
          ]
        },
        {
          "type": "details",
          "summary": [
            {
              "type": "text",
              "text": "Contents"
            }
          ],
          "content": [
            {
              "type": "list",
              "ordered": true,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#executive-summary",
                          "content": [
                            {
                              "type": "text",
                              "text": "Executive summary"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#what-ai-search-optimisation-is",
                          "content": [
                            {
                              "type": "text",
                              "text": "What AI search optimisation is"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#the-ai-search-pipeline",
                          "content": [
                            {
                              "type": "text",
                              "text": "The AI search pipeline"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#documented-facts-versus-sensible-inference",
                          "content": [
                            {
                              "type": "text",
                              "text": "Documented facts versus sensible inference"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#crawler-categories-and-access-policy",
                          "content": [
                            {
                              "type": "text",
                              "text": "Crawler categories and access policy"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#recommended-robots-txt-strategy",
                          "content": [
                            {
                              "type": "text",
                              "text": "Recommended robots.txt strategy"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#rendering-and-extraction",
                          "content": [
                            {
                              "type": "text",
                              "text": "Rendering and extraction"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#canonicalisation-and-duplicate-control",
                          "content": [
                            {
                              "type": "text",
                              "text": "Canonicalisation and duplicate control"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#chunkability-and-passage-level-retrieval",
                          "content": [
                            {
                              "type": "text",
                              "text": "Chunkability and passage-level retrieval"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#content-that-supports-ai-answers",
                          "content": [
                            {
                              "type": "text",
                              "text": "Content that supports AI answers"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#structured-data-and-entity-consistency",
                          "content": [
                            {
                              "type": "text",
                              "text": "Structured data and entity consistency"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#freshness-and-change-signalling",
                          "content": [
                            {
                              "type": "text",
                              "text": "Freshness and change signalling"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#machine-readable-discovery-paths",
                          "content": [
                            {
                              "type": "text",
                              "text": "Machine-readable discovery paths"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#performance-and-fetch-reliability",
                          "content": [
                            {
                              "type": "text",
                              "text": "Performance and fetch reliability"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#accessibility-and-agentic-usability",
                          "content": [
                            {
                              "type": "text",
                              "text": "Accessibility and agentic usability"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#measurement-framework",
                          "content": [
                            {
                              "type": "text",
                              "text": "Measurement framework"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#governance-and-legal-risk",
                          "content": [
                            {
                              "type": "text",
                              "text": "Governance and legal risk"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#technical-audit-checklist",
                          "content": [
                            {
                              "type": "text",
                              "text": "Technical audit checklist"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#prioritised-implementation-plan",
                          "content": [
                            {
                              "type": "text",
                              "text": "Prioritised implementation plan"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#useful-reporting-dashboard-structure",
                          "content": [
                            {
                              "type": "text",
                              "text": "Useful reporting dashboard structure"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#myths-and-corrections",
                          "content": [
                            {
                              "type": "text",
                              "text": "Myths and corrections"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#reference-implementation-snippets",
                          "content": [
                            {
                              "type": "text",
                              "text": "Reference implementation snippets"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#final-strategic-model",
                          "content": [
                            {
                              "type": "text",
                              "text": "Final strategic model"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#seo-scrubbox-extension",
                          "content": [
                            {
                              "type": "text",
                              "text": "SEO Scrubbox extension"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "link",
                          "url": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#sources",
                          "content": [
                            {
                              "type": "text",
                              "text": "Sources"
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "figure",
          "image": {
            "url": "https://scrubnet.org/discoveries/ultimate-tech-ai-search-optimisation-guide.webp",
            "alt": "The Ultimate Technical AI Search Optimisation Guide"
          },
          "caption": [
            {
              "type": "text",
              "text": "Helping structure the world for the AI era."
            }
          ]
        },
        {
          "type": "section",
          "id": "executive-summary",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "1. Executive summary"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI search optimisation is not a replacement for technical SEO. It is technical SEO extended for systems that crawl, retrieve, chunk, embed, rank, reason, summarise and cite."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "The strongest current evidence from Google, OpenAI, Anthropic, Perplexity, Apple, Bing, Common Crawl, enterprise retrieval documentation and RAG research points to one practical conclusion:"
                }
              ]
            },
            {
              "type": "quote",
              "content": [
                {
                  "type": "paragraph",
                  "content": [
                    {
                      "type": "text",
                      "text": "The sites most likely to perform well in AI search are the sites whose content can be reliably accessed, extracted, canonicalised, chunked, understood, refreshed, verified and cited."
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "This means the priorities are not magic AI tags or prompt-targeted content farms. The priorities are:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": true,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Let the right crawlers access the right content."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Keep essential content available in the initial HTML."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Use canonical, indexable URLs with clean sitemap coverage."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Structure pages so they split cleanly into useful evidence passages."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Use visible facts, dates, prices, entities and relationships."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Reinforce visible content with accurate JSON-LD."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Keep freshness signals honest and precise."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Separate training bot policy from AI search bot policy."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Measure server-side bot activity, referral traffic and prompt visibility."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Treat robots.txt as policy signalling, not security."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Google currently says there are no extra technical requirements to appear in AI Overviews or AI Mode beyond standard Google Search eligibility. But the retrieval layer changes how pages are used. Traditional search asks, \"Should this page rank?\" AI search also asks, \"Can this passage support a reliable answer?\""
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "That is the key shift."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "what-ai-search-optimisation-is",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "2. What AI search optimisation is"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI search optimisation is the process of improving a website so AI retrieval systems can:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "discover the right URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "fetch them without friction"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "extract the main content"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "understand the entities and relationships"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "split content into useful passages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "retrieve those passages for relevant queries"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "trust the facts enough to use them"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "cite the source or link back when appropriate"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "keep answers current when content changes"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "It includes classic SEO, but it places more emphasis on machine extraction, factual clarity, semantic structure, freshness and crawler governance."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "What it is not"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI search optimisation is not:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "keyword stuffing"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "creating thousands of thin prompt pages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "adding unsupported \"best\" or \"number one\" claims"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "relying on llms.txt alone"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "assuming every AI crawler renders JavaScript"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "assuming analytics tools capture all AI activity"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "blocking all AI bots while expecting AI search visibility"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "allowing all AI bots without thinking about training, licensing and load"
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "the-ai-search-pipeline",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "3. The AI search pipeline"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI search systems differ by provider, but most follow a similar pipeline."
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "mermaid",
              "text": "flowchart TD\n    A[User prompt or query] --> B[Query understanding]\n    B --> C[Query fan-out and sub-queries]\n    C --> D[Discovery through search indexes, sitemaps, links, feeds and direct fetches]\n    D --> E[HTTP fetch]\n    E --> F[Canonicalisation and duplicate clustering]\n    F --> G[Content extraction]\n    G --> H[Chunking into passages]\n    H --> I[Lexical, semantic and metadata representation]\n    I --> J[Hybrid retrieval]\n    J --> K[Re-ranking, freshness checks, policy checks and trust assessment]\n    K --> L[Answer synthesis]\n    L --> M[Citations, links, summaries or actions]"
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "The important point is that AI systems do not only deal with complete pages. They often deal with passages, entities, attributes, dates, snippets, media, schema fields and metadata."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "A strong AI-search page is therefore not just a page that can rank. It is a page that can be used as evidence."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "documented-facts-versus-sensible-inference",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "4. Documented facts versus sensible inference"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "A lot of public discussion around AI search mixes official documentation, experiments, vendor claims and speculation. Keep the distinction clear."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Strongly documented"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These are well supported by public provider documentation:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Google AI Overviews and AI Mode use normal Google Search eligibility. A page must be indexed and eligible for a snippet to appear as a supporting link."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Google says there are no extra special AI files or special schema required for AI features in Search."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Google AI Overviews and AI Mode may use query fan-out, issuing related searches across subtopics and data sources."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "OpenAI separates OAI-SearchBot for ChatGPT search, GPTBot for training and ChatGPT-User for user-triggered actions."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Anthropic separates ClaudeBot for training, Claude-SearchBot for search quality and Claude-User for user-initiated retrieval."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Perplexity separates PerplexityBot for search results from Perplexity-User for user-requested fetches."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Applebot powers Apple search experiences, while Applebot-Extended is a control for whether Applebot-crawled content can be used for Apple foundation-model training."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Bing has emphasised XML sitemaps, accurate lastmod and IndexNow for AI-powered search freshness."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robots.txt is not access authorisation. It is a protocol that crawlers are requested to honour."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Reasonable inference"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These are not fully disclosed for public consumer AI search, but are strongly suggested by RAG literature and enterprise retrieval systems:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "AI search systems often operate on passages rather than whole pages."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Hybrid retrieval is likely important, combining keyword search, semantic search and metadata filters."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Clear headings and short, scoped sections make chunking and retrieval easier."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Dates, named entities, product identifiers and structured attributes help exact retrieval and filtering."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Duplicate URLs can cause the wrong version of a page or product to be selected as source material."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Freshness signals are more important when an answer is generated from retrieved evidence."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Unknown or provider-specific"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These are not reliably public:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "exact chunk sizes for consumer AI search indexes"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "exact embedding models used by each public AI search system"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "exact citation-ranking weights"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "exact source selection logic for AI Overviews, ChatGPT, Claude or Perplexity"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "whether a specific platform renders JavaScript in every context"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "whether llms.txt materially affects visibility on major public AI search engines"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Treat hidden internals as unknown. Optimise around robust principles instead."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "crawler-categories-and-access-policy",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "5. Crawler categories and access policy"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Do not treat all AI bots the same. They have different purposes."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "5.1 Automatic search and retrieval bots"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These crawlers are most closely connected to AI search visibility, citations and live answers."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Examples:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "OAI-SearchBot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Claude-SearchBot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "PerplexityBot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Googlebot for Google Search AI features"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Applebot for Apple search experiences"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These should usually be allowed if the goal is visibility in AI search and answer engines."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "5.2 Training bots and training controls"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These are used, or can be used, to collect public content for model training or model improvement."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Examples:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "GPTBot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "ClaudeBot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "CCBot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Google-Extended as a control token for some Google Gemini training and grounding uses"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Applebot-Extended as a control for Apple foundation-model training use"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Allowing training bots is a commercial, legal and licensing decision. It is not the same decision as allowing AI search visibility."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "5.3 User-triggered fetchers"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These fetch pages because a user or agent requested them."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Examples:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "ChatGPT-User"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Claude-User"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Perplexity-User"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Google user-triggered fetchers"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "agentic browser fetchers"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These may behave differently from normal crawlers and may not obey robots.txt in the same way, because the fetch is user-directed. Manage them with a combination of server-side monitoring, rate limiting, authentication, WAF rules, API controls and robots policy where supported."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "5.4 Agentic action bots"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "These are not just reading pages. They may interact with forms, carts, booking flows, logins, filters, APIs and on-site tools."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "For these, classic SEO checks are not enough. You also need:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "accessible form labels"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robust ARIA where appropriate"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "clear button states"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "stable URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "predictable error messages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "safe rate limits"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "CSRF protection"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "permission checks"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "separation of public content from protected workflows"
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "recommended-robots-txt-strategy",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "6. Recommended robots.txt strategy"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Use robots.txt to express policy by purpose."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "This is an illustrative policy for a site that wants AI search visibility but wants to limit model training."
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "txt",
              "text": "# robots.txt\n\nUser-agent: *\nDisallow:\n\nSitemap: https://www.example.com/sitemap.xml\n\n# Google Search and Google AI features in Search\nUser-agent: Googlebot\nAllow: /\n\n# Google-Extended controls some Gemini training and grounding uses.\n# It is not the control for Google Search AI Overviews or AI Mode.\nUser-agent: Google-Extended\nDisallow: /\n\n# ChatGPT search visibility\nUser-agent: OAI-SearchBot\nAllow: /\n\n# OpenAI training\nUser-agent: GPTBot\nDisallow: /\n\n# Anthropic search visibility\nUser-agent: Claude-SearchBot\nAllow: /\n\n# Anthropic training\nUser-agent: ClaudeBot\nDisallow: /\n\n# Anthropic user-directed retrieval.\n# Decide based on whether you want Claude to fetch pages for user queries.\nUser-agent: Claude-User\nAllow: /\n\n# Perplexity search visibility\nUser-agent: PerplexityBot\nAllow: /\n\n# Apple search visibility\nUser-agent: Applebot\nAllow: /\n\n# Apple foundation-model training control\nUser-agent: Applebot-Extended\nDisallow: /\n\n# Common Crawl dataset collection\nUser-agent: CCBot\nDisallow: /"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Important caveats"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robots.txt is not a security layer."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Unknown, malicious or spoofed bots may ignore it."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "User-agent matching is weak on its own."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Some user-triggered fetchers may generally ignore robots.txt."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "If the content is sensitive, private, licensed or paywalled, use real access control."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Some providers publish IP ranges or verification methods. Use them where possible."
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "rendering-and-extraction",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "7. Rendering and extraction"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "The first technical requirement is simple:"
                }
              ]
            },
            {
              "type": "quote",
              "content": [
                {
                  "type": "paragraph",
                  "content": [
                    {
                      "type": "text",
                      "text": "Put the important content and signals in the initial HTML wherever possible."
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI crawlers and retrieval systems do not all render pages consistently. Some may render. Some may not. Some may fetch only raw HTML. Some may time out before hydration. Some may rely on upstream indexes that have their own rendering limitations."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "7.1 Safe rendering hierarchy"
                }
              ]
            },
            {
              "type": "table",
              "rows": [
                {
                  "header": true,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Architecture"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "AI extraction risk"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Notes"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Static Site Generation"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Low"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Excellent for stable content, documentation, articles and landing pages."
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Server-Side Rendering"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Low"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Good for dynamic websites where content must be current at request time."
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Incremental Static Regeneration"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Low to medium"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Good balance of speed and freshness if regeneration is reliable."
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Progressive enhancement"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Low to medium"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Safe if core content exists before JavaScript enhancement."
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Client-Side Rendering only"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "High"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Risky if the initial HTML is a blank app shell."
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Content behind API calls only"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "High"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Risky unless the API response is also discoverable and indexable."
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Content hidden in interactive components only"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Medium to high"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Tabs, carousels, filters and accordions can obscure key facts."
                      }
                    ]
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "7.2 What must be present in HTML"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "For priority pages, aim to expose:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "H1 and section headings"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "main body copy"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product names and descriptions"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "prices, availability and currency"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "specifications and identifiers"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "internal links"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "canonical tags"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robots meta tags"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "hreflang where relevant"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "structured data"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "author and reviewer signals where relevant"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "publication and modification dates"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "important comparison data"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "FAQs and policy content"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "breadcrumbs"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "7.3 Semantic HTML pattern"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "html",
              "text": "<main>\n  <article>\n    <header>\n      <h1>Technical AI Search Optimisation Guide</h1>\n      <p class=\"summary\">\n        AI search visibility depends on crawl access, extractable HTML,\n        canonical clarity, structured data, freshness and measurable bot activity.\n      </p>\n      <p>Last updated: <time datetime=\"2026-06-07\">7 June 2026</time></p>\n    </header>\n\n    <section id=\"crawler-access\">\n      <h2>Control AI bot access by purpose</h2>\n      <p>\n        Separate search crawlers, training crawlers and user-triggered fetchers.\n      </p>\n    </section>\n  </article>\n</main>"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "7.4 Common extraction failures"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "blank app shell in source HTML"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "missing canonical in raw HTML"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "metadata only injected after hydration"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "content loaded only after scroll"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "key facts only shown inside images"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product specs only available behind tabs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "infinite scroll with no crawlable pagination"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "filters creating crawl traps"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "same content duplicated across many parameter URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "blocking JS or CSS needed for graceful rendering"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "WAF returning 403 to AI retrieval bots"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "server returning different content by user-agent without control"
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "canonicalisation-and-duplicate-control",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "8. Canonicalisation and duplicate control"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI search systems need to know which URL is the source of truth. Duplicate or near-duplicate pages make retrieval harder and can cause the wrong version to be selected."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "8.1 Canonical signals to align"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Use the same preferred URL across:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "internal links"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "XML sitemaps"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "rel canonical tags"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "redirects"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "hreflang annotations"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "structured data mainEntityOfPage or url"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Open Graph and social metadata"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product feeds"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "llms.txt or context files"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "canonical Markdown alternatives"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "external profiles where possible"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "8.2 Canonical HTML example"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "html",
              "text": "<head>\n  <title>How AI crawlers see JavaScript websites</title>\n  <link rel=\"canonical\" href=\"https://www.example.com/guides/ai-crawlers-javascript\" />\n  <meta name=\"robots\" content=\"index,follow,max-snippet:-1,max-image-preview:large\" />\n</head>"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "8.3 Duplicate patterns to clean up"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "tracking parameters"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "sort and filter URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "internal search pages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "duplicated product variants"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "print pages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "trailing slash inconsistency"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "HTTP and HTTPS duplication"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "www and non-www duplication"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "uppercase and lowercase duplication"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "duplicated locale paths"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "paginated pages canonicalising incorrectly"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "canonical tags that conflict with redirects or sitemaps"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "8.4 Why this matters more in AI search"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "In AI search, duplicate control is not only about ranking consolidation. It is also about source selection. If the system clusters duplicates and selects an outdated URL, that outdated page may become the evidence used in an answer."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "chunkability-and-passage-level-retrieval",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "9. Chunkability and passage-level retrieval"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Modern retrieval systems often work with chunks or passages. A page may rank as a whole, but the AI system retrieves a section."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Optimise pages so each section can stand alone as useful evidence."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "9.1 Good section design"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Each important section should have:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "a descriptive heading"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "one clear question or topic"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "a short direct answer near the top"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "supporting detail below"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "dates, units and named entities where relevant"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "minimal boilerplate"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "no dependency on previous sections for basic meaning"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "9.2 Bad section design"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Avoid:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "vague headings such as \"Overview\", \"Details\" or \"More information\""
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "long introductions before the answer"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "multiple unrelated topics in one section"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "claims without evidence"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "facts only implied through design"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "tables without textual context"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "pages where the first 500 words say almost nothing"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "9.3 Page structure template"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "md",
              "text": "# Main topic\n\nShort answer summary.\n\n## What is it?\n\nDirect definition.\n\n## Who is it for?\n\nSpecific audience and use cases.\n\n## How it works\n\nStep-by-step explanation.\n\n## Technical requirements\n\nConcrete implementation details.\n\n## Pricing or cost\n\nClear numbers, currency, exclusions and update date.\n\n## Limitations\n\nCaveats, edge cases and situations where it is not suitable.\n\n## Comparison with alternatives\n\nComparison table with honest differences.\n\n## Evidence and sources\n\nAwards, reviews, references, case studies, documentation links.\n\n## Last updated\n\nVisible date and explanation of meaningful changes."
            }
          ]
        },
        {
          "type": "section",
          "id": "content-that-supports-ai-answers",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "10. Content that supports AI answers"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI answer systems need usable facts, not just marketing prose."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "10.1 Use answer-led writing"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Start each page and major section with the answer, then add detail."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Poor:"
                }
              ]
            },
            {
              "type": "quote",
              "content": [
                {
                  "type": "paragraph",
                  "content": [
                    {
                      "type": "text",
                      "text": "In today's fast-moving digital landscape, brands are increasingly looking for innovative ways to improve discoverability..."
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Better:"
                }
              ]
            },
            {
              "type": "quote",
              "content": [
                {
                  "type": "paragraph",
                  "content": [
                    {
                      "type": "text",
                      "text": "AI search optimisation improves how AI systems access, extract, understand and cite website content. The core work is crawl access, server-rendered content, canonical clarity, structured data, freshness and measurement."
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "10.2 Write for decision-making"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI systems often answer comparison and recommendation queries. Support that directly."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Useful page types:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "What is X?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "How does X work?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "How much does X cost?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Is X suitable for Y?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "How to choose X"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "X vs Y"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Best X for Y"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Alternatives to X"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Common problems with X"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "X implementation checklist"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "X limitations"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "X compliance requirements"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "10.3 Make facts explicit"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Spell out:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "brand name"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "legal entity name"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product names"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product categories"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "target users"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "location and service areas"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "prices and currencies"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "dates and validity periods"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "stock and availability"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "model numbers and SKUs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "measurements and units"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "terms and exclusions"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "policies and limitations"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "10.4 Back strong claims with evidence"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Avoid unsupported claims such as:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "best"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "leading"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "number one"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "most trusted"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "fastest"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "cheapest"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "most accurate"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "If you use them, support them with:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "awards"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "independent reviews"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "certifications"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "customer numbers"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "case studies"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "benchmark methodology"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "third-party references"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "public datasets"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "transparent dates"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "10.5 GEO study takeaways"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Academic GEO research found that content modifications such as adding citations, quotations and statistics can improve visibility in generative responses, with reported gains up to around 40 percent in the benchmarked settings."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Practical takeaway:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add useful statistics, but only when true and relevant."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add expert quotes, but avoid generic filler quotes."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Cite credible sources for factual claims."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Improve fluency and structure."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Avoid keyword stuffing."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Optimise by topic and domain, not by one universal formula."
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "structured-data-and-entity-consistency",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "11. Structured data and entity consistency"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Structured data does not replace content quality, and it should not describe information that is not visible to users. Its job is to reinforce meaning."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "11.1 Core principle"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Use schema to make visible facts easier for machines to understand."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Good structured data is:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "accurate"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "visible on the page"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "consistent with feeds and profiles"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "stable across the site"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "specific to the page type"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "connected through persistent identifiers"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Bad structured data is:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "copied from templates without page-level accuracy"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "inconsistent with visible content"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "over-marked for rich results"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "missing dates or identifiers"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "disconnected from the brand entity"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "used to make claims the page does not support"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "11.2 Useful schema types"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Depending on the site, prioritise:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Organization"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "LocalBusiness and relevant subtypes"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Person"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Article"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "BlogPosting"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Product"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Offer"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "AggregateRating"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Review"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "FAQPage"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "QAPage"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "BreadcrumbList"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "ItemList"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "WebPage"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Service"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "HowTo where genuinely applicable"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "11.3 Entity home"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Every organisation should have a clear entity home, usually the About page or homepage. This page should state:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "official organisation name"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "brand name"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "what the organisation does"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "who it serves"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "where it operates"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "founding details if relevant"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "contact information"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "official social profiles"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "parent or subsidiary relationships"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "awards or certifications"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "authoritative third-party profiles"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "sameAs links"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "11.4 Connected JSON-LD pattern"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "html",
              "text": "<script type=\"application/ld+json\">\n{\n  \"@context\": \"https://schema.org\",\n  \"@graph\": [\n    {\n      \"@type\": \"Organization\",\n      \"@id\": \"https://www.example.com/#organization\",\n      \"name\": \"Example Tools\",\n      \"url\": \"https://www.example.com/\",\n      \"logo\": \"https://www.example.com/logo.png\",\n      \"sameAs\": [\n        \"https://www.linkedin.com/company/example-tools\"\n      ]\n    },\n    {\n      \"@type\": \"WebPage\",\n      \"@id\": \"https://www.example.com/guides/torque-wrench-calibration#webpage\",\n      \"url\": \"https://www.example.com/guides/torque-wrench-calibration\",\n      \"name\": \"How to calibrate a torque wrench\",\n      \"isPartOf\": {\n        \"@id\": \"https://www.example.com/#website\"\n      },\n      \"about\": {\n        \"@id\": \"https://www.example.com/#organization\"\n      },\n      \"datePublished\": \"2026-05-30\",\n      \"dateModified\": \"2026-06-07\"\n    },\n    {\n      \"@type\": \"Article\",\n      \"@id\": \"https://www.example.com/guides/torque-wrench-calibration#article\",\n      \"headline\": \"How to calibrate a torque wrench\",\n      \"mainEntityOfPage\": {\n        \"@id\": \"https://www.example.com/guides/torque-wrench-calibration#webpage\"\n      },\n      \"author\": {\n        \"@type\": \"Person\",\n        \"name\": \"Jordan Smith\"\n      },\n      \"publisher\": {\n        \"@id\": \"https://www.example.com/#organization\"\n      }\n    }\n  ]\n}\n</script>"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "11.5 Ecommerce schema checklist"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "For products, expose:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product name"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "clear description"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "image"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "brand"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "SKU"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "GTIN where available"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "price"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "currency"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "availability"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "item condition"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "delivery information"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "return policy where supported"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "valid review and rating data"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "variant attributes"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "canonical product URL"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Category pages should include:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "useful intro copy"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "clear H1 and H2s"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "crawlable product links"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "breadcrumbs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "ItemList where appropriate"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "buying guidance"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "FAQs where genuinely present"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "indexation and canonical rules for filters"
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "freshness-and-change-signalling",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "12. Freshness and change signalling"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI answers can become wrong when source material is stale. Freshness is therefore a stack of signals."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "12.1 Freshness signals to align"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Use:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "visible last updated date"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "datePublished and dateModified in structured data"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "accurate XML sitemap lastmod"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "HTTP Last-Modified where possible"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "ETag and conditional request support"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "changelog sections for critical pages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "IndexNow for Bing and participating engines"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "updated product feeds and merchant data"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "updated business profiles"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "current author and reviewer information"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "12.2 Sitemap lastmod example"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "xml",
              "text": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n  <url>\n    <loc>https://www.example.com/guides/ai-search-optimisation</loc>\n    <lastmod>2026-06-07T09:14:00+00:00</lastmod>\n  </url>\n</urlset>"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "12.3 Rules for honest lastmod"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Do:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "update lastmod only when the page content materially changes"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "use ISO 8601 timestamps where possible"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "include time zone"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "keep visible dates aligned with schema and sitemap dates"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "use changelogs when the topic is volatile"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Do not:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "set lastmod to the sitemap generation time"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "change all dates daily to look fresh"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "hide old content behind a new date"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "let schema and visible dates contradict each other"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "12.4 IndexNow example"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "bash",
              "text": "curl -X POST \"https://api.indexnow.org/indexnow\" \\\n  -H \"Content-Type: application/json; charset=utf-8\" \\\n  -d '{\n    \"host\": \"www.example.com\",\n    \"key\": \"YOUR_KEY\",\n    \"keyLocation\": \"https://www.example.com/YOUR_KEY.txt\",\n    \"urlList\": [\n      \"https://www.example.com/guides/ai-search-optimisation\",\n      \"https://www.example.com/products/example-product\"\n    ]\n  }'"
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Use IndexNow for important additions, updates and deletions, especially on ecommerce, news, jobs, listings, pricing, stock and other time-sensitive pages."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "machine-readable-discovery-paths",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "13. Machine-readable discovery paths"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Traditional discovery files still matter most:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robots.txt"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "XML sitemaps"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "RSS or Atom feeds where relevant"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "internal linking"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "canonical URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "structured data"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product feeds"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Merchant Center or equivalent feeds"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Business Profile data"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "13.1 XML sitemaps"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Keep XML sitemaps clean."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Include only:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "canonical URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "indexable URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "final 200 URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "important content"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "accurate lastmod"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Remove:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "redirected URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "noindexed URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "blocked URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "canonicalised-away URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "soft 404s"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "parameter noise"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "internal search URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "duplicate locale URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "expired product pages unless intentionally indexable"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "13.2 llms.txt"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "llms.txt is an emerging convention, not a guaranteed ranking or inclusion mechanism."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Use it as an enhancement, especially for documentation, developer tools, support hubs, complex products and technical sites."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "It can include:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "a concise site or product summary"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "links to canonical docs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "links to Markdown versions"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "API documentation"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "policy pages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product feeds"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "support resources"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "examples of important resources"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Example:"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "md",
              "text": "# Example Tools\n\n> Example Tools provides calibration equipment and technical guides for engineering teams.\n\n## Core pages\n\n- [About Example Tools](https://www.example.com/about)\n- [Product catalogue](https://www.example.com/products)\n- [Calibration guides](https://www.example.com/guides)\n\n## Clean Markdown resources\n\n- [Torque wrench calibration guide](https://www.example.com/guides/torque-wrench-calibration.md)\n- [Return policy](https://www.example.com/policies/returns.md)\n\n## API and feeds\n\n- [Product feed](https://www.example.com/feeds/products.json)\n- [OpenAPI specification](https://www.example.com/openapi.json)"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "13.3 llms-full.txt and context files"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "For some sites, a larger context file can help agents or developer tools. Keep it scoped and curated."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Include:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "stable documentation"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "canonical definitions"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "API references"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "short examples"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "policy summaries"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "support instructions"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Exclude:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "private content"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "customer data"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "unpublished commercial information"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "outdated content"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "duplicated pages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "legal text you do not want summarised incorrectly"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "massive unstructured dumps"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "13.4 Markdown alternatives"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Markdown can reduce token overhead and layout noise for machines."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Useful patterns:"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "txt",
              "text": "/page-name.html\n/page-name.md\n\n/docs/getting-started\n/docs/getting-started.md\n\n/api/reference\n/api/reference.md"
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Do not use Markdown alternatives as a substitute for normal HTML SEO. They should support the main site, not replace it."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "performance-and-fetch-reliability",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "14. Performance and fetch reliability"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI retrieval systems often work with short timeouts and strict cost constraints. Fast, stable pages are easier to fetch and reuse."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "14.1 Technical priorities"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Prioritise:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "fast TTFB"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "stable 200 responses"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "minimal redirects"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "CDN caching where appropriate"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "lightweight HTML"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "graceful degradation"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "correct cache headers"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robust origin capacity"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "no accidental WAF blocks"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "no bot-specific server errors"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "no soft 404s"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "no redirect chains"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "no inconsistent canonical signals"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "14.2 Status code rules"
                }
              ]
            },
            {
              "type": "table",
              "rows": [
                {
                  "header": true,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Status"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Use"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "AI search risk"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "200"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Live canonical content"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Good"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "301 or 308"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Permanent redirect"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Fine if single hop and intentional"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "302 or 307"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Temporary redirect"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Fine if genuinely temporary"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "304"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Not modified"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Good for efficient recrawling"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "401"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Authentication required"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Correct for private content"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "403"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Forbidden"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Dangerous if accidental bot block"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "404"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Gone or not found"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Correct for missing content"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "410"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Permanently gone"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Useful for deliberate removals"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "429"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Rate limited"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Useful with Retry-After"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "5xx"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Server failure"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "High risk if frequent"
                      }
                    ]
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "14.3 Caching headers"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Use:"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "http",
              "text": "Cache-Control: public, max-age=300, stale-while-revalidate=3600\nETag: \"abc123\"\nLast-Modified: Sun, 07 Jun 2026 09:14:00 GMT"
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Support conditional requests:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "If-None-Match"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "If-Modified-Since"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "This helps reduce repeated downloads and improves crawl efficiency."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "accessibility-and-agentic-usability",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "15. Accessibility and agentic usability"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Agentic systems increasingly interact with pages more like assistive technologies than classic crawlers. Accessibility improvements can help both users and agents."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Prioritise:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "clear labels on form fields"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "descriptive button text"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "ARIA only where needed and correctly implemented"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "keyboard navigation"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "visible error messages"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "stable page landmarks"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "form validation that is machine-readable"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "descriptive link text"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "clear table headers"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "meaningful alt text"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "no critical instructions hidden only in images"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "For transactional journeys, agents need to understand:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "what the next step is"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "what each field expects"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "what is required or optional"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "what went wrong"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "what action was completed"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "whether a price or booking is final"
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "measurement-framework",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "16. Measurement framework"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Standard analytics capture only part of AI visibility."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "You need three measurement layers."
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "mermaid",
              "text": "flowchart TD\n    A[AI visibility measurement] --> B[Human referral tracking]\n    A --> C[Server-side bot logging]\n    A --> D[Prompt-based share of voice]\n\n    B --> B1[GA4 channels]\n    B --> B2[UTM parameters]\n    B --> B3[Referral source domains]\n\n    C --> C1[CDN logs]\n    C --> C2[WAF logs]\n    C --> C3[Server logs]\n    C --> C4[Edge workers]\n\n    D --> D1[Brand mentioned?]\n    D --> D2[Cited URLs]\n    D --> D3[Sentiment]\n    D --> D4[Accuracy]\n    D --> D5[Competitors]"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "16.1 AI assistant referral tracking"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Track known referrers and UTM sources such as:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "chatgpt.com"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "openai.com"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "perplexity.ai"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "claude.ai"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "gemini.google.com"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "copilot.microsoft.com"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "bing.com where relevant"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Example GA4 custom channel regex:"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "regex",
              "text": ".*(chatgpt\\.com|openai\\.com|perplexity\\.ai|claude\\.ai|gemini\\.google\\.com|copilot\\.microsoft\\.com|bing\\.com).*"
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Limitations:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "AI Overview clicks may appear as Google organic."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Mobile apps may strip referrers."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Some AI browsers or agentic contexts may appear as direct."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Zero-click answers send no user session at all."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Crawlers and fetchers are not measured by client-side analytics."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "16.2 Server-side AI bot logging"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Log AI bot activity separately from human sessions."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Useful fields:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "timestamp"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "requested URL"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "method"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "status code"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "user-agent"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "detected bot name"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "bot category"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "verified bot status"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "referrer where present"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "country or region where appropriate"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "cache status"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "response time"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "response size"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robots policy result"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "WAF action"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "crawl purpose if classified"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Recommended categories:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "search and retrieval bot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "training bot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "user-triggered fetcher"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "agentic action bot"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "unknown AI-like traffic"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "spoofed or failed verification"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "non-AI bot"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "16.3 Bot verification"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Do not rely only on user-agent strings."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Use:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "published IP ranges"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "reverse DNS"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "forward-confirmed reverse DNS"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "ASN checks"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "TLS fingerprinting where available"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "request patterns"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robots.txt fetch behaviour"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "crawl rate behaviour"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "WAF verified bot categories"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "known provider JSON IP files"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "16.4 Privacy and GDPR notes"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Avoid exposing raw IP addresses in public or client-facing dashboards. For AI bot analytics, prefer:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "aggregation"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "truncation"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "hashing with rotating salt"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "short retention periods"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "role-based access"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "clear lawful basis"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "separation of security logs from marketing reports"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "For UK and EU contexts, treat IP addresses carefully as personal data where they can relate to individuals, even if many AI bot IPs are public infrastructure addresses."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "16.5 Prompt-based share of voice"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Track what AI systems say, not just traffic."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Measure:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "whether your brand appears"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "where it appears in the answer"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "whether it is cited"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "which URL is cited"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "whether competitors appear"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "sentiment and framing"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "factual accuracy"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "product, price and availability accuracy"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "whether old URLs are cited"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "whether the answer changes over time"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Example prompt set:"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "txt",
              "text": "What is the best [category] for [use case]?\nWhich [provider type] is suitable for [audience]?\nCompare [brand] with [competitor].\nHow much does [product] cost?\nIs [brand] available in [location]?\nWhat are the alternatives to [brand]?\nWhat are the main problems with [category]?"
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Run prompts regularly, but treat outputs as variable. AI answers can change by location, session, model, query phrasing and time."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "governance-and-legal-risk",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "17. Governance and legal risk"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "AI search visibility is not only an SEO decision. It touches copyright, licensing, privacy, infrastructure cost and commercial strategy."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "17.1 Separate the decisions"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Make separate policy decisions for:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "traditional search indexing"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "AI search inclusion"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "model training"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "user-triggered retrieval"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "agentic interaction"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Common Crawl and third-party datasets"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "paywalled content"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "premium documentation"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "personal data"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "partner or client content"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "17.2 robots.txt is not enough for sensitive content"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "If content must not be accessed, use:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "authentication"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "paywalls"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "signed URLs"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "firewall rules"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "contractual controls"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "licence terms"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "noindex where search exclusion is needed"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "removal requests where appropriate"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "WAF enforcement"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "access logs and alerts"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Do not rely on robots.txt to protect confidential or regulated material."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "17.3 UK and Google AI search controls"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "As of June 2026, Google has begun testing a new Search Console control in the UK that lets some website owners manage whether their content appears in and helps ground generative AI Search features such as AI Overviews and AI Mode. Sites that opt out of those generative AI features should not receive traffic or impressions from them, while Google says the control will not be used as a ranking signal outside those generative AI features."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "This is new and may be limited in availability while it is tested. Treat it as an evolving governance control, not a replacement for technical SEO, robots policy, snippets or access control."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "technical-audit-checklist",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "18. Technical audit checklist"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "18.1 Crawl and access"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Important pages return 200."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "robots.txt does not block desired search crawlers."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "OAI-SearchBot is allowed if ChatGPT search visibility is desired."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Claude-SearchBot is allowed if Claude search visibility is desired."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "PerplexityBot is allowed if Perplexity visibility is desired."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Googlebot is allowed for Google Search and Google AI features."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Training bots are allowed or blocked deliberately."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "User-triggered fetcher policy is documented."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "WAF rules do not accidentally block desired bots."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Bot verification is implemented where possible."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "18.2 Rendering and extraction"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Main content appears in initial HTML."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Internal links appear in crawlable HTML."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Canonical tags appear in raw source."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Robots meta tags are not injected late by JavaScript."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Structured data appears in source or reliably rendered HTML."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Important facts are not only in images."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Product details are not only in tabs or API calls."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Infinite scroll has crawlable pagination or links."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Pages work with graceful degradation."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "18.3 Canonical and duplication"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "One canonical URL per content item."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Redirects, canonicals and sitemaps agree."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Internal links point to canonical URLs."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Parameter URLs are controlled."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Facets are indexed only where useful."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Hreflang references canonical equivalents."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Noindexed pages are not listed in sitemaps."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Redirected URLs are not listed in sitemaps."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "18.4 Structure and content"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Each priority page starts with a clear summary."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Sections have descriptive headings."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Important sections answer one clear question."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Dates, prices, locations and identifiers are visible."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Comparisons include clear criteria."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Claims are supported by evidence."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Thin prompt-targeted pages are avoided."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Content is unique and useful."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "18.5 Structured data"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "JSON-LD matches visible content."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Organization schema is consistent."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Product schema includes price, currency and availability where relevant."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Article schema includes author and dates where relevant."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Breadcrumb schema matches visible breadcrumbs."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "FAQ schema is used only for visible FAQs."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "sameAs links point to trusted profiles."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "dateModified is accurate."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "18.6 Freshness"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "XML sitemap lastmod is accurate."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Visible last updated dates are used where useful."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "dateModified matches real content changes."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "IndexNow is implemented for Bing where appropriate."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Product feeds are current."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Merchant and business profiles are current."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Changelogs exist for volatile or compliance-sensitive pages."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "18.7 Measurement"
                }
              ]
            },
            {
              "type": "list",
              "ordered": false,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "AI referrers are grouped in analytics."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "ChatGPT UTM referrals are tracked where present."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Server-side bot logging is active."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Bot categories are separated."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Raw IP exposure is minimised."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Prompt-based share of voice is monitored."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Cited URLs are tracked."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Answer accuracy is reviewed regularly."
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "prioritised-implementation-plan",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "19. Prioritised implementation plan"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Phase 1: Access and extraction"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Goal: Make sure AI systems can fetch and read priority content."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Actions:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": true,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Identify priority URL groups."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Crawl the site as Googlebot and as major AI bots."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Compare source HTML versus rendered HTML."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Fix blocked, redirected, noindexed or unstable priority pages."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Make core content available in initial HTML."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Reduce unnecessary DOM and boilerplate."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Check WAF and CDN bot handling."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add or correct robots.txt policy by bot purpose."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Phase 2: Canonical and discovery foundation"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Goal: Make source selection unambiguous."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Actions:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": true,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Clean XML sitemaps."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Align internal links with canonical URLs."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Fix canonical conflicts."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Remove redirected and noindexed URLs from sitemaps."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Control parameter and faceted URLs."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add accurate lastmod."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add IndexNow where appropriate."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Reference sitemaps in robots.txt."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Phase 3: Understanding and entities"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Goal: Help machines understand what pages, products and organisations mean."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Actions:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": true,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Improve semantic HTML."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Restructure pages into clear sections."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add or refine JSON-LD."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Build consistent Organization and Product entity patterns."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add sameAs links where useful."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Align schema, visible content and external profiles."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Update author and reviewer profiles where trust matters."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Phase 4: Answer usefulness"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Goal: Make pages usable as evidence in answers."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Actions:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": true,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add direct summaries at the top of pages."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add comparison tables and decision guidance."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Clarify pricing, suitability and limitations."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add evidence for strong claims."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add dates and change notes."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Improve images, captions and alt text."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Remove thin or duplicated AI prompt pages."
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Phase 5: Measurement and governance"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Goal: Prove visibility and manage risk."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Actions:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": true,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Add AI referral channels."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Build server-side AI bot dashboards."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Verify known bots."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Track prompt-based share of voice."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Monitor cited URLs and competitor presence."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Review training bot policy quarterly."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Review WAF and rate limits."
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "text",
                          "text": "Review privacy and IP handling."
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "useful-reporting-dashboard-structure",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "20. Useful reporting dashboard structure"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "20.1 AI bot activity dashboard"
                }
              ]
            },
            {
              "type": "table",
              "rows": [
                {
                  "header": true,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Dimensions"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Metrics"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "date; bot name; bot category; verified status; requested URL; URL type; status code; response time; cache status; country or region; WAF action"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "requests; unique URLs requested; 200 responses; 3xx responses; 4xx responses; 5xx responses; average response time; cache hit ratio; top crawled directories; crawl spikes; failed verification count"
                      }
                    ]
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "20.2 AI referral dashboard"
                }
              ]
            },
            {
              "type": "table",
              "rows": [
                {
                  "header": true,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Dimensions"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Metrics"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "source; medium; landing page; device; country; content group; conversion type"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "sessions; engaged sessions; conversions; revenue; average engagement time; assisted conversions; new users; returning users"
                      }
                    ]
                  ]
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "20.3 Prompt share-of-voice dashboard"
                }
              ]
            },
            {
              "type": "table",
              "rows": [
                {
                  "header": true,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "Dimensions"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "Metrics"
                      }
                    ]
                  ]
                },
                {
                  "header": false,
                  "cells": [
                    [
                      {
                        "type": "text",
                        "text": "platform; prompt; prompt category; date; country or locale; brand mentioned; cited URL; competitor cited; sentiment; answer accuracy"
                      }
                    ],
                    [
                      {
                        "type": "text",
                        "text": "brand appearance rate; citation rate; first citation rate; competitor share; incorrect answer rate; outdated URL rate; missing price rate; missing availability rate"
                      }
                    ]
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "myths-and-corrections",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "21. Myths and corrections"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Myth: AI search needs special AI schema"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Correction: Google says there is no special schema required for AI Overviews or AI Mode. Use normal structured data that matches visible content."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Myth: llms.txt replaces sitemaps"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Correction: llms.txt is an experimental enhancement. XML sitemaps, internal links, canonical tags and accessible HTML still matter more."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Myth: Blocking Google-Extended blocks AI Overviews"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Correction: Google-Extended is for some Gemini training and grounding uses. Google Search AI features are controlled through Googlebot access and Search preview controls."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Myth: Allowing all AI bots is always good"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Correction: Training, search, user-triggered and agentic fetches are different. Decide by purpose."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Myth: robots.txt protects private content"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Correction: robots.txt is not access authorisation. Use authentication or other enforcement for private content."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Myth: JavaScript rendering is always fine because Google can render"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Correction: Some crawlers can render, some cannot, and rendering can fail. Initial HTML remains the safest place for critical content."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Myth: AI traffic is visible in GA4"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Correction: GA4 captures some AI referrals, but misses server-side crawlers, stripped referrers, AI Overview differentiation and zero-click exposure."
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "Myth: AI only uses semantic embeddings, so keywords no longer matter"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Correction: Hybrid retrieval uses both semantic and lexical signals. Exact names, dates, product IDs and terminology still matter."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "reference-implementation-snippets",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "22. Reference implementation snippets"
                }
              ]
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "22.1 HTML summary block"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "html",
              "text": "<section class=\"answer-summary\">\n  <h2>Summary</h2>\n  <p>\n    AI search optimisation improves how AI systems discover, retrieve,\n    interpret and cite a website. The most important technical foundations\n    are crawl access, server-rendered content, canonical clarity,\n    structured data, freshness and server-side measurement.\n  </p>\n</section>"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "22.2 Product facts block"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "html",
              "text": "<section id=\"product-facts\">\n  <h2>Product facts</h2>\n  <dl>\n    <dt>Product name</dt>\n    <dd>Example Analytics Pro</dd>\n\n    <dt>Price</dt>\n    <dd>£49 per month, excluding VAT</dd>\n\n    <dt>Availability</dt>\n    <dd>Available in the United Kingdom and European Union</dd>\n\n    <dt>Last updated</dt>\n    <dd><time datetime=\"2026-06-07\">7 June 2026</time></dd>\n  </dl>\n</section>"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "22.3 Server log fields"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "json",
              "text": "{\n  \"timestamp\": \"2026-06-07T09:14:00Z\",\n  \"url\": \"https://www.example.com/guides/ai-search-optimisation\",\n  \"method\": \"GET\",\n  \"status\": 200,\n  \"user_agent\": \"OAI-SearchBot/1.0\",\n  \"bot_name\": \"OAI-SearchBot\",\n  \"bot_category\": \"ai_search_retrieval\",\n  \"verified_bot\": true,\n  \"cache_status\": \"HIT\",\n  \"response_time_ms\": 84,\n  \"waf_action\": \"allow\"\n}"
            },
            {
              "type": "heading",
              "level": 3,
              "content": [
                {
                  "type": "text",
                  "text": "22.4 AI referral regex"
                }
              ]
            },
            {
              "type": "codeBlock",
              "language": "regex",
              "text": "(chatgpt\\.com|openai\\.com|perplexity\\.ai|claude\\.ai|gemini\\.google\\.com|copilot\\.microsoft\\.com|bing\\.com)"
            }
          ]
        },
        {
          "type": "section",
          "id": "final-strategic-model",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "23. Final strategic model"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "The best technical AI search optimisation strategy is:"
                }
              ]
            },
            {
              "type": "list",
              "ordered": true,
              "items": [
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Access"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Can the right systems reach the content?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Extraction"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Can they read the important content without rendering friction?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Canonicalisation"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Can they identify the correct source URL?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Chunking"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Can they split the page into useful passages?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Understanding"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Can they identify entities, facts and relationships?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Trust"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Are claims supported, current and consistent?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Freshness"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Can changes be detected quickly and honestly?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Governance"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Are training, retrieval and user-triggered access controlled separately?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Measurement"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Can you see bot activity, referrals and answer visibility?"
                        }
                      ]
                    }
                  ]
                },
                {
                  "content": [
                    {
                      "type": "paragraph",
                      "content": [
                        {
                          "type": "strong",
                          "content": [
                            {
                              "type": "text",
                              "text": "Iteration"
                            }
                          ]
                        },
                        {
                          "type": "text",
                          "text": ": Are prompt outputs, citations and errors reviewed continuously?"
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "The ultimate goal is not to trick AI systems. It is to make your site the easiest, clearest and most reliable source for the facts your audience already needs."
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "seo-scrubbox-extension",
          "label": "SEO Scrubbox extension",
          "content": [
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "Chrome extension"
                }
              ]
            },
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "Test the technical signals with SEO Scrubbox"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "strong",
                  "content": [
                    {
                      "type": "text",
                      "text": "SEO Scrubbox"
                    }
                  ]
                },
                {
                  "type": "text",
                  "text": " is a Chrome extension for technical SEO and AI crawl optimisation. It helps compare view-source and rendered signals, spot canonical drift, validate JSON-LD, audit sitemaps, hreflang, redirects, headers, CrUX metrics, and crawler access without leaving the page."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "It is built for the workflow in this guide: checking whether bots can see the right HTML, schema, crawl controls, links and response signals before you move on to logs, dashboards and longer-term monitoring."
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "link",
                  "url": "https://scrubnet.org/seo-scrubbox.html",
                  "content": [
                    {
                      "type": "text",
                      "text": "Learn about SEO Scrubbox"
                    }
                  ]
                },
                {
                  "type": "text",
                  "text": " "
                },
                {
                  "type": "link",
                  "url": "https://chromewebstore.google.com/detail/seo-scrubbox/bpmnoonigkfiidagbibkbfcoikiclmic",
                  "content": [
                    {
                      "type": "text",
                      "text": "Download Extension"
                    }
                  ]
                }
              ]
            }
          ]
        },
        {
          "type": "section",
          "id": "sources",
          "content": [
            {
              "type": "heading",
              "level": 2,
              "content": [
                {
                  "type": "text",
                  "text": "Sources"
                }
              ]
            },
            {
              "type": "paragraph",
              "content": [
                {
                  "type": "text",
                  "text": "The references below were used while preparing this guide and related Scrubnet AI visibility guidance. External sources were accessed on 7 June 2026 unless noted otherwise."
                }
              ]
            },
            {
              "type": "details",
              "summary": [
                {
                  "type": "text",
                  "text": "AI search optimisation and GEO research"
                }
              ],
              "content": [
                {
                  "type": "list",
                  "ordered": true,
                  "items": [
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/fundamentals/ai-optimization-guide",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google's guide to optimizing for generative AI features on Google"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://geoptie.com/blog/generative-engine-optimization",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Generative Engine Optimization (GEO): The Definitive Guide"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.optimizegeo.ai/blog/step-by-step-guide-to-geo-2026",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Step-by-step guide to Generative Engine Optimization in 2026"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.digitalapplied.com/blog/geo-guide-generative-engine-optimization-2026",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "GEO Guide 2026: Generative Engine Optimization explained"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.getfancy.ai/article-princeton-geo-decoded",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "The Princeton paper, decoded"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://derivatex.agency/blog/princeton-geo-paper-plain-english/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "The Princeton GEO paper in plain English"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/pdf/2311.09735",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "GEO: Generative Engine Optimization"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.reddit.com/r/aeo/comments/1shg6vc/the_complete_geo_guide_to_ranking_in_ai_answers/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "The complete GEO guide to ranking in AI answers"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://searchengineland.com/schema-markup-ai-search-no-hype-472339",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "How schema markup fits into AI search without the hype"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://cassieclarkmarketing.com/structured-data-101/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Structured Data 101: The simplest fix for AI search visibility"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.reddit.com/r/GEO__AI__SEO/comments/1rsj0ec/why_structured_data_in_ai_search_matters_more/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Why structured data in AI search matters more than ever in 2026"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "details",
              "summary": [
                {
                  "type": "text",
                  "text": "Rendering, llms.txt and machine-readable resources"
                }
              ],
              "content": [
                {
                  "type": "list",
                  "ordered": true,
                  "items": [
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://visively.com/kb/ai/ai-crawlers-javascript-rendering",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "AI crawlers and JavaScript: why LLMs cannot see your client-rendered content"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.searchviu.com/en/ai-crawlers-javascript-rendering/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "AI crawlers & JavaScript rendering"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.reddit.com/r/nextjs/comments/1sjb7a2/ai_search_tools_cant_see_your_site_if_youre/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "AI search tools cannot see your site if you are serving blank JavaScript to bots"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.averi.ai/blog/javascript-rendering-gap-ai-crawlers#:~:text=AI%20crawlers%20don't%20render,prerendering%20for%20AI%20crawlers%20specifically.",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "JavaScript rendering gap for AI crawlers"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.mintlify.com/docs/ai/llmstxt",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "llms.txt - Mintlify documentation"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://buildwithfern.com/learn/docs/ai-features/llms-txt",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "llms.txt and llms-full.txt - Fern documentation"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.twilio.com/en-us/blog/developers/docs-llms-txt-markdown-support",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Announcing Twilio Docs support for llms.txt and Markdown"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.gitbook.com/blog/what-is-llms-txt",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "What is llms.txt? Why it is important and how to create it for your docs"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://medium.com/data-science/llms-txt-explained-414d5121bcb3",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "LLMs.txt explained"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://llmstxt.org/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "llms.txt: the /llms.txt file proposal"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.answer.ai/posts/2024-09-03-llmstxt.html",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "The /llms.txt file, helping language models use your website"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://github.com/AnswerDotAI/llms-txt",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "llms.txt GitHub repository"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developer.mastercard.com/platform/documentation/agent-toolkit/working-with-llmstxt/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Working with llms.txt - Mastercard Developers"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "details",
              "summary": [
                {
                  "type": "text",
                  "text": "Crawler controls, bots and governance"
                }
              ],
              "content": [
                {
                  "type": "list",
                  "ordered": true,
                  "items": [
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://platform.openai.com/docs/bots",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "OpenAI web crawlers and user agents"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google crawlers and fetchers overview"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google common crawlers"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google special-case crawlers"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google user-triggered fetchers"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/google-extended",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google-Extended"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/robots/intro",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Introduction to robots.txt"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Robots.txt specifications"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.rfc-editor.org/rfc/rfc9309",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "RFC 9309: Robots Exclusion Protocol"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://datatracker.ietf.org/doc/html/rfc9309",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "IETF Datatracker: RFC 9309"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.bing.com/webmasters/help/webmaster-guidelines-30fba23a",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Bing Webmaster Guidelines"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Bingbot crawler documentation"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://support.apple.com/en-us/119829",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "About Applebot"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Anthropic ClaudeBot documentation"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://docs.perplexity.ai/docs/perplexitybot",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "PerplexityBot documentation"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://commoncrawl.org/ccbot",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Common Crawl CCBot"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://commoncrawl.org/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Common Crawl overview"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://support.archive-it.org/hc/en-us/articles/208001096-Archive-It-Crawling-Technology",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Internet Archive Archive-It crawling technology"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.cloudflare.com/bots/concepts/bot/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Cloudflare bot concepts"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.cloudflare.com/bots/concepts/bot/ai-crawlers/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Cloudflare AI crawler controls"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.cloudflare.com/bots/concepts/bot/verified-bots/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Cloudflare verified bots"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.cloudflare.com/bots/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Cloudflare Bot Management"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.mariehaynes.com/should-you-use-google-extended-in-robots-txt/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Should you block Google Extended in robots.txt?"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://blog.google/innovation-and-ai/products/an-update-on-web-publisher-controls/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "An update on web publisher controls"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://blog.cloudflare.com/ai-audit-enforcing-robots-txt/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Robotcop: enforcing robots.txt policies and stopping bots"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.365i.co.uk/news/2026/01/07/ai-crawler-compliance-tracking-cloudflare/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "AI crawlers violate robots.txt on 72% of UK sites, Cloudflare data shows"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://blog.cloudflare.com/ai-bots/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Easily manage AI crawlers with Cloudflare bot categories"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11962364/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Balancing security and privacy: web bot detection, privacy challenges, and regulatory compliance"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.penningtonslaw.com/insights/legal-perspectives-on-ai-data-poisoning/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Legal perspectives on AI data poisoning"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://higoodie.com/blog/llms-txt-robots-txt-ai-optimization/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "LLMs.txt and robots.txt: optimizing for AI bots and answer engines"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "details",
              "summary": [
                {
                  "type": "text",
                  "text": "Search, structured data, sitemaps and content documentation"
                }
              ],
              "content": [
                {
                  "type": "list",
                  "ordered": true,
                  "items": [
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Build and submit a sitemap"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google sitemap best practices"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.sitemaps.org/protocol.html",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Sitemaps.org protocol"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "JavaScript SEO basics"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Specify a canonical URL"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/appearance/structured-data/intro-structured-data",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Introduction to structured data"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/appearance/structured-data/product",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Product structured data"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/appearance/structured-data/organization",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Organization structured data"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/appearance/structured-data/article",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Article structured data"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/appearance/structured-data/faqpage",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "FAQ structured data"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/appearance/structured-data/breadcrumb",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Breadcrumb structured data"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.google.com/search/docs/fundamentals/creating-helpful-content",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Helpful, reliable, people-first content"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://schema.org/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Schema.org"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://schema.org/Organization",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Schema.org Organization"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://schema.org/Product",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Schema.org Product"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://schema.org/Article",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Schema.org Article"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://schema.org/FAQPage",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Schema.org FAQPage"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://schema.org/BreadcrumbList",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Schema.org BreadcrumbList"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://schema.org/ItemList",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Schema.org ItemList"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "details",
              "summary": [
                {
                  "type": "text",
                  "text": "Measurement, AI referrals and server logs"
                }
              ],
              "content": [
                {
                  "type": "list",
                  "ordered": true,
                  "items": [
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://measureu.com/ai-traffic-ga4/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Your Google Analytics is hiding your AI traffic"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://support.google.com/analytics/answer/12931766",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google Analytics traffic acquisition report"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://support.google.com/analytics/answer/9756891",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Google Analytics default channel group"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.cloudflare.com/logs/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Cloudflare Logs"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://developers.cloudflare.com/workers/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Cloudflare Workers"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.reddit.com/r/GEO_optimization/comments/1sq9867/anyone_else_noticing_that_ga4_undercounts/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "GA4 undercounting AI-referred traffic discussion"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://martech.org/how-ga4-records-traffic-from-perplexity-comet-and-chatgpt-atlas/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "How GA4 records traffic from Perplexity Comet and ChatGPT Atlas"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://discoveredlabs.com/blog/how-to-track-chatgpt-perplexity-and-ai-overviews-traffic-in-ga4-without-guessing",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "How to track ChatGPT, Perplexity, and AI Overviews traffic in GA4"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.swydo.com/blog/track-ai-traffic-in-ga4/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "The agency guide to tracking AI traffic in GA4"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.reddit.com/r/dotnet/comments/147n1tc/logging_pii_and_gdpr/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Logging PII and GDPR discussion"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.reddit.com/r/PFSENSE/comments/12i2edc/gdpr_how_to_anonymize_ips_etc/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "GDPR IP anonymisation in logs discussion"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://brokkr.net/2021/12/25/gdpr-server-logs/",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Server logs and GDPR compliance"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.konfirmity.com/blog/gdpr-logging-and-monitoring",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "GDPR logging and monitoring: practical guide"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://serverfault.com/questions/920455/how-to-create-a-gdpr-compliant-http-server-access-log-with-focus-on-remote-ip-an",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "GDPR-compliant HTTP access logs and IP anonymisation"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ]
            },
            {
              "type": "details",
              "summary": [
                {
                  "type": "text",
                  "text": "Research papers, reporting and internal source material"
                }
              ],
              "content": [
                {
                  "type": "list",
                  "ordered": true,
                  "items": [
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "text",
                              "text": "Scrubberduck / Scrubnet - AI Ingestion & Visibility Technical Guidelines (internal uploaded source)"
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/html/2508.03130v1",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Protecting small organizations from AI bots with Logrip: hierarchical IP hashing"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/abs/2505.21733",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Scrapers selectively respect robots.txt directives: evidence from a large-scale empirical study"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/abs/2604.27790",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "How Generative AI disrupts search: an empirical study of Google Search, Gemini, and AI Overviews"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/abs/2605.14021",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Measuring Google AI Overviews: activation, source quality, claim fidelity, and publisher impact"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/abs/2510.10315",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Is misinformation more open? A study of robots.txt gatekeeping on the web"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/abs/2503.06035",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "The liabilities of robots.txt"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/abs/2411.15091",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Somesite I used to crawl: awareness, agency and efficacy in protecting content creators from AI crawlers"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://arxiv.org/abs/2510.09031",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Web crawler restrictions, AI training datasets and political biases"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.businessinsider.com/several-top-news-sites-shun-openai-searchgpt-search-engine-2024-8",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Top news sites block OpenAI's SearchGPT web crawling bot"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.axios.com/2023/08/31/major-websites-are-blocking-ai-crawlers-from-accessing-their-content",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Major websites are blocking AI crawlers from accessing their content"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.theverge.com/2024/7/25/24205943/anthropic-ai-web-crawler-claudebot-ifixit-scraping-training-data",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Anthropic's crawler is ignoring websites' anti-AI scraping policies"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.theverge.com/news/718319/perplexity-stealth-crawling-cloudflare-ai-bots-report",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Cloudflare says Perplexity's AI bots are stealth crawling blocked sites"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    },
                    {
                      "content": [
                        {
                          "type": "paragraph",
                          "content": [
                            {
                              "type": "link",
                              "url": "https://www.wired.com/story/aws-perplexity-bot-scraping-investigation",
                              "content": [
                                {
                                  "type": "text",
                                  "text": "Amazon is investigating Perplexity over scraping abuse claims"
                                }
                              ]
                            }
                          ]
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ]
        }
      ]
    }
  ],
  "structuredData": [
    {
      "@context": "https://schema.org",
      "@graph": [
        {
          "@type": "BlogPosting",
          "@id": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#article",
          "mainEntityOfPage": {
            "@type": "WebPage",
            "@id": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html"
          },
          "headline": "The Ultimate Technical AI Search Optimisation Guide",
          "alternativeHeadline": "Technical SEO foundations for AI crawlers, retrieval systems, agents, and answer engines",
          "description": "Technical AI search guidance covering crawler access, rendering, structured data, freshness, measurement, governance and implementation.",
          "image": "https://scrubnet.org/discoveries/ultimate-tech-ai-search-optimisation-guide.webp",
          "datePublished": "2026-06-07",
          "dateModified": "2026-06-08",
          "author": {
            "@id": "https://scrubnet.org/#founder"
          },
          "publisher": {
            "@id": "https://scrubnet.org/#org"
          },
          "inLanguage": "en-GB"
        },
        {
          "@type": "BreadcrumbList",
          "@id": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html#breadcrumbs",
          "itemListElement": [
            {
              "@type": "ListItem",
              "position": 1,
              "name": "Home",
              "item": "https://scrubnet.org/"
            },
            {
              "@type": "ListItem",
              "position": 2,
              "name": "Articles",
              "item": "https://scrubnet.org/discoveries.html"
            },
            {
              "@type": "ListItem",
              "position": 3,
              "name": "The Ultimate Technical AI Search Optimisation Guide",
              "item": "https://scrubnet.org/discoveries/ultimate-technical-ai-search-optimisation-guide.html"
            }
          ]
        },
        {
          "@type": "Organization",
          "@id": "https://scrubnet.org/#org",
          "name": "Scrubnet Ltd",
          "url": "https://scrubnet.org/",
          "logo": {
            "@type": "ImageObject",
            "url": "https://scrubnet.org/scrubberduck-72.webp"
          }
        },
        {
          "@type": "Person",
          "@id": "https://scrubnet.org/#founder",
          "name": "Charles Vayssié",
          "jobTitle": "Founder, Technical SEO Specialist and Web Developer",
          "image": "https://scrubnet.org/charles-vayssie.webp",
          "url": "https://scrubnet.org/about-us.html",
          "worksFor": {
            "@id": "https://scrubnet.org/#org"
          },
          "sameAs": [
            "https://www.linkedin.com/in/charles-vayssie/",
            "https://x.com/Scrubnet_"
          ]
        },
        {
          "@type": "WebSite",
          "@id": "https://scrubnet.org/#website",
          "url": "https://scrubnet.org/",
          "name": "Scrubnet",
          "publisher": {
            "@id": "https://scrubnet.org/#org"
          },
          "inLanguage": "en-GB"
        }
      ]
    }
  ]
}