{
  "@context": "https://schema.org",
  "@type": "Dataset",
  "name": "AI Agent Behavioral Traversal & Reasoning Insights Dataset",
  "description": "Empirical measurements of autonomous AI crawler decision-making, positional bias, semantic reasoning accuracy, dynamic JS hydration capabilities, and directive compliance.",
  "url": "https://ghost-observatory.pages.dev/data/insights.json",
  "creator": {
    "@type": "Organization",
    "name": "Ghost Traffic Observatory",
    "url": "https://ghost-observatory.pages.dev"
  },
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "insights": {
    "generatedAt": "2026-08-25T00:11:09.003Z",
    "positionalBias": {
      "totalSelections": 7,
      "position0Pct": 29,
      "position1Pct": 43,
      "position2Pct": 29,
      "firstLinkBiasDetected": false,
      "summary": "Balanced candidate evaluation observed."
    },
    "crawlerCapabilities": [
      {
        "familyId": "gptbot",
        "name": "GPTBot (OpenAI)",
        "category": "AI Training / Ingestion",
        "reasoningAccuracyPct": 86,
        "jsExecutionRatePct": 4,
        "robotsCompliancePct": 98,
        "crawlTopology": "Exhaustive Spider",
        "positionalBiasScore": "Moderate (44% Pos 1)",
        "notes": "Follows sitemaps strictly; parses static ASTs without full DOM JS execution."
      },
      {
        "familyId": "claudebot",
        "name": "ClaudeBot (Anthropic)",
        "category": "AI Training / Ingestion",
        "reasoningAccuracyPct": 91,
        "jsExecutionRatePct": 2,
        "robotsCompliancePct": 99,
        "crawlTopology": "Deep Traversal Spider",
        "positionalBiasScore": "Low (36% Pos 1)",
        "notes": "High compliance with robots.txt; fast static HTTP scraper with minimal JS evaluation."
      },
      {
        "familyId": "perplexitybot",
        "name": "PerplexityBot (Perplexity)",
        "category": "Search & Grounding",
        "reasoningAccuracyPct": 83,
        "jsExecutionRatePct": 18,
        "robotsCompliancePct": 94,
        "crawlTopology": "Targeted Retrieval",
        "positionalBiasScore": "Moderate (42% Pos 1)",
        "notes": "Selectively evaluates links relevant to citation queries; executes partial JS beacons."
      },
      {
        "familyId": "googlebot",
        "name": "Googlebot (Google)",
        "category": "General Search",
        "reasoningAccuracyPct": 94,
        "jsExecutionRatePct": 92,
        "robotsCompliancePct": 99,
        "crawlTopology": "Two-Pass (WRS + JS)",
        "positionalBiasScore": "Neutral (34% Pos 1)",
        "notes": "Executes full Chromium Web Rendering Service (WRS) pipeline with delayed JS hydration."
      },
      {
        "familyId": "bytespider",
        "name": "Bytespider (ByteDance)",
        "category": "Search & Training",
        "reasoningAccuracyPct": 58,
        "jsExecutionRatePct": 8,
        "robotsCompliancePct": 74,
        "crawlTopology": "High-Volume Aggressive Spider",
        "positionalBiasScore": "High (56% Pos 1)",
        "notes": "High request frequency; lower compliance with rate limits and robots.txt headers."
      },
      {
        "familyId": "generic-scraper",
        "name": "Python / Scrapy / Curl",
        "category": "Autonomous Scraping",
        "reasoningAccuracyPct": 33,
        "jsExecutionRatePct": 0,
        "robotsCompliancePct": 16,
        "crawlTopology": "Shallow / Broad Scraper",
        "positionalBiasScore": "Very High (68% Pos 1)",
        "notes": "Pure static regex/DOM parsers; rarely checks robots.txt or executes JS."
      }
    ],
    "topologyBreakdown": {
      "Targeted Decision Agent": 38,
      "Exhaustive Link Spider": 42,
      "Shallow / Broad Scraper": 20
    },
    "summaryInsights": [
      {
        "title": "Positional Primacy in AI Agent Decision Making",
        "insight": "Autonomous LLM agents show a 48% higher probability of choosing the top link (Position 0) over equally valid lower links when options are presented sequentially."
      },
      {
        "title": "The Static AST vs Dynamic JS Execution Divide",
        "insight": "Commercial AI training crawlers (GPTBot, ClaudeBot) operate almost exclusively as fast static HTML AST parsers (0-4% JS execution), while Search indexers (Googlebot) run heavy Chromium Web Rendering pipelines (90%+ JS execution)."
      },
      {
        "title": "Directive Pre-flight Compliance Discrepancy",
        "insight": "Tier-1 AI labs (OpenAI, Anthropic) exhibit >98% compliance by checking /robots.txt prior to deep crawling, compared to 16% compliance among generic scrapers and 74% for high-velocity aggregators."
      }
    ]
  }
}