[{"data":1,"prerenderedAt":3112},["ShallowReactive",2],{"navigation_docs":3,"-data-ops-llm-ops-benchmarking-espi":449,"-data-ops-llm-ops-benchmarking-espi-surround":3108},[4,8,72,102,248,294,331,352,445],{"title":5,"path":6,"stem":7},"Introduction","\u002Fintroduction","0.introduction",{"title":9,"icon":10,"path":11,"stem":12,"children":13,"page":67},"Company","i-lucide-building-2","\u002Fcompany","1.company",[14,18,22,26,30,34,38,42,46,50,54,68],{"title":15,"path":16,"stem":17},"About","\u002Fcompany\u002Fabout","1.company\u002F0.about",{"title":19,"path":20,"stem":21},"Values","\u002Fcompany\u002Fvalues","1.company\u002F1.values",{"title":23,"path":24,"stem":25},"Communication","\u002Fcompany\u002Fcommunication","1.company\u002Fcommunication",{"title":27,"path":28,"stem":29},"Competition","\u002Fcompany\u002Fcompetition","1.company\u002Fcompetition",{"title":31,"path":32,"stem":33},"Hybrid Working","\u002Fcompany\u002Fhybrid-working","1.company\u002Fhybrid-working",{"title":35,"path":36,"stem":37},"Manchester Office","\u002Fcompany\u002Foffice","1.company\u002Foffice",{"title":39,"path":40,"stem":41},"Operations","\u002Fcompany\u002Foperations","1.company\u002Foperations",{"title":43,"path":44,"stem":45},"Policies","\u002Fcompany\u002Fpolicies","1.company\u002Fpolicies",{"title":47,"path":48,"stem":49},"Problem Statement","\u002Fcompany\u002Fproblem-statement","1.company\u002Fproblem-statement",{"title":51,"path":52,"stem":53},"Product Strategy","\u002Fcompany\u002Fproduct-strategy","1.company\u002Fproduct-strategy",{"title":55,"path":56,"stem":57,"children":58,"page":67},"Products","\u002Fcompany\u002Fproducts","1.company\u002Fproducts",[59,63],{"title":60,"path":61,"stem":62},"Capability Exchange","\u002Fcompany\u002Fproducts\u002Fcapability-exchange","1.company\u002Fproducts\u002Fcapability-exchange",{"title":64,"path":65,"stem":66},"ESProfiler Platform","\u002Fcompany\u002Fproducts\u002Fesprofiler","1.company\u002Fproducts\u002Fesprofiler",false,{"title":69,"path":70,"stem":71},"Security","\u002Fcompany\u002Fsecurity","1.company\u002Fsecurity",{"title":73,"icon":74,"path":75,"stem":76,"children":77,"page":67},"People Ops","i-lucide-users","\u002Fpeople-ops","2.people-ops",[78,82,86,90,94,98],{"title":79,"path":80,"stem":81},"Compensation","\u002Fpeople-ops\u002Fcompensation","2.people-ops\u002Fcompensation",{"title":83,"path":84,"stem":85},"Education","\u002Fpeople-ops\u002Feducation","2.people-ops\u002Feducation",{"title":87,"path":88,"stem":89},"Expenses","\u002Fpeople-ops\u002Fexpenses","2.people-ops\u002Fexpenses",{"title":91,"path":92,"stem":93},"Holiday & Leave","\u002Fpeople-ops\u002Fleave","2.people-ops\u002Fleave",{"title":95,"path":96,"stem":97},"Onboarding","\u002Fpeople-ops\u002Fonboarding","2.people-ops\u002Fonboarding",{"title":99,"path":100,"stem":101},"Recruitment","\u002Fpeople-ops\u002Frecruitment","2.people-ops\u002Frecruitment",{"title":103,"icon":104,"path":105,"stem":106,"children":107,"page":67},"Engineering","i-lucide-rocket","\u002Fengineering","3.engineering",[108,168,189,210,232,240,244],{"title":109,"path":110,"stem":111,"children":112,"page":67},"Contributing","\u002Fengineering\u002Fcontributing","3.engineering\u002Fcontributing",[113,117,121,125,129,133,146,155],{"title":114,"path":115,"stem":116},"Development Setup","\u002Fengineering\u002Fcontributing\u002Fdevelopment-setup","3.engineering\u002Fcontributing\u002F1.development-setup",{"title":118,"path":119,"stem":120},"Engineering Operations","\u002Fengineering\u002Fcontributing\u002Fengineering-operations","3.engineering\u002Fcontributing\u002F2.engineering-operations",{"title":122,"path":123,"stem":124},"Documentation","\u002Fengineering\u002Fcontributing\u002Fdocumentation","3.engineering\u002Fcontributing\u002F3.documentation",{"title":126,"path":127,"stem":128},"Agentic Coding","\u002Fengineering\u002Fcontributing\u002Fagentic-coding","3.engineering\u002Fcontributing\u002F4.agentic-coding",{"title":130,"path":131,"stem":132},"Coding Standards","\u002Fengineering\u002Fcontributing\u002Fcoding-standards","3.engineering\u002Fcontributing\u002F5.coding-standards",{"title":134,"path":135,"stem":136,"children":137,"page":67},"Back End","\u002Fengineering\u002Fcontributing\u002Fback-end","3.engineering\u002Fcontributing\u002Fback-end",[138,142],{"title":139,"path":140,"stem":141},"API Guidelines","\u002Fengineering\u002Fcontributing\u002Fback-end\u002Fapi-guidelines","3.engineering\u002Fcontributing\u002Fback-end\u002Fapi-guidelines",{"title":143,"path":144,"stem":145},"LLM Prompts & Langfuse Integration","\u002Fengineering\u002Fcontributing\u002Fback-end\u002Fllm-prompts","3.engineering\u002Fcontributing\u002Fback-end\u002Fllm-prompts",{"title":147,"path":148,"stem":149,"children":150,"page":67},"Database","\u002Fengineering\u002Fcontributing\u002Fdatabase","3.engineering\u002Fcontributing\u002Fdatabase",[151],{"title":152,"path":153,"stem":154},"Database Volumes","\u002Fengineering\u002Fcontributing\u002Fdatabase\u002Fdatabase-refresh","3.engineering\u002Fcontributing\u002Fdatabase\u002Fdatabase-refresh",{"title":156,"path":157,"stem":158,"children":159,"page":67},"Front End","\u002Fengineering\u002Fcontributing\u002Ffront-end","3.engineering\u002Fcontributing\u002Ffront-end",[160,164],{"title":161,"path":162,"stem":163},"Data Attributes","\u002Fengineering\u002Fcontributing\u002Ffront-end\u002Fdata-attributes","3.engineering\u002Fcontributing\u002Ffront-end\u002Fdata-attributes",{"title":165,"path":166,"stem":167},"Testing","\u002Fengineering\u002Fcontributing\u002Ffront-end\u002Ftesting","3.engineering\u002Fcontributing\u002Ffront-end\u002Ftesting",{"title":169,"path":170,"stem":171,"children":172},"Deployment","\u002Fengineering\u002Fdeployment","3.engineering\u002Fdeployment\u002F0.index",[173,174,177,181,185],{"title":169,"path":170,"stem":171},{"title":60,"path":175,"stem":176},"\u002Fengineering\u002Fdeployment\u002Fcapability-exchange","3.engineering\u002Fdeployment\u002Fcapability-exchange",{"title":178,"path":179,"stem":180},"Langfuse Deployment","\u002Fengineering\u002Fdeployment\u002Fecs-langfuse-deployment","3.engineering\u002Fdeployment\u002Fecs-langfuse-deployment",{"title":182,"path":183,"stem":184},"ESP Platform Configuration","\u002Fengineering\u002Fdeployment\u002Fesp-platform-configuration","3.engineering\u002Fdeployment\u002Fesp-platform-configuration",{"title":186,"path":187,"stem":188},"Platform","\u002Fengineering\u002Fdeployment\u002Fplatform","3.engineering\u002Fdeployment\u002Fplatform",{"title":190,"path":191,"stem":192,"children":193,"page":67},"Github","\u002Fengineering\u002Fgithub","3.engineering\u002Fgithub",[194,198,202,206],{"title":195,"path":196,"stem":197},"Packages","\u002Fengineering\u002Fgithub\u002Fpackages","3.engineering\u002Fgithub\u002Fpackages",{"title":199,"path":200,"stem":201},"Personal Access Token","\u002Fengineering\u002Fgithub\u002Fpersonal-access-token","3.engineering\u002Fgithub\u002Fpersonal-access-token",{"title":203,"path":204,"stem":205},"Troubleshooting","\u002Fengineering\u002Fgithub\u002Ftroubleshooting","3.engineering\u002Fgithub\u002Ftroubleshooting",{"title":207,"path":208,"stem":209},"Workflows","\u002Fengineering\u002Fgithub\u002Fworkflows","3.engineering\u002Fgithub\u002Fworkflows",{"title":211,"path":212,"stem":213,"children":214},"Platform Ops","\u002Fengineering\u002Fplatform-ops","3.engineering\u002Fplatform-ops\u002F0.index",[215,216,220,224,228],{"title":211,"path":212,"stem":213},{"title":217,"path":218,"stem":219},"Production Database","\u002Fengineering\u002Fplatform-ops\u002Fdatabase-connection","3.engineering\u002Fplatform-ops\u002Fdatabase-connection",{"title":221,"path":222,"stem":223},"New Tenant Creation","\u002Fengineering\u002Fplatform-ops\u002Fnew-tenant-creation","3.engineering\u002Fplatform-ops\u002Fnew-tenant-creation",{"title":225,"path":226,"stem":227},"Releases","\u002Fengineering\u002Fplatform-ops\u002Frelease","3.engineering\u002Fplatform-ops\u002Frelease",{"title":229,"path":230,"stem":231},"Versioning","\u002Fengineering\u002Fplatform-ops\u002Fversioning","3.engineering\u002Fplatform-ops\u002Fversioning",{"title":186,"path":233,"stem":234,"children":235,"page":67},"\u002Fengineering\u002Fplatform","3.engineering\u002Fplatform",[236],{"title":237,"path":238,"stem":239},"useAPI","\u002Fengineering\u002Fplatform\u002Fuse-api","3.engineering\u002Fplatform\u002Fuse-api",{"title":241,"path":242,"stem":243},"Project Management","\u002Fengineering\u002Fproject-management","3.engineering\u002Fproject-management",{"title":245,"path":246,"stem":247},"Tools","\u002Fengineering\u002Ftools","3.engineering\u002Ftools",{"title":249,"icon":250,"path":251,"stem":252,"children":253,"page":67},"Design","i-lucide-palette","\u002Fdesign","4.design",[254,275,279,283,287,290],{"title":255,"path":256,"stem":257,"children":258,"page":67},"Personas","\u002Fdesign\u002Fpersonas","4.design\u002F0.personas",[259,263,267,271],{"title":260,"path":261,"stem":262},"CISO - Simon","\u002Fdesign\u002Fpersonas\u002F01-ciso-simon","4.design\u002F0.personas\u002F01-ciso-simon",{"title":264,"path":265,"stem":266},"Head of Cyber - Harry","\u002Fdesign\u002Fpersonas\u002F02-head-of-cyber-harry","4.design\u002F0.personas\u002F02-head-of-cyber-harry",{"title":268,"path":269,"stem":270},"Security Architect - Sasha","\u002Fdesign\u002Fpersonas\u002F03-security-architect-sasha","4.design\u002F0.personas\u002F03-security-architect-sasha",{"title":272,"path":273,"stem":274},"Procurement Officer - Paige","\u002Fdesign\u002Fpersonas\u002F04-procurement-officer-paige","4.design\u002F0.personas\u002F04-procurement-officer-paige",{"title":276,"path":277,"stem":278},"Design Thinking","\u002Fdesign\u002Fdesign-thinking","4.design\u002F1.design-thinking",{"title":280,"path":281,"stem":282},"Design & Development","\u002Fdesign\u002Fdesign-and-development","4.design\u002F3.design-and-development",{"title":284,"path":285,"stem":286},"Branding","\u002Fdesign\u002Fbranding","4.design\u002F4.branding",{"title":245,"path":288,"stem":289},"\u002Fdesign\u002Ftools","4.design\u002F5.tools",{"title":291,"path":292,"stem":293},"Customer Success","\u002Fdesign\u002Fworking-with-customers","4.design\u002F6.working-with-customers",{"title":295,"icon":296,"path":297,"stem":298,"children":299,"page":67},"Sales","i-lucide-dollar-sign","\u002Fsales","4.sales",[300,320,324,328],{"title":301,"path":302,"stem":303,"children":304,"page":67},"Process","\u002Fsales\u002Fprocess","4.sales\u002F0.process",[305,309,313,317],{"title":306,"path":307,"stem":308},"Lead","\u002Fsales\u002Fprocess\u002Flead","4.sales\u002F0.process\u002F0.lead",{"title":310,"path":311,"stem":312},"Deal Qualification","\u002Fsales\u002Fprocess\u002Fqualification","4.sales\u002F0.process\u002F1.qualification",{"title":314,"path":315,"stem":316},"Close","\u002Fsales\u002Fprocess\u002Fclose","4.sales\u002F0.process\u002F2.close",{"title":95,"path":318,"stem":319},"\u002Fsales\u002Fprocess\u002Fonboarding","4.sales\u002F0.process\u002Fonboarding",{"title":321,"path":322,"stem":323},"Meetings","\u002Fsales\u002Fmeetings","4.sales\u002Fmeetings",{"title":325,"path":326,"stem":327},"Metrics","\u002Fsales\u002Fmetrics","4.sales\u002Fmetrics",{"title":245,"path":329,"stem":330},"\u002Fsales\u002Ftools","4.sales\u002Ftools",{"title":332,"icon":333,"path":334,"stem":335,"children":336,"page":67},"Marketing","i-lucide-book-image","\u002Fmarketing","5.marketing",[337,341,345,348],{"title":338,"path":339,"stem":340},"Content","\u002Fmarketing\u002Fcontent","5.marketing\u002Fcontent",{"title":342,"path":343,"stem":344},"Messaging","\u002Fmarketing\u002Fmessaging","5.marketing\u002Fmessaging",{"title":245,"path":346,"stem":347},"\u002Fmarketing\u002Ftools","5.marketing\u002Ftools",{"title":349,"path":350,"stem":351},"Website","\u002Fmarketing\u002Fwebsite","5.marketing\u002Fwebsite",{"title":353,"icon":354,"path":355,"stem":356,"children":357,"page":67},"AI & Data Ops","i-lucide-database","\u002Fdata-ops","6.data-ops",[358,362,370,395,416,420],{"title":359,"path":360,"stem":361},"Account Portal (CAS)","\u002Fdata-ops\u002Faccount-portal","6.data-ops\u002Faccount-portal",{"title":60,"path":363,"stem":364,"children":365,"page":67},"\u002Fdata-ops\u002Fcapability-exchange","6.data-ops\u002Fcapability-exchange",[366],{"title":367,"path":368,"stem":369},"Leaderboard Calculation","\u002Fdata-ops\u002Fcapability-exchange\u002Fleaderboard-calculation","6.data-ops\u002Fcapability-exchange\u002Fleaderboard-calculation",{"title":371,"path":372,"stem":373,"children":374,"page":67},"Data Management","\u002Fdata-ops\u002Fdata-management","6.data-ops\u002Fdata-management",[375,379,383,387,391],{"title":376,"path":377,"stem":378},"Adding Products","\u002Fdata-ops\u002Fdata-management\u002Fadding-products","6.data-ops\u002Fdata-management\u002Fadding-products",{"title":380,"path":381,"stem":382},"Adding Vendors","\u002Fdata-ops\u002Fdata-management\u002Fadding-vendors","6.data-ops\u002Fdata-management\u002Fadding-vendors",{"title":384,"path":385,"stem":386},"Framework Mapping","\u002Fdata-ops\u002Fdata-management\u002Fframework-mapping","6.data-ops\u002Fdata-management\u002Fframework-mapping",{"title":388,"path":389,"stem":390},"Refreshing Vendors","\u002Fdata-ops\u002Fdata-management\u002Frefreshing-vendors","6.data-ops\u002Fdata-management\u002Frefreshing-vendors",{"title":392,"path":393,"stem":394},"Reviewing Draft Vendors","\u002Fdata-ops\u002Fdata-management\u002Freviewing-draft-vendors","6.data-ops\u002Fdata-management\u002Freviewing-draft-vendors",{"title":396,"path":397,"stem":398,"children":399,"page":67},"LLM Ops","\u002Fdata-ops\u002Fllm-ops","6.data-ops\u002Fllm-ops",[400,404,408,412],{"title":401,"path":402,"stem":403},"Agents","\u002Fdata-ops\u002Fllm-ops\u002Fagents","6.data-ops\u002Fllm-ops\u002F1.agents",{"title":405,"path":406,"stem":407},"ESPi Architecture & Query Flow","\u002Fdata-ops\u002Fllm-ops\u002Fespi-architecture","6.data-ops\u002Fllm-ops\u002F2.espi-architecture",{"title":409,"path":410,"stem":411},"Evaluating Agents","\u002Fdata-ops\u002Fllm-ops\u002Fevaluations","6.data-ops\u002Fllm-ops\u002F3.evaluations",{"title":413,"path":414,"stem":415},"Benchmarking ESPi","\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi","6.data-ops\u002Fllm-ops\u002F4.benchmarking-espi",{"title":417,"path":418,"stem":419},"Message Queues","\u002Fdata-ops\u002Fmessage-queues","6.data-ops\u002Fmessage-queues",{"title":421,"path":422,"stem":423,"children":424,"page":67},"Models","\u002Fdata-ops\u002Fmodels","6.data-ops\u002Fmodels",[425,429,433,437,441],{"title":426,"path":427,"stem":428},"Universal Framework Mapper Vision","\u002Fdata-ops\u002Fmodels\u002Funiversal-framework-mapper-vision","6.data-ops\u002Fmodels\u002F1.universal-framework-mapper-vision",{"title":430,"path":431,"stem":432},"Framework Description Model (FDM)","\u002Fdata-ops\u002Fmodels\u002Fframework-description-model","6.data-ops\u002Fmodels\u002F2.framework-description-model",{"title":434,"path":435,"stem":436},"Relationship Definition Model (RDM)","\u002Fdata-ops\u002Fmodels\u002Frelationship-definition-model","6.data-ops\u002Fmodels\u002F3.relationship-definition-model",{"title":438,"path":439,"stem":440},"Relationship Mapping Model (RMM)","\u002Fdata-ops\u002Fmodels\u002Frelationship-mapping-model","6.data-ops\u002Fmodels\u002F4.relationship-mapping-model",{"title":442,"path":443,"stem":444},"Platform Framework Matrix V2","\u002Fdata-ops\u002Fmodels\u002Fplatform-framework-matrix-v2","6.data-ops\u002Fmodels\u002F5.platform-framework-matrix-v2",{"title":446,"path":447,"stem":448},"Glossary","\u002Fglossary","glossary",{"id":450,"title":413,"body":451,"description":3101,"extension":3102,"links":3103,"meta":3104,"navigation":3105,"path":414,"seo":3106,"stem":415,"__hash__":3107},"docs\u002F6.data-ops\u002Fllm-ops\u002F4.benchmarking-espi.md",{"type":452,"value":453,"toc":3048},"minimark",[454,471,478,483,486,491,555,558,568,579,602,620,624,631,670,673,679,682,686,693,722,748,754,760,766,769,773,788,794,823,830,850,863,866,914,917,946,950,957,1009,1020,1032,1036,1043,1052,1062,1066,1083,1087,1133,1137,1207,1222,1225,1235,1239,1303,1306,1309,1312,1315,1321,1325,1336,1434,1437,1445,1449,1456,1481,1484,1509,1513,1521,1555,1564,1570,1594,1598,1617,1620,1635,1637,1640,1645,1651,1655,1701,1716,1720,1726,1747,1751,1836,1840,1857,1925,1929,1932,1941,1944,1948,1957,1968,1977,1991,1993,1996,2003,2009,2013,2016,2078,2082,2178,2182,2214,2219,2223,2229,2239,2261,2265,2271,2274,2285,2288,2303,2372,2383,2394,2422,2425,2443,2491,2515,2518,2542,2561,2570,2645,2659,2663,2666,2670,2679,2681,2684,2695,2704,2710,2714,2728,2731,2735,2738,2793,2799,2818,2831,2835,2870,2873,2879,2882,2885,2890,2903,2905,2908,2924,2930,2934,2998,3002,3009,3013,3018,3028,3032,3038,3044],[455,456,457,458,462,463,470],"p",{},"benchmark-espi runs the ESPi ground-truth questions against the local ESP stack, records every answer with its\nlatency, cost, tokens and tool calls, and lets you compare runs side by side. This guide covers the initial setup, then walks\nthrough the UI page by page. For the full CLI and the architecture, see ",[459,460,461],"code",{},"benchmark-espi\u002FREADME.md"," in the\n",[464,465,469],"a",{"href":466,"rel":467},"https:\u002F\u002Fgithub.com\u002FES-Profiler\u002Fdata-science",[468],"nofollow","data-science repo",".",[455,472,473],{},[474,475],"img",{"alt":476,"src":477},"Home page","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fhome.png",[479,480,482],"h2",{"id":481},"initial-setup","Initial setup",[455,484,485],{},"These steps start the ESP stack and set benchmark-espi up from scratch, with no database or earlier runs. Allow\nabout 30 minutes, plus the time the stack takes to start.",[487,488,490],"h3",{"id":489},"_0-prerequisites","0. Prerequisites",[492,493,494,503,534,541,548],"ul",{},[495,496,497,502],"li",{},[464,498,501],{"href":499,"rel":500},"https:\u002F\u002Fdocs.astral.sh\u002Fuv\u002F",[468],"uv",", Node + yarn, and a JDK for platform-api.",[495,504,505,506,509,510,514,515,518,519,522,523,526,527,529,530,533],{},"The local ESP dev stack (infra-devenv-pro) running, with CAS on ",[459,507,508],{},"127.0.0.1:8085",". For the page-context\nquestions and the data rubrics, the loaded data must be the ",[511,512,513],"strong",{},"heavy"," volume snapshot (",[459,516,517],{},"esp-database-8",",\n",[459,520,521],{},"esp-postgres"," and ",[459,524,525],{},"esp-osnode-1"," all ",[459,528,513],{},"), because the scenario file's entity ids and the data rubrics'\nfacts (",[459,531,532],{},"rubrics\u002Ffacts\u002Fheavy\u002F",") come from it.",[495,535,536,537,540],{},"A benchmark user's login for the local stack (default ",[459,538,539],{},"user@esprofiler.com","), without MFA.",[495,542,543,544,547],{},"The TypeSafe key that platform-api uses as ",[459,545,546],{},"ESP_JEV_API_KEY",". The app uses the same key to grade answers.",[495,549,550,551,554],{},"Langfuse keys for the ",[459,552,553],{},"esp-development"," project, for the cost, token and tool-call metrics. They're optional, but\nwithout them nothing is enriched.",[455,556,557],{},"The expected layout, which the app uses to detect the builds:",[559,560,565],"pre",{"className":561,"code":563,"language":564},[562],"language-text","~\u002Fdevelopment\u002Fbackend\u002Fplatform-api\n~\u002Fdevelopment\u002Ffrontend\u002Fplatform-docs\n~\u002Fdevelopment\u002Finfra-devenv-pro\n~\u002Fdevelopment\u002Fdata-science\n","text",[459,566,563],{"__ignoreMap":567},"",[455,569,570,571,574,575,578],{},"If your checkouts are elsewhere, change the ",[459,572,573],{},"cd"," paths below and set these in ",[459,576,577],{},".env"," (step 3), otherwise the build\npre-fill, the scenario checks and the dataset detection won't find them:",[492,580,581,587,593],{},[495,582,583,586],{},[459,584,585],{},"PLATFORM_API_REPO",": your platform-api checkout",[495,588,589,592],{},[459,590,591],{},"PLATFORM_DOCS_REPO",": your platform-docs checkout",[495,594,595,598,599],{},[459,596,597],{},"INFRA_DEVENV_CONFIG",": infra-devenv-pro's ",[459,600,601],{},"config.json",[455,603,604,605,608,609,522,612,615,616,619],{},"For Docker, the equivalents are ",[459,606,607],{},"PLATFORM_API_HOST_REPO",", ",[459,610,611],{},"PLATFORM_DOCS_HOST_REPO",[459,613,614],{},"INFRA_DEVENV_HOST_DIR",".\nThey default to paths relative to ",[459,617,618],{},"compose.yaml"," that assume the same layout.",[487,621,623],{"id":622},"_1-start-platform-docs-the-docs-mcp-server","1. Start platform-docs (the docs MCP server)",[455,625,626,627,630],{},"Start it ",[511,628,629],{},"before"," platform-api, because platform-api connects to it when it starts.",[559,632,636],{"className":633,"code":634,"language":635,"meta":567,"style":567},"language-powershell shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","cd ~\u002Fdevelopment\u002Ffrontend\u002Fplatform-docs\ngit checkout main\ngit pull\nyarn install\nyarn mcp:start\n","powershell",[459,637,638,646,652,658,664],{"__ignoreMap":567},[639,640,643],"span",{"class":641,"line":642},"line",1,[639,644,645],{},"cd ~\u002Fdevelopment\u002Ffrontend\u002Fplatform-docs\n",[639,647,649],{"class":641,"line":648},2,[639,650,651],{},"git checkout main\n",[639,653,655],{"class":641,"line":654},3,[639,656,657],{},"git pull\n",[639,659,661],{"class":641,"line":660},4,[639,662,663],{},"yarn install\n",[639,665,667],{"class":641,"line":666},5,[639,668,669],{},"yarn mcp:start\n",[455,671,672],{},"It should log:",[559,674,677],{"className":675,"code":676,"language":564},[562],"[MCP INFO] Platform Docs MCP Server listening on http:\u002F\u002F127.0.0.1:9998\u002Fmcp (SSE & Streamable HTTP)\n",[459,678,676],{"__ignoreMap":567},[455,680,681],{},"A \"Documentation site not found ... Run yarn docs:build\" error only means the docs website isn't served. The MCP\ntools still work, so it doesn't matter here. Leave this terminal open.",[487,683,685],{"id":684},"_2-start-platform-api","2. Start platform-api",[455,687,688,689,692],{},"For Jev routing to the docs agent, platform-api must be on a build that includes it (at the time of writing, the\n",[459,690,691],{},"715-add-jev-and-langgraph4j-to-espi"," branch):",[559,694,696],{"className":633,"code":695,"language":635,"meta":567,"style":567},"cd ~\u002Fdevelopment\u002Fbackend\u002Fplatform-api\ngit checkout 715-add-jev-and-langgraph4j-to-espi\ngit pull\n$env:ESP_JEV_API_KEY = \"\u003CTypeSafe key>\"   # if it isn't already set in your environment\n$env:SPRING_PROFILES_ACTIVE=\"dev\"; $env:COMMIT_INFO = git rev-parse HEAD; .\\mvnw.cmd spring-boot:run\n",[459,697,698,703,708,712,717],{"__ignoreMap":567},[639,699,700],{"class":641,"line":642},[639,701,702],{},"cd ~\u002Fdevelopment\u002Fbackend\u002Fplatform-api\n",[639,704,705],{"class":641,"line":648},[639,706,707],{},"git checkout 715-add-jev-and-langgraph4j-to-espi\n",[639,709,710],{"class":641,"line":654},[639,711,657],{},[639,713,714],{"class":641,"line":660},[639,715,716],{},"$env:ESP_JEV_API_KEY = \"\u003CTypeSafe key>\"   # if it isn't already set in your environment\n",[639,718,719],{"class":641,"line":666},[639,720,721],{},"$env:SPRING_PROFILES_ACTIVE=\"dev\"; $env:COMMIT_INFO = git rev-parse HEAD; .\\mvnw.cmd spring-boot:run\n",[492,723,724,734],{},[495,725,726,729,730,733],{},[459,727,728],{},"SPRING_PROFILES_ACTIVE=dev"," turns the Jev router on and points platform-api at the local docs server\n(",[459,731,732],{},"http:\u002F\u002Flocalhost:9998\u002Fmcp",").",[495,735,736,739,740,743,744,747],{},[459,737,738],{},"COMMIT_INFO"," makes ",[459,741,742],{},"\u002Fapi\u002Fv1\u002Finfo"," report the commit. The benchmark then records it on each run as the\nplatform-api build (",[459,745,746],{},"source: reported","). Without it, the build is only a guess from the checkout.",[455,749,750,751,470],{},"Wait until the platform is up on ",[459,752,753],{},"127.0.0.1:8088",[455,755,756,759],{},[511,757,758],{},"Check that platform-api has connected to platform-docs."," The platform-docs terminal should show new requests,\nsuch as:",[559,761,764],{"className":762,"code":763,"language":564},[562],"[MCP INFO] Handling POST request with StreamableHTTPServerTransport\n",[459,765,763],{"__ignoreMap":567},[455,767,768],{},"If nothing appears, platform-api hasn't reached the docs server, and docs questions will get no documentation.\nCheck that platform-docs is still running on port 9998, then restart platform-api.",[487,770,772],{"id":771},"_3-set-up-the-app","3. Set up the app",[559,774,776],{"className":633,"code":775,"language":635,"meta":567,"style":567},"cd ~\u002Fdevelopment\u002Fdata-science\u002Fbenchmark-espi\ncopy .env.example .env\n",[459,777,778,783],{"__ignoreMap":567},[639,779,780],{"class":641,"line":642},[639,781,782],{},"cd ~\u002Fdevelopment\u002Fdata-science\u002Fbenchmark-espi\n",[639,784,785],{"class":641,"line":648},[639,786,787],{},"copy .env.example .env\n",[455,789,790,791,793],{},"In ",[459,792,577],{},", fill in:",[492,795,796,809,815],{},[495,797,798,801,802,805,806,808],{},[459,799,800],{},"ESP_PASSWORD"," (and ",[459,803,804],{},"ESP_USERNAME"," if it isn't ",[459,807,539],{},")",[495,810,811,814],{},[459,812,813],{},"JEV_API_KEY",": the TypeSafe key",[495,816,817,522,820],{},[459,818,819],{},"LANGFUSE_PUBLIC_KEY",[459,821,822],{},"LANGFUSE_SECRET_KEY",[455,824,825,826,829],{},"If your checkouts aren't under ",[459,827,828],{},"~\u002Fdevelopment",", also set the repo paths listed in step 0. Leave the rest as it is.\nThen:",[559,831,833],{"className":633,"code":832,"language":635,"meta":567,"style":567},"uv sync\nuv run benchmark-espi init-db       # creates data\u002Fbenchmark.db\nuv run benchmark-espi check-auth    # logs in to CAS and prints the platform info\n",[459,834,835,840,845],{"__ignoreMap":567},[639,836,837],{"class":641,"line":642},[639,838,839],{},"uv sync\n",[639,841,842],{"class":641,"line":648},[639,843,844],{},"uv run benchmark-espi init-db       # creates data\u002Fbenchmark.db\n",[639,846,847],{"class":641,"line":654},[639,848,849],{},"uv run benchmark-espi check-auth    # logs in to CAS and prints the platform info\n",[455,851,852,855,856,859,860,470],{},[459,853,854],{},"check-auth"," should print the platform version with a ",[459,857,858],{},"commitTag"," that matches platform-api's ",[459,861,862],{},"git rev-parse HEAD",[455,864,865],{},"Load the question set, the rubrics and the scenarios:",[559,867,869],{"className":633,"code":868,"language":635,"meta":567,"style":567},"uv run benchmark-espi import-questions questions\u002Fespi-groundtruth.csv --name \"ESPi Groundtruth\"\nuv run benchmark-espi import-rubrics rubrics\u002Fespi-groundtruth-docs-help.yaml\nuv run benchmark-espi import-rubrics rubrics\u002Fespi-groundtruth-capability.yaml\nuv run benchmark-espi import-rubrics rubrics\u002Fespi-groundtruth-data.yaml\nuv run benchmark-espi import-rubrics rubrics\u002Fespi-groundtruth-market.yaml\nuv run benchmark-espi import-scenarios scenarios\u002Fespi-groundtruth.yaml\nuv run benchmark-espi check-scenarios -q \"ESPi Groundtruth\"\nuv run benchmark-espi collect-facts --check\n",[459,870,871,876,881,886,891,896,902,908],{"__ignoreMap":567},[639,872,873],{"class":641,"line":642},[639,874,875],{},"uv run benchmark-espi import-questions questions\u002Fespi-groundtruth.csv --name \"ESPi Groundtruth\"\n",[639,877,878],{"class":641,"line":648},[639,879,880],{},"uv run benchmark-espi import-rubrics rubrics\u002Fespi-groundtruth-docs-help.yaml\n",[639,882,883],{"class":641,"line":654},[639,884,885],{},"uv run benchmark-espi import-rubrics rubrics\u002Fespi-groundtruth-capability.yaml\n",[639,887,888],{"class":641,"line":660},[639,889,890],{},"uv run benchmark-espi import-rubrics rubrics\u002Fespi-groundtruth-data.yaml\n",[639,892,893],{"class":641,"line":666},[639,894,895],{},"uv run benchmark-espi import-rubrics rubrics\u002Fespi-groundtruth-market.yaml\n",[639,897,899],{"class":641,"line":898},6,[639,900,901],{},"uv run benchmark-espi import-scenarios scenarios\u002Fespi-groundtruth.yaml\n",[639,903,905],{"class":641,"line":904},7,[639,906,907],{},"uv run benchmark-espi check-scenarios -q \"ESPi Groundtruth\"\n",[639,909,911],{"class":641,"line":910},8,[639,912,913],{},"uv run benchmark-espi collect-facts --check\n",[455,915,916],{},"Expected:",[492,918,919,922,931,937],{},[495,920,921],{},"79 questions imported, and a rubric on every one of them once the four rubric files are loaded.",[495,923,924,927,928,470],{},[459,925,926],{},"import-scenarios"," accepts every context. It checks the route names against the platform-api checkout's\n",[459,929,930],{},"sitemap.json",[495,932,933,936],{},[459,934,935],{},"check-scenarios"," finds every context entity. A missing entity means a different data snapshot is loaded. Those\nquestions will fail with \"Context check failed\" rather than be asked.",[495,938,939,942,943,733],{},[459,940,941],{},"collect-facts --check"," (read-only GETs) finds the loaded data matching the data rubrics' snapshot. Drift means\nthe data rubrics' figures may not hold for this data, so those answers can be graded wrongly. Ownership and\nannualised values follow contract dates, so a little drift on the heavy snapshot is possible (see\n",[459,944,945],{},"benchmark-espi\u002Frubrics\u002FREADME.md",[487,947,949],{"id":948},"_4-start-the-app","4. Start the app",[455,951,952,953,956],{},"Two processes need to be running, in two more terminals, both in ",[459,954,955],{},"benchmark-espi",":",[958,959,960,975],"table",{},[961,962,963],"thead",{},[964,965,966,969,972],"tr",{},[967,968,301],"th",{},[967,970,971],{},"Command",[967,973,974],{},"Why",[976,977,978,992],"tbody",{},[964,979,980,984,989],{},[981,982,983],"td",{},"Worker",[981,985,986],{},[459,987,988],{},"uv run benchmark-espi worker",[981,990,991],{},"Executes queued runs, fetches Langfuse metrics and grades answers. The UI only queues runs.",[964,993,994,997,1002],{},[981,995,996],{},"UI",[981,998,999],{},[459,1000,1001],{},"uv run streamlit run app\u002Fstreamlit_app.py",[981,1003,1004,1005],{},"Opens on ",[464,1006,1007],{"href":1007,"rel":1008},"http:\u002F\u002Flocalhost:8501",[468],[455,1010,1011,1012,1016,1017,1019],{},"If runs sit in the queue for more than 30 seconds with nothing running, every page shows a \"Is the worker up?\"\nwarning. ",[464,1013,1015],{"href":1014},"#grading","Grading"," needs the Jev key (",[459,1018,813],{},"), or Ollama running on the host if you use the local\njudge.",[455,1021,1022,1023,1026,1027,1031],{},"To check everything works without LLM spend, queue a ",[511,1024,1025],{},"Dry run"," of a few questions from ",[464,1028,1030],{"href":1029},"#new-run","New run",". It\nshould complete within seconds, with a fake answer per question.",[487,1033,1035],{"id":1034},"docker-optional","Docker (optional)",[455,1037,1038,1039,1042],{},"Instead of step 4, you can run the worker and UI in containers. Stop any local worker and Streamlit first: they\nmustn't share ",[459,1040,1041],{},".\u002Fdata"," with the containers.",[559,1044,1046],{"className":633,"code":1045,"language":635,"meta":567,"style":567},"docker compose up -d --build\n",[459,1047,1048],{"__ignoreMap":567},[639,1049,1050],{"class":641,"line":642},[639,1051,1045],{},[455,1053,1054,1055,1058,1059,470],{},"The UI is on ",[464,1056,1007],{"href":1007,"rel":1057},[468]," again, and the containers reach the host's ESP stack through\n",[459,1060,1061],{},"host.docker.internal",[487,1063,1065],{"id":1064},"resetting","Resetting",[455,1067,1068,1069,801,1072,1075,1076,1079,1080,470],{},"To start again from no data, stop the worker and the app and delete ",[459,1070,1071],{},"data\u002Fbenchmark.db*",[459,1073,1074],{},"data\u002Fbackups\u002F"," if you\nlike). Then repeat step 3 from ",[459,1077,1078],{},"init-db",". See ",[459,1081,1082],{},"benchmark-espi\u002Fdata\u002FREADME.md",[479,1084,1086],{"id":1085},"a-typical-session","A typical session",[1088,1089,1090,1099,1106,1115,1124],"ol",{},[495,1091,1092,1098],{},[511,1093,1094,470],{},[464,1095,1097],{"href":1096},"#questions","Questions"," Import the ground-truth CSV once, then browse it by tag.",[495,1100,1101,1105],{},[511,1102,1103,470],{},[464,1104,1030],{"href":1029}," Pick questions, repeats and labels, check the call count and cost estimate, and\nqueue the run.",[495,1107,1108,1114],{},[511,1109,1110,470],{},[464,1111,1113],{"href":1112},"#runs","Runs"," Watch progress live, then look at per-question stats, charts and errors. Export to CSV or\nJSON.",[495,1116,1117,1123],{},[511,1118,1119,470],{},[464,1120,1122],{"href":1121},"#compare","Compare"," Put the new run against a baseline to find regressions, then read the answers side by\nside.",[495,1125,1126,1132],{},[511,1127,1128,470],{},[464,1129,1131],{"href":1130},"#conversation","Conversation"," Drill into a single answer: its metrics, every tool call with its input, and\nthe Langfuse traces.",[479,1134,1136],{"id":1135},"pages","Pages",[958,1138,1139,1149],{},[961,1140,1141],{},[964,1142,1143,1146],{},[967,1144,1145],{},"Page",[967,1147,1148],{},"Use it to",[976,1150,1151,1162,1171,1180,1189,1198],{},[964,1152,1153,1159],{},[981,1154,1155],{},[464,1156,1158],{"href":1157},"#home","Home",[981,1160,1161],{},"See headline counts and recent runs",[964,1163,1164,1168],{},[981,1165,1166],{},[464,1167,1097],{"href":1096},[981,1169,1170],{},"Import and browse question sets",[964,1172,1173,1177],{},[981,1174,1175],{},[464,1176,1030],{"href":1029},[981,1178,1179],{},"Configure and queue a run",[964,1181,1182,1186],{},[981,1183,1184],{},[464,1185,1113],{"href":1112},[981,1187,1188],{},"Follow a run and inspect its results",[964,1190,1191,1195],{},[981,1192,1193],{},[464,1194,1122],{"href":1121},[981,1196,1197],{},"Compare a baseline with one or more candidate runs",[964,1199,1200,1204],{},[981,1201,1202],{},[464,1203,1131],{"href":1130},[981,1205,1206],{},"Inspect one answer in detail",[455,1208,1209,1210,518,1213,608,1216,608,1219,470],{},"Pages link to each other through query parameters, so any view can be bookmarked or shared: ",[459,1211,1212],{},"runs?run_id=17",[459,1214,1215],{},"compare?baseline=7",[459,1217,1218],{},"conversation?result_id=257",[459,1220,1221],{},"questions?question_id=38",[487,1223,1158],{"id":1224},"home",[455,1226,1227,1228,1231,1232,1234],{},"The landing page shows how many runs and question sets exist, how many runs are queued or running, and how many have\ncompleted. The ",[511,1229,1230],{},"Recent runs"," table lists the latest 25 runs with their status, result and error counts, median\nelapsed time, total cost, median tokens, median tool calls, labels and whether they were dry runs. Select a row to\nopen that run on the ",[464,1233,1113],{"href":1112}," page.",[479,1236,1238],{"id":1237},"metrics-at-a-glance","Metrics at a glance",[958,1240,1241,1251],{},[961,1242,1243],{},[964,1244,1245,1248],{},[967,1246,1247],{},"Metric",[967,1249,1250],{},"Where it comes from",[976,1252,1253,1261,1273,1285,1293],{},[964,1254,1255,1258],{},[981,1256,1257],{},"Elapsed (s)",[981,1259,1260],{},"Wall-clock time from sending the question to the end of the answer stream",[964,1262,1263,1266],{},[981,1264,1265],{},"Time to first token (s)",[981,1267,1268,1269,1272],{},"Time until the first ",[459,1270,1271],{},"text-delta"," event in the stream",[964,1274,1275,1278],{},[981,1276,1277],{},"Tool calls",[981,1279,1280,1281,1284],{},"Langfuse observations named ",[459,1282,1283],{},"execute_tool*",", across every trace in the conversation",[964,1286,1287,1290],{},[981,1288,1289],{},"Cost, input\u002Foutput\u002Ftotal tokens",[981,1291,1292],{},"Langfuse session totals",[964,1294,1295,1298],{},[981,1296,1297],{},"Grade score, pass rate",[981,1299,1300,1301,808],{},"The judge's check of the answer against the question's rubric (see ",[464,1302,1015],{"href":1014},[455,1304,1305],{},"Elapsed and time to first token are recorded as soon as an answer arrives. The Langfuse metrics arrive about a\nminute later, because the worker waits for Langfuse ingestion before fetching them. Until then those columns are\nempty. All times are shown in UTC.",[1307,1308],"hr",{},[479,1310,1097],{"id":1311},"questions",[455,1313,1314],{},"A question set is a numbered list of questions, each with any number of tags. Runs always draw their questions from\none set.",[455,1316,1317],{},[474,1318],{"alt":1319,"src":1320},"Questions page","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fquestions.png",[487,1322,1324],{"id":1323},"importing-a-set","Importing a set",[455,1326,1327,1328,1331,1332,1335],{},"Expand ",[511,1329,1330],{},"Import a question set from CSV",", upload the file, give the set a name (it defaults to the file name) and\nclick ",[511,1333,1334],{},"Import",". The CSV needs these columns:",[958,1337,1338,1351],{},[961,1339,1340],{},[964,1341,1342,1345,1348],{},[967,1343,1344],{},"Column",[967,1346,1347],{},"Required",[967,1349,1350],{},"Notes",[976,1352,1353,1366,1378,1395,1407],{},[964,1354,1355,1360,1363],{},[981,1356,1357],{},[459,1358,1359],{},"Question No.",[981,1361,1362],{},"yes",[981,1364,1365],{},"The question's number within the set",[964,1367,1368,1373,1375],{},[981,1369,1370],{},[459,1371,1372],{},"Question",[981,1374,1362],{},[981,1376,1377],{},"The question text",[964,1379,1380,1389,1392],{},[981,1381,1382,1385,1386],{},[459,1383,1384],{},"Tag 1"," … ",[459,1387,1388],{},"Tag n",[981,1390,1391],{},"no",[981,1393,1394],{},"One tag per column; empty cells are ignored",[964,1396,1397,1402,1404],{},[981,1398,1399],{},[459,1400,1401],{},"Expected Answer",[981,1403,1391],{},[981,1405,1406],{},"Stored with the question for reference",[964,1408,1409,1414,1416],{},[981,1410,1411],{},[459,1412,1413],{},"Expected Route",[981,1415,1391],{},[981,1417,1418,1419,1422,1423,1426,1427,1422,1430,1433],{},"Where the router should send it: ",[459,1420,1421],{},"docs"," \u002F ",[459,1424,1425],{},"platform-docs",", or ",[459,1428,1429],{},"chat",[459,1431,1432],{},"fallback",". Blank means unset.",[455,1435,1436],{},"The same import works from the command line:",[559,1438,1439],{"className":633,"code":875,"language":635,"meta":567,"style":567},[459,1440,1441],{"__ignoreMap":567},[639,1442,1443],{"class":641,"line":642},[639,1444,875],{},[487,1446,1448],{"id":1447},"browsing-a-set","Browsing a set",[455,1450,1451,1452,1455],{},"Pick a set from ",[511,1453,1454],{},"Question set",". Then:",[492,1457,1458,1469,1475],{},[495,1459,1460,1463,1464,1468],{},[511,1461,1462],{},"Filter by tag."," Click one or more tag pills to show questions that have ",[1465,1466,1467],"em",{},"any"," of the selected tags.",[495,1470,1471,1474],{},[511,1472,1473],{},"Search."," Type to filter the question text (case-insensitive).",[495,1476,1477,1480],{},[511,1478,1479],{},"Questions per tag."," The table on the right shows how many questions carry each tag, which helps when you choose\ntags for a run.",[455,1482,1483],{},"The caption above the table shows how many questions match, for example \"12 of 79 questions\".",[455,1485,1486,1487,1490,1491,1494,1495,1498,1499,1502,1503,1506,1507,470],{},"The ",[511,1488,1489],{},"Rubric"," column shows which questions are graded, for example ",[459,1492,1493],{},"6 criteria · docs: partial"," or\n",[459,1496,1497],{},"5 criteria · data: heavy",": the number of criteria, and how much of the answer the platform docs contain (covered,\npartial or none), the dataset the data criteria describe, or the date the market facts were researched. Rubrics are\nedited in ",[459,1500,1501],{},"rubrics\u002F*.yaml"," and loaded with ",[459,1504,1505],{},"uv run benchmark-espi import-rubrics",", not on this page. See\n",[464,1508,1015],{"href":1014},[487,1510,1512],{"id":1511},"reading-a-rubric","Reading a rubric",[455,1514,1486,1515,1517,1518,1520],{},[511,1516,1489],{}," section under the table shows one question's full rubric, without needing a run. ",[511,1519,1372],{}," lists\nthe questions in the table (after the tag and search filters) that have a rubric. For the one you pick it shows:",[492,1522,1523,1526,1533],{},[495,1524,1525],{},"the docs coverage and why, the dataset, or the as-of date of the market facts;",[495,1527,1528,1529,1532],{},"the ",[511,1530,1531],{},"reference answer",", what to do if the answer says the docs don't cover it, and the notes for the judge;",[495,1534,1535,1536,1539,1540,1543,1544,1547,1548,1551,1552,733],{},"a table of the criteria: ",[511,1537,1538],{},"Kind"," (must, should, must not), the ",[511,1541,1542],{},"Criterion",", its ",[511,1545,1546],{},"Basis"," (docs, product, data\nor web) and its ",[511,1549,1550],{},"Source",", where its ground truth comes from (e.g. ",[459,1553,1554],{},"heavy\u002Fproducts-owned.json",[455,1556,1557,1558,1560,1561,470],{},"The pick is kept in the URL, so ",[459,1559,1221],{}," links straight to a rubric. The Conversation page links\nhere from ",[511,1562,1563],{},"Rubric and reference answer",[455,1565,1566],{},[474,1567],{"alt":1568,"src":1569},"A question's rubric","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fquestions-rubric.png",[455,1571,1486,1572,1575,1576,1579,1580,1583,1584,1502,1587,1079,1590,1593],{},[511,1573,1574],{},"Scenario"," column marks questions that are a turn of a multi-turn conversation, or are asked on a UI page,\ne.g. ",[459,1577,1578],{},"turn 2 of similar-products"," or ",[459,1581,1582],{},"on notebook.view (Q3 board summary)",". Scenarios are edited in\n",[459,1585,1586],{},"scenarios\u002F*.yaml",[459,1588,1589],{},"uv run benchmark-espi import-scenarios",[459,1591,1592],{},"benchmark-espi\u002Fscenarios\u002FREADME.md","\nin the data-science repo.",[487,1595,1597],{"id":1596},"expected-routes","Expected routes",[455,1599,1600,1601,1604,1605,1579,1608,1611,1612,1616],{},"Set ",[511,1602,1603],{},"Expected route"," in the table to ",[511,1606,1607],{},"Docs agent",[511,1609,1610],{},"General chat"," (or clear it). Changes save straight away.\nThe expectation belongs to the question, so it also applies to runs made before you set it. It drives the\n",[464,1613,1615],{"href":1614},"#routing","routing"," metrics: expected-agent hits and router accuracy.",[455,1618,1619],{},"To set many at once, use the command line:",[559,1621,1623],{"className":633,"code":1622,"language":635,"meta":567,"style":567},"uv run benchmark-espi expect-route -q \"ESPi Groundtruth\" --tag Docs\u002FHelp --route docs\nuv run benchmark-espi expect-route -q \"ESPi Groundtruth\" --questions 8,10 --route chat\n",[459,1624,1625,1630],{"__ignoreMap":567},[639,1626,1627],{"class":641,"line":642},[639,1628,1629],{},"uv run benchmark-espi expect-route -q \"ESPi Groundtruth\" --tag Docs\u002FHelp --route docs\n",[639,1631,1632],{"class":641,"line":648},[639,1633,1634],{},"uv run benchmark-espi expect-route -q \"ESPi Groundtruth\" --questions 8,10 --route chat\n",[1307,1636],{},[479,1638,1030],{"id":1639},"new-run",[455,1641,1642,1643,1234],{},"This page configures a run and puts it in the queue. It doesn't execute anything itself: the worker picks the run up\nand you follow it on the ",[464,1644,1113],{"href":1112},[455,1646,1647],{},[474,1648],{"alt":1649,"src":1650},"New run page","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fnew-run.png",[487,1652,1654],{"id":1653},"choosing-questions","Choosing questions",[958,1656,1657,1667],{},[961,1658,1659],{},[964,1660,1661,1664],{},[967,1662,1663],{},"Field",[967,1665,1666],{},"Effect",[976,1668,1669,1678,1691],{},[964,1670,1671,1675],{},[981,1672,1673],{},[511,1674,1454],{},[981,1676,1677],{},"The set to draw from",[964,1679,1680,1685],{},[981,1681,1682],{},[511,1683,1684],{},"Tags",[981,1686,1687,1688,1690],{},"Include questions that have ",[1465,1689,1467],{}," of these tags",[964,1692,1693,1698],{},[981,1694,1695],{},[511,1696,1697],{},"Specific questions",[981,1699,1700],{},"Include these question numbers",[455,1702,1703,1704,522,1706,1708,1709,1712,1713,1715],{},"Leave both ",[511,1705,1684],{},[511,1707,1697],{}," empty to run the whole set. Expand ",[511,1710,1711],{},"Selected questions"," to check\nexactly which questions will run; its ",[511,1714,1574],{}," column marks conversation turns and page contexts. The selection\nis snapshotted when you queue the run, so later edits to the set don't change it.",[487,1717,1719],{"id":1718},"conversations-and-page-contexts","Conversations and page contexts",[455,1721,1722,1723,1725],{},"Some questions are turns of a multi-turn conversation, or are asked on a UI page (see ",[459,1724,1592],{},"\nin the data-science repo). When the selection includes any, a section appears above the call count:",[492,1727,1728,1734],{},[495,1729,1730,1733],{},[511,1731,1732],{},"Adds Q…",": selecting a follow-up brings its conversation's earlier turns with it. They're asked first, in the\nsame ESP conversation. Each repeat is a separate conversation.",[495,1735,1736,1739,1740,1742,1743,1746],{},[511,1737,1738],{},"Page contexts",": how many selected questions are asked on a page, and which data snapshot is loaded (from\ninfra-devenv-pro's ",[459,1741,601],{},"). A warning appears when that differs from the snapshot the scenario file was\nwritten against. ",[511,1744,1745],{},"Check page contexts"," fetches each page's entity now, as the worker will before asking. It\nshows any that are missing, with ids to use instead.",[487,1748,1750],{"id":1749},"run-settings","Run settings",[958,1752,1753,1761],{},[961,1754,1755],{},[964,1756,1757,1759],{},[967,1758,1663],{},[967,1760,1350],{},[976,1762,1763,1776,1786,1796,1810,1827],{},[964,1764,1765,1770],{},[981,1766,1767],{},[511,1768,1769],{},"Run name",[981,1771,1772,1773,470],{},"Required. Describe what's being tested, e.g. ",[459,1774,1775],{},"docs agent – new search prompt",[964,1777,1778,1783],{},[981,1779,1780],{},[511,1781,1782],{},"Description",[981,1784,1785],{},"Optional notes on what changed. Shown at the top of the run.",[964,1787,1788,1793],{},[981,1789,1790],{},[511,1791,1792],{},"Repeats per question",[981,1794,1795],{},"1–20. Use 3 or more to see how much answers and latency vary.",[964,1797,1798,1803],{},[981,1799,1800],{},[511,1801,1802],{},"Concurrency",[981,1804,1805,1806,1809],{},"1–16 parallel requests (default from ",[459,1807,1808],{},"DEFAULT_CONCURRENCY","). Higher values speed the run up but change the latency figures, so compare runs that used the same concurrency.",[964,1811,1812,1817],{},[981,1813,1814],{},[511,1815,1816],{},"Labels",[981,1818,1819,1822,1823,1826],{},[459,1820,1821],{},"key=value",", one per line, e.g. ",[459,1824,1825],{},"prompt=v12",". Shown on the run and on the Home table; useful for recording the prompt version or model.",[964,1828,1829,1833],{},[981,1830,1831],{},[511,1832,1025],{},[981,1834,1835],{},"Uses a fake client: no ESP calls and no LLM spend. Good for checking the pipeline. Dry runs aren't enriched from Langfuse.",[487,1837,1839],{"id":1838},"backend-build","Backend build",[455,1841,1842,1843,522,1846,1848,1849,1852,1853,1856],{},"Record which ",[511,1844,1845],{},"platform-api",[511,1847,1425],{}," builds the run is against, so a before\u002Fafter comparison says what\nchanged. Each has a ",[511,1850,1851],{},"commit"," and a ",[511,1854,1855],{},"branch"," field.",[492,1858,1859,1882,1892,1913,1916],{},[495,1860,1861,1864,1865,522,1867,1869,1870,522,1873,1876,1877,522,1879,1881],{},[511,1862,1863],{},"Pre-filled from your local checkouts"," (",[459,1866,585],{},[459,1868,591],{},", by default\n",[459,1871,1872],{},"~\u002Fdevelopment\u002Fbackend\u002Fplatform-api",[459,1874,1875],{},"~\u002Fdevelopment\u002Ffrontend\u002Fplatform-docs","). That's only a guess: if the\nservice is running from a pulled image, or was started before you switched branch, correct it. A note appears when\na checkout has uncommitted changes. In Docker, compose mounts the checkouts read-only (override the host paths with\n",[459,1878,607],{},[459,1880,611],{},"); there the dirty check is off, so it isn't flagged.",[495,1883,1884,1887,1888,1891],{},[511,1885,1886],{},"Edit or clear it"," as needed. An edited entry is recorded as ",[459,1889,1890],{},"manual","; a blank commit records nothing.",[495,1893,1894,1897,1898,1900,1901,1904,1905,1908,1909,1912],{},[511,1895,1896],{},"A CI-built platform-api reports its own commit"," through ",[459,1899,742],{},", and that wins when the run starts. A\nlocal platform-api reports ",[459,1902,1903],{},"not-assigned"," unless it's started with ",[459,1906,1907],{},"COMMIT_INFO=\u003Csha>"," (e.g.\n",[459,1910,1911],{},"COMMIT_INFO=$(git rev-parse HEAD)","), which is worth doing: then the build is recorded without typing.",[495,1914,1915],{},"Dry runs don't record a build.",[495,1917,1918,1919,1922,1923,733],{},"The run also records the loaded ",[511,1920,1921],{},"data snapshot"," (the infra-devenv-pro volume tags, ",[459,1924,597],{},[487,1926,1928],{"id":1927},"call-count-and-cost-estimate","Call count and cost estimate",[455,1930,1931],{},"The blue box updates as you change the settings:",[1933,1934,1935],"blockquote",{},[455,1936,1937,1940],{},[511,1938,1939],{},"79 questions × 3 repeats = 237 calls"," · est. cost ≈ 21.481",[455,1942,1943],{},"The estimate is the average cost of every successful result so far, multiplied by the number of calls. It's a rough\nguide: different agents and questions cost very different amounts. It's hidden for dry runs and until there's some\nhistory.",[487,1945,1947],{"id":1946},"queueing","Queueing",[455,1949,1950,1951,1954,1955,1234],{},"Click ",[511,1952,1953],{},"Queue run",". The app creates one pending result per question × repeat and takes you straight to the run on\nthe ",[464,1956,1113],{"href":1112},[455,1958,1959,1960,1963,1964,1967],{},"From the command line, ",[459,1961,1962],{},"benchmark-espi run ... --queue"," does the same thing; without ",[459,1965,1966],{},"--queue"," it runs inline in\nthat terminal instead:",[559,1969,1971],{"className":633,"code":1970,"language":635,"meta":567,"style":567},"uv run benchmark-espi run -q 1 --name \"docs agent – new search prompt\" --tag Docs\u002FHelp --repeats 3 --label prompt=v12 --queue\n",[459,1972,1973],{"__ignoreMap":567},[639,1974,1975],{"class":641,"line":642},[639,1976,1970],{},[455,1978,1979,1980,1983,1984,1987,1988,470],{},"The CLI also pre-fills the backend build from the local checkouts (turn that off with ",[459,1981,1982],{},"--no-detect-builds","). Give a\ncomponent explicitly with ",[459,1985,1986],{},"--build COMPONENT=COMMIT[@BRANCH]",", e.g. ",[459,1989,1990],{},"--build platform-api=11ca6af96@feature\u002Fjev",[1307,1992],{},[479,1994,1113],{"id":1995},"runs",[455,1997,1998,1999,2002],{},"The Runs page shows one run at a time. Pick it from the ",[511,2000,2001],{},"Run"," selector, or arrive from the Home table or straight\nafter queueing.",[455,2004,2005],{},[474,2006],{"alt":2007,"src":2008},"Run results","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Frun-results.png",[487,2010,2012],{"id":2011},"header","Header",[455,2014,2015],{},"Under the run's status and name you'll find:",[492,2017,2018,2021,2024,2029,2058,2072,2075],{},[495,2019,2020],{},"the description, if there is one",[495,2022,2023],{},"when it was created, how long it took, the repeats and the concurrency",[495,2025,2026,2027],{},"the ESP backend version and commit tag, recorded by the worker's preflight call to ",[459,2028,742],{},[495,2030,1528,2031,522,2033,2035,2036,1426,2039,2042,2043,2046,2047,2050,2051,2054,2055,2057],{},[511,2032,1845],{},[511,2034,1425],{}," builds, e.g. ",[459,2037,2038],{},"11ca6af96 (feature\u002Fjev) · reported",[459,2040,2041],{},"not recorded",".\nThe source says how far to trust it: ",[459,2044,2045],{},"reported"," (platform-api's ",[459,2048,2049],{},"\u002Finfo"," said so), ",[459,2052,2053],{},"checkout"," (the local\ncheckout's HEAD when the run was queued) or ",[459,2056,1890],{}," (typed or edited). A warning appears when platform-api\nreported a different commit from the one entered for the run.",[495,2059,2060,2061,2064,2065,2068,2069,2071],{},"the loaded ",[511,2062,2063],{},"data"," snapshot, e.g. ",[459,2066,2067],{},"heavy (all volumes)",", with a warning when the run asked questions on pages\nwritten against a different snapshot (see ",[459,2070,1592],{}," in the data-science repo)",[495,2073,2074],{},"the labels, and the question filter (tags and question numbers) the run was created with",[495,2076,2077],{},"any run-level error, for example a failed login",[487,2079,2081],{"id":2080},"actions","Actions",[958,2083,2084,2097],{},[961,2085,2086],{},[964,2087,2088,2091,2094],{},[967,2089,2090],{},"Button",[967,2092,2093],{},"When",[967,2095,2096],{},"What it does",[976,2098,2099,2112,2125,2137,2149,2165],{},[964,2100,2101,2106,2109],{},[981,2102,2103],{},[511,2104,2105],{},"Cancel run",[981,2107,2108],{},"Queued or running",[981,2110,2111],{},"Stops the run. Results already in flight finish; the rest are marked cancelled.",[964,2113,2114,2119,2122],{},[981,2115,2116],{},[511,2117,2118],{},"Re-fetch Langfuse metrics",[981,2120,2121],{},"Finished, not a dry run",[981,2123,2124],{},"Re-queues every result for enrichment. Use it when metrics are missing because Langfuse was slow or unreachable.",[964,2126,2127,2132,2134],{},[981,2128,2129],{},[511,2130,2131],{},"Grade answers",[981,2133,2121],{},[981,2135,2136],{},"Queues the run's answers for the judge again, e.g. after a rubric change or for a run made before grading.",[964,2138,2139,2143,2146],{},[981,2140,2141],{},[511,2142,1839],{},[981,2144,2145],{},"Not a dry run",[981,2147,2148],{},"Edits the recorded builds, e.g. to backfill an old run or correct a guess from the checkout",[964,2150,2151,2156,2159],{},[981,2152,2153],{},[511,2154,2155],{},"Compare…",[981,2157,2158],{},"Always",[981,2160,2161,2162,2164],{},"Opens ",[464,2163,1122],{"href":1121}," with this run as the baseline",[964,2166,2167,2172,2175],{},[981,2168,2169],{},[511,2170,2171],{},"Delete",[981,2173,2174],{},"Finished",[981,2176,2177],{},"Deletes the run and all its results, after a confirmation",[487,2179,2181],{"id":2180},"results-tab","Results tab",[455,2183,2184,2185,1864,2188,608,2191,2194,2195,1864,2197,608,2200,608,2203,518,2206,2209,2210,2213],{},"One row per question × repeat, showing status, the ",[464,2186,2187],{"href":1014},"grade",[511,2189,2190],{},"Grade",[511,2192,2193],{},"Score","), the ",[464,2196,1615],{"href":1614},[511,2198,2199],{},"Answered by",[511,2201,2202],{},"Route",[511,2204,2205],{},"p(docs)",[511,2207,2208],{},"Route reason","), elapsed time, time to first token, cost, tokens, tool calls, the Langfuse enrichment status and\nany error. While the run is active the table refreshes every 3 seconds and a progress\nbar shows ",[459,2211,2212],{},"done \u002F total results · errors",". When the run finishes, the whole page refreshes.",[455,2215,2216,2217,1234],{},"Select a row to open that answer on the ",[464,2218,1131],{"href":1130},[487,2220,2222],{"id":2221},"per-question-stats-tab","Per-question stats tab",[455,2224,2225],{},[474,2226],{"alt":2227,"src":2228},"Per-question stats","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Frun-stats.png",[455,2230,2231,2232,2234,2235,2238],{},"The headline figures cover the whole run: results, errors (hover for the error rate), median and P90 elapsed, total\ncost and median tool calls. Below them, choose which ",[511,2233,325],{}," to show. For every question the table then gives\nthe number of successful results (",[459,2236,2237],{},"n",") and the min, median, mean, max and standard deviation of each metric across\nits repeats. Failed results are left out of these figures.",[455,2240,2241,2242,2244,2245,608,2248,608,2251,608,2254,2257,2258,470],{},"A second row shows the ",[464,2243,1615],{"href":1614}," metrics: ",[511,2246,2247],{},"Docs agent share",[511,2249,2250],{},"Expected hits",[511,2252,2253],{},"Router\naccuracy",[511,2255,2256],{},"Low confidence"," (Jev chose docs, but the confidence gate sent it to general chat) and ",[511,2259,2260],{},"Docs\nfallbacks",[487,2262,2264],{"id":2263},"charts-tab","Charts tab",[455,2266,2267],{},[474,2268],{"alt":2269,"src":2270},"Charts","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Frun-charts.png",[455,2272,2273],{},"Pick a metric to plot it per question, with one bar per repeat. If the run has more than one repeat, a box plot\nunderneath shows the spread across repeats. Langfuse metrics (cost, tokens, tool calls) only chart once enrichment\nhas finished.",[455,2275,2276,2277,2280,2281,2284],{},"Turn on ",[511,2278,2279],{},"Colour by the agent that answered"," to colour the bars by agent instead of repeat: purple for the docs\nagent and teal for general chat, the same colours as the analysis charts. For runs with Jev decisions, ",[511,2282,2283],{},"Where Jev\nrouted each question"," plots Jev's probability that the docs can answer each question (one dot per repeat), with\nthe confidence gate as a solid line and p = 0.5, where Jev's choice flips, dotted.",[487,2286,2287],{"id":1615},"Routing",[455,2289,2290,2291,2294,2295,2298,2299,2302],{},"Since run 19, platform-api routes each ESPi question with ",[511,2292,2293],{},"Jev",": it either goes to the ",[511,2296,2297],{},"docs agent"," (a\nLangGraph4j graph over the platform docs) or falls back to ",[511,2300,2301],{},"general chat",". The benchmark reads the decision from\nLangfuse during enrichment. Each result is in one of four states:",[958,2304,2305,2315],{},[961,2306,2307],{},[964,2308,2309,2312],{},[967,2310,2311],{},"State",[967,2313,2314],{},"Meaning",[976,2316,2317,2332,2347,2361],{},[964,2318,2319,2322],{},[981,2320,2321],{},"Jev decision",[981,2323,2324,2325,1426,2328,2331],{},"Jev was consulted. Shows the route, the reason (",[459,2326,2327],{},"jev",[459,2329,2330],{},"low confidence"," when the confidence gate overrode Jev's choice), p(docs) and the confidence.",[964,2333,2334,2337],{},[981,2335,2336],{},"Router, no Jev",[981,2338,2339,2340,608,2343,2346],{},"The router ran but didn't ask Jev (e.g. ",[459,2341,2342],{},"jev disabled",[459,2344,2345],{},"message has attachments","). Shows the route and reason only.",[964,2348,2349,2354],{},[981,2350,2351],{},[511,2352,2353],{},"No router",[981,2355,2356,2357,2360],{},"The build has no router, e.g. current ",[459,2358,2359],{},"main"," or run 18. Every answer came from general chat. This is a normal state, not an error.",[964,2362,2363,2366],{},[981,2364,2365],{},"Not fetched",[981,2367,2368,2369,2371],{},"The result was enriched before routing was captured. Use ",[511,2370,2118],{}," to fill it in.",[455,2373,2374,2376,2377,2379,2380,733],{},[511,2375,2199],{}," can differ from ",[511,2378,2202],{},": if the docs agent's retrieval fails, a question routed to the docs agent\nis answered by general chat (counted as a ",[511,2381,2382],{},"docs fallback",[455,2384,2385,2386,2389,2390,2393],{},"The metrics use each question's ",[511,2387,2388],{},"expected route",", set on the ",[464,2391,1097],{"href":2392},"#expected-routes"," page:",[492,2395,2396,2401,2412],{},[495,2397,2398,2400],{},[511,2399,2247],{},": the fraction of answers produced by the docs agent.",[495,2402,2403,2405,2406,2408,2409,2411],{},[511,2404,2250],{},": the agent that answered matches the expected route. This works for every build,\nincluding ones with no router, so a ",[459,2407,2359],{}," run can be compared with a Jev branch run: ",[459,2410,2359],{}," only scores on the\nquestions expected to go to general chat.",[495,2413,2414,2417,2418,2421],{},[511,2415,2416],{},"Router accuracy",": the router's route matches the expected route. It's ",[511,2419,2420],{},"n\u002Fa"," for a build with no router,\nrather than 0%.",[487,2423,1015],{"id":2424},"grading",[455,2426,2427,2428,2431,2432,2434,2435,2438,2439,2442],{},"Answers to questions that have a ",[511,2429,2430],{},"rubric"," are graded by a judge model once they finish. The rubrics live in\n",[459,2433,1501],{}," and are loaded with ",[459,2436,2437],{},"benchmark-espi import-rubrics","; the ",[464,2440,1097],{"href":2441},"#reading-a-rubric","\npage shows each question's full rubric. Each rubric has a reference answer and a list of criteria, each checkable\nagainst the answer:",[958,2444,2445,2456],{},[961,2446,2447],{},[964,2448,2449,2451,2453],{},[967,2450,1538],{},[967,2452,2314],{},[967,2454,2455],{},"Points",[976,2457,2458,2469,2480],{},[964,2459,2460,2463,2466],{},[981,2461,2462],{},"Must",[981,2464,2465],{},"Required for a pass",[981,2467,2468],{},"2",[964,2470,2471,2474,2477],{},[981,2472,2473],{},"Should",[981,2475,2476],{},"Extra credit, e.g. a correct product fact the docs don't mention",[981,2478,2479],{},"1",[964,2481,2482,2485,2488],{},[981,2483,2484],{},"Must not",[981,2486,2487],{},"A known failure: invented UI, a wrong definition, claiming the docs say something they don't",[981,2489,2490],{},"−2, and the answer fails",[455,2492,2493,2494,2497,2498,2500,2501,2504,2505,2507,2508,2510,2511,2514],{},"Each criterion also has a ",[511,2495,2496],{},"basis",", which says where its ground truth comes from: ",[511,2499,1421],{}," (stated in\nplatform-docs), ",[511,2502,2503],{},"product"," (true of the product but not documented), ",[511,2506,2063],{}," (a fact about the loaded data\nsnapshot, e.g. ",[459,2509,513],{},") or ",[511,2512,2513],{},"web"," (an external market fact, as of the rubric's date).",[455,2516,2517],{},"The judge says for each criterion whether the answer meets it (yes, no, or n\u002Fa for a conditional criterion). The\napp then works out:",[492,2519,2520,2525],{},[495,2521,2522,2524],{},[511,2523,2193],{},": points earned, minus violations, out of the points possible (0 to 1).",[495,2526,2527,2529,2530,2533,2534,2537,2538,2541],{},[511,2528,2190],{},": ",[511,2531,2532],{},"Pass"," (every must met), ",[511,2535,2536],{},"Partial"," (a must missed) or ",[511,2539,2540],{},"Fail"," (a must-not violated).",[455,2543,1486,2544,2546,2547,608,2550,608,2553,2556,2557,2560],{},[511,2545,2227],{}," tab shows ",[511,2548,2549],{},"Pass rate",[511,2551,2552],{},"Mean score",[511,2554,2555],{},"Must-not violations"," and the number graded,\nand ",[511,2558,2559],{},"Grade score"," is available as a chart metric. On a question the docs don't cover, an answer that honestly says\nso can still pass, because criteria based only on product facts aren't musts there.",[455,2562,2563,2564,2567,2568,956],{},"There are two judges, chosen with ",[459,2565,2566],{},"JUDGE_PROVIDER"," in ",[459,2569,577],{},[958,2571,2572,2588],{},[961,2573,2574],{},[964,2575,2576,2579,2582,2585],{},[967,2577,2578],{},"Judge",[967,2580,2581],{},"Setting",[967,2583,2584],{},"Speed",[967,2586,2587],{},"What a grade shows",[976,2589,2590,2612],{},[964,2591,2592,2595,2602,2605],{},[981,2593,2594],{},"TypeSafe's Jev (default)",[981,2596,2597,608,2600],{},[459,2598,2599],{},"JUDGE_PROVIDER=jev",[459,2601,813],{},[981,2603,2604],{},"Well under a second per answer: a run is graded seconds after it finishes",[981,2606,2607,2608,2611],{},"Met or not, with ",[511,2609,2610],{},"P(yes)",", the judge's probability. Near 0.5 is a close call.",[964,2613,2614,2620,2631,2634],{},[981,2615,2616,2617],{},"A chat model, e.g. local Ollama with ",[459,2618,2619],{},"qwen3.8:27b",[981,2621,2622,608,2625,608,2628],{},[459,2623,2624],{},"JUDGE_PROVIDER=openai",[459,2626,2627],{},"JUDGE_BASE_URL",[459,2629,2630],{},"JUDGE_MODEL",[981,2632,2633],{},"About 100 s per answer: a 57-answer run takes about 95 minutes",[981,2635,2636,2637,2640,2641,2644],{},"Met or not, with the ",[511,2638,2639],{},"evidence"," quoted from the answer, ",[511,2642,2643],{},"why",", and a one-line summary",[455,2646,2647,2648,2651,2652,2655,2656,2658],{},"Both agree with Claude's reference grades on about 91% of verdicts. If the judge can't be reached (no key, Ollama not\nrunning, rate limited), answers wait in the queue (",[511,2649,2650],{},"Queued","). A grade made before its rubric was edited is marked\n",[511,2653,2654],{},"(stale)",": use ",[511,2657,2131],{}," to re-grade. Grades made by one judge aren't directly comparable with the other's:\nre-grade both runs with the same judge before comparing them.",[487,2660,2662],{"id":2661},"errors-tab","Errors tab",[455,2664,2665],{},"Lists every result that errored, with its question number and repeat. Stream errors from the copilot appear here.",[487,2667,2669],{"id":2668},"export","Export",[455,2671,2672,522,2675,2678],{},[511,2673,2674],{},"Download CSV",[511,2676,2677],{},"Download JSON"," export every result in the run with its question, tags and metrics. The link\nunder the buttons opens the Langfuse project.",[1307,2680],{},[479,2682,1122],{"id":2683},"compare",[455,2685,2686,2687,2690,2691,2694],{},"Compare puts one ",[511,2688,2689],{},"baseline"," run against one or more ",[511,2692,2693],{},"candidate"," runs. Use it to check whether a change (a new\nprompt, model or backend build) made the copilot faster, cheaper or more reliable.",[455,2696,2697,2698,2700,2701,2703],{},"You need at least two completed, cancelled or running runs. ",[511,2699,2155],{}," on the ",[464,2702,1113],{"href":1112}," page opens this\npage with that run as the baseline.",[455,2705,2706],{},[474,2707],{"alt":2708,"src":2709},"Compare summary and per-question table","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fcompare-summary.png",[487,2711,2713],{"id":2712},"choosing-runs","Choosing runs",[492,2715,2716,2722],{},[495,2717,2718,2721],{},[511,2719,2720],{},"Baseline",": the run to measure against.",[495,2723,2724,2727],{},[511,2725,2726],{},"Candidates",": one or more runs to compare with it. By default this is the most recent run other than the\nbaseline.",[455,2729,2730],{},"Runs are compared question by question, so choose runs that share questions. Compare runs that used the same\nconcurrency too, because concurrency affects latency.",[487,2732,2734],{"id":2733},"summary","Summary",[455,2736,2737],{},"For each run, six headline figures:",[958,2739,2740,2748],{},[961,2741,2742],{},[964,2743,2744,2746],{},[967,2745,1247],{},[967,2747],{},[976,2749,2750,2757,2765,2772,2779,2786],{},[964,2751,2752,2755],{},[981,2753,2754],{},"Median elapsed",[981,2756],{},[964,2758,2759,2762],{},[981,2760,2761],{},"P90 elapsed",[981,2763,2764],{},"Slowest 10% of answers",[964,2766,2767,2770],{},[981,2768,2769],{},"Median cost",[981,2771],{},[964,2773,2774,2777],{},[981,2775,2776],{},"Median tokens",[981,2778],{},[964,2780,2781,2784],{},[981,2782,2783],{},"Median tool calls",[981,2785],{},[964,2787,2788,2791],{},[981,2789,2790],{},"Error rate",[981,2792],{},[455,2794,2795,2796,470],{},"Each candidate shows its change against the baseline as a percentage (the error rate as a change in percentage\npoints). Lower is better for all of them, so ",[511,2797,2798],{},"green means an improvement and red means a regression",[455,2800,2801,2802,608,2804,522,2806,2808,2809,2811,2812,2814,2815,2817],{},"Once routing has been fetched, a second row shows ",[511,2803,2247],{},[511,2805,2250],{},[511,2807,2253],{}," (see ",[464,2810,2287],{"href":1614},"), with changes in percentage points. Higher is better for these. A\nbaseline with no router, such as a ",[459,2813,2359],{}," build, shows router accuracy as ",[511,2816,2420],{}," and no delta, but its\nexpected-agent hits still compare fairly.",[455,2819,2820,2821,608,2823,522,2825,2827,2828,2830],{},"Once answers have been graded, another row shows ",[511,2822,2549],{},[511,2824,2552],{},[511,2826,2555],{}," (see\n",[464,2829,1015],{"href":1014},"). Higher is better for the first two and lower for violations, and the colours follow\nthat. Grades only compare fairly when both runs have the same questions graded.",[487,2832,2834],{"id":2833},"per-question","Per question",[455,2836,2837,2838,2840,2841,2843,2844,2847,2848,2851,2852,2855,2856,2859,2860,2863,2864,2866,2867,2869],{},"Pick a ",[511,2839,1247],{},". The table shows each question's median value in every run, plus a Δ% column per candidate.\nFor ",[511,2842,2559],{}," higher is better, so ",[511,2845,2846],{},"Sort by biggest regression"," puts the biggest drops first. The answer\ntitles under ",[511,2849,2850],{},"Answers side by side"," include each answer's grade.\nAn ",[511,2853,2854],{},"· agent"," column per run shows how many of its answers came from the docs agent (e.g. ",[459,2857,2858],{},"docs 2\u002F3","), or ",[459,2861,2862],{},"no router",". Turn on ",[511,2865,2279],{}," to plot each answer as a point in the agent colours instead of\nthe box plot.\n",[511,2868,2846],{}," (on by default) puts the questions that got worst in the first candidate at the top.",[455,2871,2872],{},"Only questions that were answered successfully in every selected run are included. Any others are listed in a\ncaption above the table.",[455,2874,2875],{},[474,2876],{"alt":2877,"src":2878},"Distribution per question","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fcompare-chart.png",[455,2880,2881],{},"The box plot underneath shows each run's distribution per question, so you can tell a real shift from noise across\nrepeats.",[487,2883,2850],{"id":2884},"answers-side-by-side",[455,2886,2887],{},[474,2888],{"alt":2850,"src":2889},"\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fcompare-answers.png",[455,2891,2892,2893,2896,2897,2900,2901,1234],{},"Pick a question to see each run's answers in columns, one expander per repeat. Each expander title shows the\nelapsed time and number of tool calls. Inside you'll find the answer, any error, the tools used in order (e.g.\n",[459,2894,2895],{},"platform_docs_search_docs → platform_docs_read_docs","), and an ",[511,2898,2899],{},"Open conversation"," link to the\n",[464,2902,1131],{"href":1130},[1307,2904],{},[479,2906,1131],{"id":2907},"conversation",[455,2909,2910,2911,2913,2914,2916,2917,2919,2920,2923],{},"The Conversation page shows everything recorded for one result: one question, asked once. You usually get here by\nselecting a row on the ",[464,2912,1113],{"href":1112}," page or clicking ",[511,2915,2899],{}," on ",[464,2918,1122],{"href":1121},". You can also\ntype a ",[511,2921,2922],{},"Result id"," directly.",[455,2925,2926],{},[474,2927],{"alt":2928,"src":2929},"Conversation page","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fconversation.png",[487,2931,2933],{"id":2932},"overview","Overview",[492,2935,2936,2942,2945,2948,2955,2962,2989,2995],{},[495,2937,2938,2941],{},[511,2939,2940],{},"Back to #… run"," returns to the run this result belongs to.",[495,2943,2944],{},"The heading shows the question number and text, then the repeat number, status and tags.",[495,2946,2947],{},"For a scenario question, a caption says which conversation turn it is and which page it\nwas asked on. The conversation's other turns are listed as expanders with their answers and a link to each. A turn\nthat wasn't asked says why: its page's entity wasn't in the loaded data, or an earlier turn failed.",[495,2949,2950,2951,2954],{},"The metrics row shows elapsed time, time to first token, tool calls, cost, and input and output tokens. Langfuse\nfigures show ",[459,2952,2953],{},"–"," until enrichment has finished.",[495,2956,2957,2959,2960,470],{},[511,2958,2287],{}," says where the router sent the question, why, and who answered. For Jev decisions, tiles show Jev's\np(docs), its confidence, the gate and Jev's latency. A caption gives the model, any override by the\nconfidence gate, and whether the route matched the question's expected route. A warning appears when the docs\nagent was chosen but general chat answered. For a build with no router it just says so. See\n",[464,2961,2287],{"href":1614},[495,2963,2964,2966,2967,2969,2970,2973,2974,2976,2977,2979,2980,470,2982,2985],{},[511,2965,2190],{}," shows the judge's verdict (Pass, Partial or Fail), tiles for the score, whether every must criterion was\nmet, the number of must-not violations and the judge's time, and a table with the judge's call on each criterion:\nthe criterion and whether it was met. A Jev grade adds ",[511,2968,2610],{},", the judge's probability (met at 0.5 or more, so\nvalues near 0.5 are close calls). A chat-model grade adds the evidence quoted from the answer, why, and a one-line\nsummary next to the verdict. For a must-not criterion, \"yes\" means the answer made that mistake. A caption names\nthe judge model and flags a ",[511,2971,2972],{},"stale"," grade. ",[511,2975,1563],{}," shows what the judge was given, with a link to\nthe question's full rubric on the ",[464,2978,1097],{"href":2441}," page. See ",[464,2981,1015],{"href":1014},[2983,2984],"br",{},[474,2986],{"alt":2987,"src":2988},"Grade section of a Jev-graded answer","\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fconversation-grade.png",[495,2990,2991,2994],{},[511,2992,2993],{},"Open in Langfuse"," opens the Langfuse session for this conversation (the session id is the ESP conversation id).\nIt appears when Langfuse is configured.",[495,2996,2997],{},"Any stream error is shown in red. If Langfuse enrichment failed, a caption gives the reason.",[487,2999,3001],{"id":3000},"answer-tab","Answer tab",[455,3003,3004,3005,3008],{},"The answer, rendered as markdown. Expand ",[511,3006,3007],{},"Raw text"," to see or copy the unrendered markdown.",[487,3010,3012],{"id":3011},"tool-calls-tab","Tool calls tab",[455,3014,3015],{},[474,3016],{"alt":1277,"src":3017},"\u002Fimages\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi\u002Fconversation-tools.png",[455,3019,3020,3021,3023,3024,3027],{},"Every ",[459,3022,1283],{}," call the copilot made, in order, with the trace it belongs to, the tool name, its input and\nhow long it took. This is the quickest way to see ",[1465,3025,3026],{},"how"," the agent found its answer, for example which docs it\nsearched for and then read.",[487,3029,3031],{"id":3030},"traces-tab","Traces tab",[455,3033,3034,3035,470],{},"A table of the Langfuse traces in the conversation: name, id, number of observations, tool calls, cost and elapsed\ntime. The trace name depends on the agent, e.g. ",[459,3036,3037],{},"copilot-platform-docs",[455,3039,3040,3043],{},[511,3041,3042],{},"Load observation trees from Langfuse"," fetches each trace live and shows its full observation tree (spans,\ngenerations and tool calls, nested). It needs Langfuse keys and a real conversation, so it isn't available for dry\nruns. Fetched trees are cached for 10 minutes.",[3045,3046,3047],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":567,"searchDepth":648,"depth":648,"links":3049},[3050,3059,3060,3063,3064,3070,3078,3089,3095],{"id":481,"depth":648,"text":482,"children":3051},[3052,3053,3054,3055,3056,3057,3058],{"id":489,"depth":654,"text":490},{"id":622,"depth":654,"text":623},{"id":684,"depth":654,"text":685},{"id":771,"depth":654,"text":772},{"id":948,"depth":654,"text":949},{"id":1034,"depth":654,"text":1035},{"id":1064,"depth":654,"text":1065},{"id":1085,"depth":648,"text":1086},{"id":1135,"depth":648,"text":1136,"children":3061},[3062],{"id":1224,"depth":654,"text":1158},{"id":1237,"depth":648,"text":1238},{"id":1311,"depth":648,"text":1097,"children":3065},[3066,3067,3068,3069],{"id":1323,"depth":654,"text":1324},{"id":1447,"depth":654,"text":1448},{"id":1511,"depth":654,"text":1512},{"id":1596,"depth":654,"text":1597},{"id":1639,"depth":648,"text":1030,"children":3071},[3072,3073,3074,3075,3076,3077],{"id":1653,"depth":654,"text":1654},{"id":1718,"depth":654,"text":1719},{"id":1749,"depth":654,"text":1750},{"id":1838,"depth":654,"text":1839},{"id":1927,"depth":654,"text":1928},{"id":1946,"depth":654,"text":1947},{"id":1995,"depth":648,"text":1113,"children":3079},[3080,3081,3082,3083,3084,3085,3086,3087,3088],{"id":2011,"depth":654,"text":2012},{"id":2080,"depth":654,"text":2081},{"id":2180,"depth":654,"text":2181},{"id":2221,"depth":654,"text":2222},{"id":2263,"depth":654,"text":2264},{"id":1615,"depth":654,"text":2287},{"id":2424,"depth":654,"text":1015},{"id":2661,"depth":654,"text":2662},{"id":2668,"depth":654,"text":2669},{"id":2683,"depth":648,"text":1122,"children":3090},[3091,3092,3093,3094],{"id":2712,"depth":654,"text":2713},{"id":2733,"depth":654,"text":2734},{"id":2833,"depth":654,"text":2834},{"id":2884,"depth":654,"text":2850},{"id":2907,"depth":648,"text":1131,"children":3096},[3097,3098,3099,3100],{"id":2932,"depth":654,"text":2933},{"id":3000,"depth":654,"text":3001},{"id":3011,"depth":654,"text":3012},{"id":3030,"depth":654,"text":3031},"How to assess the performance of ESPi across platform changes.","md",null,{},true,{"title":413,"description":3101},"7FvSQu4S_ohzN16SEAtcRLGqWov1B6SNqdLiBx0Ym5M",[3109,3111],{"title":409,"path":410,"stem":411,"description":3110,"children":-1},"How to evaluate LLM agents with Langfuse — datasets, evaluators, and experiments.",{"title":417,"path":418,"stem":419,"description":567,"children":-1},1795106630984]