[{"data":1,"prerenderedAt":4494},["ShallowReactive",2],{"navigation_docs":3,"-data-ops-llm-ops-evaluations":449,"-data-ops-llm-ops-evaluations-surround":4489},[4,8,72,102,248,294,331,352,445],{"title":5,"path":6,"stem":7},"Introduction","\u002Fintroduction","0.introduction",{"title":9,"icon":10,"path":11,"stem":12,"children":13,"page":67},"Company","i-lucide-building-2","\u002Fcompany","1.company",[14,18,22,26,30,34,38,42,46,50,54,68],{"title":15,"path":16,"stem":17},"About","\u002Fcompany\u002Fabout","1.company\u002F0.about",{"title":19,"path":20,"stem":21},"Values","\u002Fcompany\u002Fvalues","1.company\u002F1.values",{"title":23,"path":24,"stem":25},"Communication","\u002Fcompany\u002Fcommunication","1.company\u002Fcommunication",{"title":27,"path":28,"stem":29},"Competition","\u002Fcompany\u002Fcompetition","1.company\u002Fcompetition",{"title":31,"path":32,"stem":33},"Hybrid Working","\u002Fcompany\u002Fhybrid-working","1.company\u002Fhybrid-working",{"title":35,"path":36,"stem":37},"Manchester Office","\u002Fcompany\u002Foffice","1.company\u002Foffice",{"title":39,"path":40,"stem":41},"Operations","\u002Fcompany\u002Foperations","1.company\u002Foperations",{"title":43,"path":44,"stem":45},"Policies","\u002Fcompany\u002Fpolicies","1.company\u002Fpolicies",{"title":47,"path":48,"stem":49},"Problem Statement","\u002Fcompany\u002Fproblem-statement","1.company\u002Fproblem-statement",{"title":51,"path":52,"stem":53},"Product Strategy","\u002Fcompany\u002Fproduct-strategy","1.company\u002Fproduct-strategy",{"title":55,"path":56,"stem":57,"children":58,"page":67},"Products","\u002Fcompany\u002Fproducts","1.company\u002Fproducts",[59,63],{"title":60,"path":61,"stem":62},"Capability Exchange","\u002Fcompany\u002Fproducts\u002Fcapability-exchange","1.company\u002Fproducts\u002Fcapability-exchange",{"title":64,"path":65,"stem":66},"ESProfiler Platform","\u002Fcompany\u002Fproducts\u002Fesprofiler","1.company\u002Fproducts\u002Fesprofiler",false,{"title":69,"path":70,"stem":71},"Security","\u002Fcompany\u002Fsecurity","1.company\u002Fsecurity",{"title":73,"icon":74,"path":75,"stem":76,"children":77,"page":67},"People Ops","i-lucide-users","\u002Fpeople-ops","2.people-ops",[78,82,86,90,94,98],{"title":79,"path":80,"stem":81},"Compensation","\u002Fpeople-ops\u002Fcompensation","2.people-ops\u002Fcompensation",{"title":83,"path":84,"stem":85},"Education","\u002Fpeople-ops\u002Feducation","2.people-ops\u002Feducation",{"title":87,"path":88,"stem":89},"Expenses","\u002Fpeople-ops\u002Fexpenses","2.people-ops\u002Fexpenses",{"title":91,"path":92,"stem":93},"Holiday & Leave","\u002Fpeople-ops\u002Fleave","2.people-ops\u002Fleave",{"title":95,"path":96,"stem":97},"Onboarding","\u002Fpeople-ops\u002Fonboarding","2.people-ops\u002Fonboarding",{"title":99,"path":100,"stem":101},"Recruitment","\u002Fpeople-ops\u002Frecruitment","2.people-ops\u002Frecruitment",{"title":103,"icon":104,"path":105,"stem":106,"children":107,"page":67},"Engineering","i-lucide-rocket","\u002Fengineering","3.engineering",[108,168,189,210,232,240,244],{"title":109,"path":110,"stem":111,"children":112,"page":67},"Contributing","\u002Fengineering\u002Fcontributing","3.engineering\u002Fcontributing",[113,117,121,125,129,133,146,155],{"title":114,"path":115,"stem":116},"Development Setup","\u002Fengineering\u002Fcontributing\u002Fdevelopment-setup","3.engineering\u002Fcontributing\u002F1.development-setup",{"title":118,"path":119,"stem":120},"Engineering Operations","\u002Fengineering\u002Fcontributing\u002Fengineering-operations","3.engineering\u002Fcontributing\u002F2.engineering-operations",{"title":122,"path":123,"stem":124},"Documentation","\u002Fengineering\u002Fcontributing\u002Fdocumentation","3.engineering\u002Fcontributing\u002F3.documentation",{"title":126,"path":127,"stem":128},"Agentic Coding","\u002Fengineering\u002Fcontributing\u002Fagentic-coding","3.engineering\u002Fcontributing\u002F4.agentic-coding",{"title":130,"path":131,"stem":132},"Coding Standards","\u002Fengineering\u002Fcontributing\u002Fcoding-standards","3.engineering\u002Fcontributing\u002F5.coding-standards",{"title":134,"path":135,"stem":136,"children":137,"page":67},"Back End","\u002Fengineering\u002Fcontributing\u002Fback-end","3.engineering\u002Fcontributing\u002Fback-end",[138,142],{"title":139,"path":140,"stem":141},"API Guidelines","\u002Fengineering\u002Fcontributing\u002Fback-end\u002Fapi-guidelines","3.engineering\u002Fcontributing\u002Fback-end\u002Fapi-guidelines",{"title":143,"path":144,"stem":145},"LLM Prompts & Langfuse Integration","\u002Fengineering\u002Fcontributing\u002Fback-end\u002Fllm-prompts","3.engineering\u002Fcontributing\u002Fback-end\u002Fllm-prompts",{"title":147,"path":148,"stem":149,"children":150,"page":67},"Database","\u002Fengineering\u002Fcontributing\u002Fdatabase","3.engineering\u002Fcontributing\u002Fdatabase",[151],{"title":152,"path":153,"stem":154},"Database Volumes","\u002Fengineering\u002Fcontributing\u002Fdatabase\u002Fdatabase-refresh","3.engineering\u002Fcontributing\u002Fdatabase\u002Fdatabase-refresh",{"title":156,"path":157,"stem":158,"children":159,"page":67},"Front End","\u002Fengineering\u002Fcontributing\u002Ffront-end","3.engineering\u002Fcontributing\u002Ffront-end",[160,164],{"title":161,"path":162,"stem":163},"Data Attributes","\u002Fengineering\u002Fcontributing\u002Ffront-end\u002Fdata-attributes","3.engineering\u002Fcontributing\u002Ffront-end\u002Fdata-attributes",{"title":165,"path":166,"stem":167},"Testing","\u002Fengineering\u002Fcontributing\u002Ffront-end\u002Ftesting","3.engineering\u002Fcontributing\u002Ffront-end\u002Ftesting",{"title":169,"path":170,"stem":171,"children":172},"Deployment","\u002Fengineering\u002Fdeployment","3.engineering\u002Fdeployment\u002F0.index",[173,174,177,181,185],{"title":169,"path":170,"stem":171},{"title":60,"path":175,"stem":176},"\u002Fengineering\u002Fdeployment\u002Fcapability-exchange","3.engineering\u002Fdeployment\u002Fcapability-exchange",{"title":178,"path":179,"stem":180},"Langfuse Deployment","\u002Fengineering\u002Fdeployment\u002Fecs-langfuse-deployment","3.engineering\u002Fdeployment\u002Fecs-langfuse-deployment",{"title":182,"path":183,"stem":184},"ESP Platform Configuration","\u002Fengineering\u002Fdeployment\u002Fesp-platform-configuration","3.engineering\u002Fdeployment\u002Fesp-platform-configuration",{"title":186,"path":187,"stem":188},"Platform","\u002Fengineering\u002Fdeployment\u002Fplatform","3.engineering\u002Fdeployment\u002Fplatform",{"title":190,"path":191,"stem":192,"children":193,"page":67},"Github","\u002Fengineering\u002Fgithub","3.engineering\u002Fgithub",[194,198,202,206],{"title":195,"path":196,"stem":197},"Packages","\u002Fengineering\u002Fgithub\u002Fpackages","3.engineering\u002Fgithub\u002Fpackages",{"title":199,"path":200,"stem":201},"Personal Access Token","\u002Fengineering\u002Fgithub\u002Fpersonal-access-token","3.engineering\u002Fgithub\u002Fpersonal-access-token",{"title":203,"path":204,"stem":205},"Troubleshooting","\u002Fengineering\u002Fgithub\u002Ftroubleshooting","3.engineering\u002Fgithub\u002Ftroubleshooting",{"title":207,"path":208,"stem":209},"Workflows","\u002Fengineering\u002Fgithub\u002Fworkflows","3.engineering\u002Fgithub\u002Fworkflows",{"title":211,"path":212,"stem":213,"children":214},"Platform Ops","\u002Fengineering\u002Fplatform-ops","3.engineering\u002Fplatform-ops\u002F0.index",[215,216,220,224,228],{"title":211,"path":212,"stem":213},{"title":217,"path":218,"stem":219},"Production Database","\u002Fengineering\u002Fplatform-ops\u002Fdatabase-connection","3.engineering\u002Fplatform-ops\u002Fdatabase-connection",{"title":221,"path":222,"stem":223},"New Tenant Creation","\u002Fengineering\u002Fplatform-ops\u002Fnew-tenant-creation","3.engineering\u002Fplatform-ops\u002Fnew-tenant-creation",{"title":225,"path":226,"stem":227},"Releases","\u002Fengineering\u002Fplatform-ops\u002Frelease","3.engineering\u002Fplatform-ops\u002Frelease",{"title":229,"path":230,"stem":231},"Versioning","\u002Fengineering\u002Fplatform-ops\u002Fversioning","3.engineering\u002Fplatform-ops\u002Fversioning",{"title":186,"path":233,"stem":234,"children":235,"page":67},"\u002Fengineering\u002Fplatform","3.engineering\u002Fplatform",[236],{"title":237,"path":238,"stem":239},"useAPI","\u002Fengineering\u002Fplatform\u002Fuse-api","3.engineering\u002Fplatform\u002Fuse-api",{"title":241,"path":242,"stem":243},"Project Management","\u002Fengineering\u002Fproject-management","3.engineering\u002Fproject-management",{"title":245,"path":246,"stem":247},"Tools","\u002Fengineering\u002Ftools","3.engineering\u002Ftools",{"title":249,"icon":250,"path":251,"stem":252,"children":253,"page":67},"Design","i-lucide-palette","\u002Fdesign","4.design",[254,275,279,283,287,290],{"title":255,"path":256,"stem":257,"children":258,"page":67},"Personas","\u002Fdesign\u002Fpersonas","4.design\u002F0.personas",[259,263,267,271],{"title":260,"path":261,"stem":262},"CISO - Simon","\u002Fdesign\u002Fpersonas\u002F01-ciso-simon","4.design\u002F0.personas\u002F01-ciso-simon",{"title":264,"path":265,"stem":266},"Head of Cyber - Harry","\u002Fdesign\u002Fpersonas\u002F02-head-of-cyber-harry","4.design\u002F0.personas\u002F02-head-of-cyber-harry",{"title":268,"path":269,"stem":270},"Security Architect - Sasha","\u002Fdesign\u002Fpersonas\u002F03-security-architect-sasha","4.design\u002F0.personas\u002F03-security-architect-sasha",{"title":272,"path":273,"stem":274},"Procurement Officer - Paige","\u002Fdesign\u002Fpersonas\u002F04-procurement-officer-paige","4.design\u002F0.personas\u002F04-procurement-officer-paige",{"title":276,"path":277,"stem":278},"Design Thinking","\u002Fdesign\u002Fdesign-thinking","4.design\u002F1.design-thinking",{"title":280,"path":281,"stem":282},"Design & Development","\u002Fdesign\u002Fdesign-and-development","4.design\u002F3.design-and-development",{"title":284,"path":285,"stem":286},"Branding","\u002Fdesign\u002Fbranding","4.design\u002F4.branding",{"title":245,"path":288,"stem":289},"\u002Fdesign\u002Ftools","4.design\u002F5.tools",{"title":291,"path":292,"stem":293},"Customer Success","\u002Fdesign\u002Fworking-with-customers","4.design\u002F6.working-with-customers",{"title":295,"icon":296,"path":297,"stem":298,"children":299,"page":67},"Sales","i-lucide-dollar-sign","\u002Fsales","4.sales",[300,320,324,328],{"title":301,"path":302,"stem":303,"children":304,"page":67},"Process","\u002Fsales\u002Fprocess","4.sales\u002F0.process",[305,309,313,317],{"title":306,"path":307,"stem":308},"Lead","\u002Fsales\u002Fprocess\u002Flead","4.sales\u002F0.process\u002F0.lead",{"title":310,"path":311,"stem":312},"Deal Qualification","\u002Fsales\u002Fprocess\u002Fqualification","4.sales\u002F0.process\u002F1.qualification",{"title":314,"path":315,"stem":316},"Close","\u002Fsales\u002Fprocess\u002Fclose","4.sales\u002F0.process\u002F2.close",{"title":95,"path":318,"stem":319},"\u002Fsales\u002Fprocess\u002Fonboarding","4.sales\u002F0.process\u002Fonboarding",{"title":321,"path":322,"stem":323},"Meetings","\u002Fsales\u002Fmeetings","4.sales\u002Fmeetings",{"title":325,"path":326,"stem":327},"Metrics","\u002Fsales\u002Fmetrics","4.sales\u002Fmetrics",{"title":245,"path":329,"stem":330},"\u002Fsales\u002Ftools","4.sales\u002Ftools",{"title":332,"icon":333,"path":334,"stem":335,"children":336,"page":67},"Marketing","i-lucide-book-image","\u002Fmarketing","5.marketing",[337,341,345,348],{"title":338,"path":339,"stem":340},"Content","\u002Fmarketing\u002Fcontent","5.marketing\u002Fcontent",{"title":342,"path":343,"stem":344},"Messaging","\u002Fmarketing\u002Fmessaging","5.marketing\u002Fmessaging",{"title":245,"path":346,"stem":347},"\u002Fmarketing\u002Ftools","5.marketing\u002Ftools",{"title":349,"path":350,"stem":351},"Website","\u002Fmarketing\u002Fwebsite","5.marketing\u002Fwebsite",{"title":353,"icon":354,"path":355,"stem":356,"children":357,"page":67},"AI & Data Ops","i-lucide-database","\u002Fdata-ops","6.data-ops",[358,362,370,395,416,420],{"title":359,"path":360,"stem":361},"Account Portal (CAS)","\u002Fdata-ops\u002Faccount-portal","6.data-ops\u002Faccount-portal",{"title":60,"path":363,"stem":364,"children":365,"page":67},"\u002Fdata-ops\u002Fcapability-exchange","6.data-ops\u002Fcapability-exchange",[366],{"title":367,"path":368,"stem":369},"Leaderboard Calculation","\u002Fdata-ops\u002Fcapability-exchange\u002Fleaderboard-calculation","6.data-ops\u002Fcapability-exchange\u002Fleaderboard-calculation",{"title":371,"path":372,"stem":373,"children":374,"page":67},"Data Management","\u002Fdata-ops\u002Fdata-management","6.data-ops\u002Fdata-management",[375,379,383,387,391],{"title":376,"path":377,"stem":378},"Adding Products","\u002Fdata-ops\u002Fdata-management\u002Fadding-products","6.data-ops\u002Fdata-management\u002Fadding-products",{"title":380,"path":381,"stem":382},"Adding Vendors","\u002Fdata-ops\u002Fdata-management\u002Fadding-vendors","6.data-ops\u002Fdata-management\u002Fadding-vendors",{"title":384,"path":385,"stem":386},"Framework Mapping","\u002Fdata-ops\u002Fdata-management\u002Fframework-mapping","6.data-ops\u002Fdata-management\u002Fframework-mapping",{"title":388,"path":389,"stem":390},"Refreshing Vendors","\u002Fdata-ops\u002Fdata-management\u002Frefreshing-vendors","6.data-ops\u002Fdata-management\u002Frefreshing-vendors",{"title":392,"path":393,"stem":394},"Reviewing Draft Vendors","\u002Fdata-ops\u002Fdata-management\u002Freviewing-draft-vendors","6.data-ops\u002Fdata-management\u002Freviewing-draft-vendors",{"title":396,"path":397,"stem":398,"children":399,"page":67},"LLM Ops","\u002Fdata-ops\u002Fllm-ops","6.data-ops\u002Fllm-ops",[400,404,408,412],{"title":401,"path":402,"stem":403},"Agents","\u002Fdata-ops\u002Fllm-ops\u002Fagents","6.data-ops\u002Fllm-ops\u002F1.agents",{"title":405,"path":406,"stem":407},"ESPi Architecture & Query Flow","\u002Fdata-ops\u002Fllm-ops\u002Fespi-architecture","6.data-ops\u002Fllm-ops\u002F2.espi-architecture",{"title":409,"path":410,"stem":411},"Evaluating Agents","\u002Fdata-ops\u002Fllm-ops\u002Fevaluations","6.data-ops\u002Fllm-ops\u002F3.evaluations",{"title":413,"path":414,"stem":415},"Benchmarking ESPi","\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi","6.data-ops\u002Fllm-ops\u002F4.benchmarking-espi",{"title":417,"path":418,"stem":419},"Message Queues","\u002Fdata-ops\u002Fmessage-queues","6.data-ops\u002Fmessage-queues",{"title":421,"path":422,"stem":423,"children":424,"page":67},"Models","\u002Fdata-ops\u002Fmodels","6.data-ops\u002Fmodels",[425,429,433,437,441],{"title":426,"path":427,"stem":428},"Universal Framework Mapper Vision","\u002Fdata-ops\u002Fmodels\u002Funiversal-framework-mapper-vision","6.data-ops\u002Fmodels\u002F1.universal-framework-mapper-vision",{"title":430,"path":431,"stem":432},"Framework Description Model (FDM)","\u002Fdata-ops\u002Fmodels\u002Fframework-description-model","6.data-ops\u002Fmodels\u002F2.framework-description-model",{"title":434,"path":435,"stem":436},"Relationship Definition Model (RDM)","\u002Fdata-ops\u002Fmodels\u002Frelationship-definition-model","6.data-ops\u002Fmodels\u002F3.relationship-definition-model",{"title":438,"path":439,"stem":440},"Relationship Mapping Model (RMM)","\u002Fdata-ops\u002Fmodels\u002Frelationship-mapping-model","6.data-ops\u002Fmodels\u002F4.relationship-mapping-model",{"title":442,"path":443,"stem":444},"Platform Framework Matrix V2","\u002Fdata-ops\u002Fmodels\u002Fplatform-framework-matrix-v2","6.data-ops\u002Fmodels\u002F5.platform-framework-matrix-v2",{"title":446,"path":447,"stem":448},"Glossary","\u002Fglossary","glossary",{"id":450,"title":409,"body":451,"description":4483,"extension":4484,"links":4485,"meta":4486,"navigation":2509,"path":410,"seo":4487,"stem":411,"__hash__":4488},"docs\u002F6.data-ops\u002Fllm-ops\u002F3.evaluations.md",{"type":452,"value":453,"toc":4460},"minimark",[454,466,469,513,563,566,593,597,600,605,608,653,658,701,704,708,718,759,768,771,775,800,804,2722,2724,2728,2731,2738,2760,2763,2777,2781,2824,2831,2834,2857,2861,2867,2876,2880,2929,2937,2941,3432,3434,3438,3452,3461,3475,3479,3548,3554,3564,3568,3571,3611,3615,3691,3694,3698,3704,3707,3719,3734,3738,3741,3754,3757,3761,4408,4410,4414,4456],[455,456,457,458,465],"p",{},"We use ",[459,460,464],"a",{"href":461,"rel":462},"https:\u002F\u002Fesplf.esprofiler.com\u002Fproject\u002Fesp-development",[463],"nofollow","Langfuse"," to evaluate LLM agents in the ESProfiler ecosystem.",[455,467,468],{},"Evaluations follow three steps:",[470,471,472,495,504],"ol",{},[473,474,475,482,483,487,488,487,491,494],"li",{},[476,477,478],"strong",{},[459,479,481],{"href":480},"#1-creating-datasets","Create datasets"," — test cases (",[484,485,486],"code",{},"input"," \u002F ",[484,489,490],{},"expectedOutput",[484,492,493],{},"metadata",")",[473,496,497,503],{},[476,498,499],{},[459,500,502],{"href":501},"#2-creating-evaluators","Create evaluators"," — scoring definitions (mostly LLM-as-judge)",[473,505,506,512],{},[476,507,508],{},[459,509,511],{"href":510},"#3-running-experiments","Run experiments"," — prompt + dataset + evaluators",[514,515,516,529],"table",{},[517,518,519],"thead",{},[520,521,522,526],"tr",{},[523,524,525],"th",{},"Piece",[523,527,528],{},"Role",[530,531,532,543,553],"tbody",{},[520,533,534,540],{},[535,536,537],"td",{},[476,538,539],{},"Dataset",[535,541,542],{},"Reusable test cases",[520,544,545,550],{},[535,546,547],{},[476,548,549],{},"Evaluator",[535,551,552],{},"Scores one quality dimension of an agent output",[520,554,555,560],{},[535,556,557],{},[476,558,559],{},"Experiment",[535,561,562],{},"Runs a prompt against a dataset and applies evaluators",[455,564,565],{},"Existing resources in ESP Development:",[567,568,569,576,583],"ul",{},[473,570,571],{},[459,572,575],{"href":573,"rel":574},"https:\u002F\u002Fesplf.esprofiler.com\u002Fproject\u002Fesp-development\u002Fdatasets?pageIndex=0&pageSize=50",[463],"Datasets",[473,577,578],{},[459,579,582],{"href":580,"rel":581},"https:\u002F\u002Fesplf.esprofiler.com\u002Fproject\u002Fesp-development\u002Fevals\u002Ftemplates",[463],"Evaluator templates",[473,584,585],{},[459,586,589,590],{"href":587,"rel":588},"https:\u002F\u002Fesplf.esprofiler.com\u002Fproject\u002Fesp-development\u002Fdatasets\u002Fcmruuouiv0021mk07r7itcz46\u002Fexperiments",[463],"Example experiments: ",[484,591,592],{},"findings\u002Fgate_test",[594,595,596],"note",{},"A walkthrough video of the full Langfuse UI will be recorded soon and added to this page later.",[598,599],"hr",{},[601,602,604],"h2",{"id":603},"_1-creating-datasets","1. Creating Datasets",[455,606,607],{},"An evaluation dataset is a collection of test cases. Each item typically has:",[514,609,610,620],{},[517,611,612],{},[520,613,614,617],{},[523,615,616],{},"Field",[523,618,619],{},"Purpose",[530,621,622,631,644],{},[520,623,624,628],{},[535,625,626],{},[476,627,486],{},[535,629,630],{},"What the agent\u002Fprompt receives at runtime",[520,632,633,637],{},[535,634,635],{},[476,636,490],{},[535,638,639,640,643],{},"Golden answer ",[476,641,642],{},"or"," evaluation guidance",[520,645,646,650],{},[535,647,648],{},[476,649,493],{},[535,651,652],{},"Filtering, debugging, and audit context (not fed to the agent)",[654,655,657],"h3",{"id":656},"_11-create-your-first-dataset-5-steps","1.1 Create your first dataset (5 steps)",[470,659,660,669,675,689,695],{},[473,661,662,665,666,668],{},[476,663,664],{},"Pick one agent"," — see ",[459,667,401],{"href":402},".",[473,670,671,674],{},[476,672,673],{},"Decide expected-output style"," — golden answer for deterministic tasks; judge guidance for open-ended ones.",[473,676,677,680,681,683,684,686,687,668],{},[476,678,679],{},"Write 3–5 items"," — ",[484,682,486],{}," keys must match that agent’s prompt variables; add ",[484,685,490],{}," and optional ",[484,688,493],{},[473,690,691,694],{},[476,692,693],{},"Anonymize if data came from production"," — never upload raw tenant\u002FPII.",[473,696,697,700],{},[476,698,699],{},"Create and upload"," — UI for tiny flat cases; Python SDK for nested JSON.",[455,702,703],{},"Then spot-check 2–3 items in Langfuse before attaching evaluators.",[654,705,707],{"id":706},"_12-dataset-naming","1.2 Dataset naming",[709,710,716],"pre",{"className":711,"code":713,"language":714,"meta":715},[712],"language-text","{agentName}\u002F{datasetRole}\n","text","",[484,717,713],{"__ignoreMap":715},[514,719,720,731],{},[517,721,722],{},[520,723,724,726,728],{},[523,725,539],{},[523,727,528],{},[523,729,730],{},"Size guidance",[530,732,733,746],{},[520,734,735,740,743],{},[535,736,737],{},[484,738,739],{},"{agent}\u002Fsmoke_test",[535,741,742],{},"Quick sanity checks after prompt or wiring changes",[535,744,745],{},"~5–15 items",[520,747,748,753,756],{},[535,749,750],{},[484,751,752],{},"{agent}\u002Fgate_test",[535,754,755],{},"Broader frozen set for prompt comparison and release decisions",[535,757,758],{},"~20–50 items",[455,760,761,762,765,766,668],{},"Examples: ",[484,763,764],{},"conversation-namer\u002Fsmoke_test",", ",[484,767,592],{},[455,769,770],{},"Keep both sets reviewed and anonymized. Update them deliberately — do not treat one as a staging queue for the other.",[654,772,774],{"id":773},"_13-upload-overview","1.3 Upload overview",[567,776,777,783,789],{},[473,778,779,782],{},[476,780,781],{},"UI"," — Datasets → New dataset → add items (or CSV for flat strings). Best for small \u002F simple cases.",[473,784,785,788],{},[476,786,787],{},"Python SDK"," — preferred for nested JSON (Findings transcripts, structured guidance). Keys: Langfuse UI → Settings → API Keys.",[473,790,791,794,795],{},[476,792,793],{},"Reference:"," ",[459,796,799],{"href":797,"rel":798},"https:\u002F\u002Flangfuse.com\u002Fdocs\u002Fevaluation\u002Fexperiments\u002Fdatasets",[463],"Langfuse Datasets",[654,801,803],{"id":802},"_14-dataset-reference-expand-as-needed","1.4 Dataset reference (expand as needed)",[805,806,807,877,930,962,1150,1352,2443,2653],"accordion",{},[808,809,812,816,823,838,842,848,857,867,870],"accordion-item",{"icon":810,"label":811},"i-lucide-table","1.4.1 Field details (input \u002F expectedOutput \u002F metadata)",[813,814,815],"h4",{"id":486},"Input",[455,817,818,819,822],{},"What is injected into the prompt or application under test: a user message, nested ",[476,820,821],{},"prompt variables",", or a multi-turn conversation.",[455,824,825,794,828,830,831,834,835,668],{},[476,826,827],{},"Rule:",[484,829,486],{}," keys must match the prompt variables of the agent you are evaluating. Find those in the agent’s ",[484,832,833],{},".st"," prompt \u002F config in ",[484,836,837],{},"platform-api",[813,839,841],{"id":840},"expected-output","Expected output",[455,843,844,847],{},[476,845,846],{},"A) Golden answer (deterministic)"," — use when the correct answer is known and comparable (tags, short titles, exact fields). A code check is often enough. Curate with human review.",[455,849,850,853,854,856],{},[476,851,852],{},"B) Judge guidance (non-deterministic)"," — use when many good outputs exist (summaries, reports, findings). Put criteria in ",[484,855,490],{}," (what must be covered, constraints, grounding rules) — not a full golden dump.",[594,858,859,860,862,863,866],{},"For Findings-style agents: keep guidance in ",[484,861,490],{},"; put full historical report\u002Ffindings in ",[484,864,865],{},"metadata.reference_output"," for human review only.",[813,868,869],{"id":493},"Metadata",[455,871,872,873,876],{},"Anything that should ",[476,874,875],{},"not"," be fed as prompt input: tags, tool expectations, anonymization flags, debug goldens.",[808,878,881],{"icon":879,"label":880},"i-lucide-git-compare","1.4.2 Choose evaluation style",[514,882,883,895],{},[517,884,885],{},[520,886,887,890,893],{},[523,888,889],{},"Use case",[523,891,892],{},"Expected output style",[523,894,549],{},[530,896,897,908,919],{},[520,898,899,902,905],{},[535,900,901],{},"Categorization \u002F tagging \u002F short titles",[535,903,904],{},"Golden labels",[535,906,907],{},"Code check (app or unit tests)",[520,909,910,913,916],{},[535,911,912],{},"Tone, grounding, alignment, finding types",[535,914,915],{},"Guidance + rubric",[535,917,918],{},"LLM-as-judge",[520,920,921,924,927],{},[535,922,923],{},"Mixed",[535,925,926],{},"Guidance + optional reference in metadata",[535,928,929],{},"Code + LLM-as-judge",[808,931,934,937],{"icon":932,"label":933},"i-lucide-list-filter","1.4.3 How to select records",[455,935,936],{},"Start from production failure modes, not random sampling.",[470,938,939,942,945,948,951],{},[473,940,941],{},"Observe where the agent fails (wrong types, invented claims, missed coverage, etc.)",[473,943,944],{},"Pull representative completed cases for those modes",[473,946,947],{},"Prefer diverse scenarios over near-duplicates",[473,949,950],{},"Anonymize before upload",[473,952,953,954,957,958,961],{},"Put stable regression cases in ",[484,955,956],{},"gate_test","; keep a smaller ",[484,959,960],{},"smoke_test"," set for quick checks",[808,963,966,971,1045,1048],{"icon":964,"label":965},"i-lucide-shield","1.4.4 Data anonymization (mandatory for production data)",[455,967,968],{},[476,969,970],{},"Do not upload raw tenant data to Langfuse.",[514,972,973,983],{},[517,974,975],{},[520,976,977,980],{},[523,978,979],{},"Original",[523,981,982],{},"Anonymized form",[530,984,985,996,1004,1015,1023,1034],{},[520,986,987,990],{},[535,988,989],{},"Tenant \u002F org name",[535,991,992,993,494],{},"Synthetic org (e.g. ",[476,994,995],{},"Black Mesa",[520,997,998,1001],{},[535,999,1000],{},"Real people",[535,1002,1003],{},"Stable synthetic names",[520,1005,1006,1009],{},[535,1007,1008],{},"Emails \u002F tenant domains",[535,1010,1011,1014],{},[484,1012,1013],{},"@blackmesa.example"," placeholders",[520,1016,1017,1020],{},[535,1018,1019],{},"Real task \u002F product \u002F vendor UUIDs",[535,1021,1022],{},"Synthetic UUIDs from hashes",[520,1024,1025,1028],{},[535,1026,1027],{},"Original task id",[535,1029,1030,1033],{},[484,1031,1032],{},"original_task_id_hash"," only",[520,1035,1036,1039],{},[535,1037,1038],{},"Images \u002F screenshots",[535,1040,1041,1042,494],{},"Stripped (",[484,1043,1044],{},"imageStr: null",[455,1046,1047],{},"Replace tenant strings everywhere, keep people mapping stable across items, keep market product names only when they are not the customer identity, then sanity-check that known customer tokens are gone.",[709,1049,1053],{"className":1050,"code":1051,"language":1052,"meta":715,"style":715},"language-json shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","{\n  \"anonymization\": {\n    \"tenant_replaced_with\": \"Black Mesa\",\n    \"people_synthetic\": true,\n    \"images_stripped\": true\n  }\n}\n","json",[484,1054,1055,1064,1083,1108,1123,1138,1144],{"__ignoreMap":715},[1056,1057,1060],"span",{"class":1058,"line":1059},"line",1,[1056,1061,1063],{"class":1062},"sMK4o","{\n",[1056,1065,1067,1070,1074,1077,1080],{"class":1058,"line":1066},2,[1056,1068,1069],{"class":1062},"  \"",[1056,1071,1073],{"class":1072},"spNyl","anonymization",[1056,1075,1076],{"class":1062},"\"",[1056,1078,1079],{"class":1062},":",[1056,1081,1082],{"class":1062}," {\n",[1056,1084,1086,1089,1093,1095,1097,1100,1103,1105],{"class":1058,"line":1085},3,[1056,1087,1088],{"class":1062},"    \"",[1056,1090,1092],{"class":1091},"sBMFI","tenant_replaced_with",[1056,1094,1076],{"class":1062},[1056,1096,1079],{"class":1062},[1056,1098,1099],{"class":1062}," \"",[1056,1101,995],{"class":1102},"sfazB",[1056,1104,1076],{"class":1062},[1056,1106,1107],{"class":1062},",\n",[1056,1109,1111,1113,1116,1118,1120],{"class":1058,"line":1110},4,[1056,1112,1088],{"class":1062},[1056,1114,1115],{"class":1091},"people_synthetic",[1056,1117,1076],{"class":1062},[1056,1119,1079],{"class":1062},[1056,1121,1122],{"class":1062}," true,\n",[1056,1124,1126,1128,1131,1133,1135],{"class":1058,"line":1125},5,[1056,1127,1088],{"class":1062},[1056,1129,1130],{"class":1091},"images_stripped",[1056,1132,1076],{"class":1062},[1056,1134,1079],{"class":1062},[1056,1136,1137],{"class":1062}," true\n",[1056,1139,1141],{"class":1058,"line":1140},6,[1056,1142,1143],{"class":1062},"  }\n",[1056,1145,1147],{"class":1058,"line":1146},7,[1056,1148,1149],{"class":1062},"}\n",[808,1151,1154,1164],{"icon":1152,"label":1153},"i-lucide-file-json","1.4.5 Example: Conversation Namer item",[455,1155,1156,1157,1159,1160,1163],{},"First user message in → short title out. Confirm the real prompt variable name in ",[484,1158,837],{}," before uploading (replace ",[484,1161,1162],{},"message"," if it differs). Synthetic cases need no anonymization.",[709,1165,1167],{"className":1050,"code":1166,"language":1052,"meta":715,"style":715},"{\n  \"input\": {\n    \"message\": \"Can you compare our CrowdStrike and Wiz renewals for next quarter?\"\n  },\n  \"expectedOutput\": {\n    \"title\": \"CrowdStrike vs Wiz renewals\"\n  },\n  \"metadata\": {\n    \"dataset\": \"conversation-namer\u002Fgate_test\",\n    \"tags\": [\"conversation_namer\", \"synthetic\"],\n    \"constraints\": { \"max_words\": 5 }\n  }\n}\n",[484,1168,1169,1173,1185,1203,1208,1220,1238,1242,1255,1276,1311,1342,1347],{"__ignoreMap":715},[1056,1170,1171],{"class":1058,"line":1059},[1056,1172,1063],{"class":1062},[1056,1174,1175,1177,1179,1181,1183],{"class":1058,"line":1066},[1056,1176,1069],{"class":1062},[1056,1178,486],{"class":1072},[1056,1180,1076],{"class":1062},[1056,1182,1079],{"class":1062},[1056,1184,1082],{"class":1062},[1056,1186,1187,1189,1191,1193,1195,1197,1200],{"class":1058,"line":1085},[1056,1188,1088],{"class":1062},[1056,1190,1162],{"class":1091},[1056,1192,1076],{"class":1062},[1056,1194,1079],{"class":1062},[1056,1196,1099],{"class":1062},[1056,1198,1199],{"class":1102},"Can you compare our CrowdStrike and Wiz renewals for next quarter?",[1056,1201,1202],{"class":1062},"\"\n",[1056,1204,1205],{"class":1058,"line":1110},[1056,1206,1207],{"class":1062},"  },\n",[1056,1209,1210,1212,1214,1216,1218],{"class":1058,"line":1125},[1056,1211,1069],{"class":1062},[1056,1213,490],{"class":1072},[1056,1215,1076],{"class":1062},[1056,1217,1079],{"class":1062},[1056,1219,1082],{"class":1062},[1056,1221,1222,1224,1227,1229,1231,1233,1236],{"class":1058,"line":1140},[1056,1223,1088],{"class":1062},[1056,1225,1226],{"class":1091},"title",[1056,1228,1076],{"class":1062},[1056,1230,1079],{"class":1062},[1056,1232,1099],{"class":1062},[1056,1234,1235],{"class":1102},"CrowdStrike vs Wiz renewals",[1056,1237,1202],{"class":1062},[1056,1239,1240],{"class":1058,"line":1146},[1056,1241,1207],{"class":1062},[1056,1243,1245,1247,1249,1251,1253],{"class":1058,"line":1244},8,[1056,1246,1069],{"class":1062},[1056,1248,493],{"class":1072},[1056,1250,1076],{"class":1062},[1056,1252,1079],{"class":1062},[1056,1254,1082],{"class":1062},[1056,1256,1258,1260,1263,1265,1267,1269,1272,1274],{"class":1058,"line":1257},9,[1056,1259,1088],{"class":1062},[1056,1261,1262],{"class":1091},"dataset",[1056,1264,1076],{"class":1062},[1056,1266,1079],{"class":1062},[1056,1268,1099],{"class":1062},[1056,1270,1271],{"class":1102},"conversation-namer\u002Fgate_test",[1056,1273,1076],{"class":1062},[1056,1275,1107],{"class":1062},[1056,1277,1279,1281,1284,1286,1288,1291,1293,1296,1298,1301,1303,1306,1308],{"class":1058,"line":1278},10,[1056,1280,1088],{"class":1062},[1056,1282,1283],{"class":1091},"tags",[1056,1285,1076],{"class":1062},[1056,1287,1079],{"class":1062},[1056,1289,1290],{"class":1062}," [",[1056,1292,1076],{"class":1062},[1056,1294,1295],{"class":1102},"conversation_namer",[1056,1297,1076],{"class":1062},[1056,1299,1300],{"class":1062},",",[1056,1302,1099],{"class":1062},[1056,1304,1305],{"class":1102},"synthetic",[1056,1307,1076],{"class":1062},[1056,1309,1310],{"class":1062},"],\n",[1056,1312,1314,1316,1319,1321,1323,1326,1328,1332,1334,1336,1339],{"class":1058,"line":1313},11,[1056,1315,1088],{"class":1062},[1056,1317,1318],{"class":1091},"constraints",[1056,1320,1076],{"class":1062},[1056,1322,1079],{"class":1062},[1056,1324,1325],{"class":1062}," {",[1056,1327,1099],{"class":1062},[1056,1329,1331],{"class":1330},"sbssI","max_words",[1056,1333,1076],{"class":1062},[1056,1335,1079],{"class":1062},[1056,1337,1338],{"class":1330}," 5",[1056,1340,1341],{"class":1062}," }\n",[1056,1343,1345],{"class":1058,"line":1344},12,[1056,1346,1143],{"class":1062},[1056,1348,1350],{"class":1058,"line":1349},13,[1056,1351,1149],{"class":1062},[808,1353,1355,1358,1377,1380,1387,1845,1852,2179,2186],{"icon":1152,"label":1354},"1.4.6 Example: Findings Agent shapes",[455,1356,1357],{},"Turns interview transcript + directive into summary, report, and structured findings.",[455,1359,1360,1363,1364,1366,1367,1370,1371,1373,1374,1376],{},[476,1361,1362],{},"Process:"," pull production cases → map prompt variables to ",[484,1365,486],{}," → anonymize → put ",[476,1368,1369],{},"judge guidance"," in ",[484,1372,490],{}," → put historical summary\u002Freport\u002Ffindings in ",[484,1375,865],{}," → upload via SDK.",[594,1378,1379],{},"Internal pull\u002Fbuild scripts live outside this handbook. Use the shapes below as the contract; ask Data Ops \u002F LLM Ops for the current scripts if you need a production refresh.",[455,1381,1382],{},[476,1383,1384,1386],{},[484,1385,486],{}," (prompt variables only)",[709,1388,1390],{"className":1050,"code":1389,"language":1052,"meta":715,"style":715},"{\n  \"user_name\": \"Marcus Silva\",\n  \"user_role\": \"Team Lead and L3 Engineer\",\n  \"organisation_context\": \"Black Mesa is a diversified technology and research organization...\",\n  \"source_type\": \"PROD\",\n  \"source_info\": {\n    \"id\": \"02e4e876-4c33-b27b-9c8e-82cd402f0f85\",\n    \"name\": \"Akamai App & API Protector\",\n    \"type\": \"PROD\",\n    \"vendor\": { \"id\": \"...\", \"name\": \"Akamai\" }\n  },\n  \"task_directive\": \"You are conducting a structured interview to gather information about a product...\",\n  \"interview_transcript\": [\n    {\n      \"id\": \"rufsaa\",\n      \"role\": \"assistant\",\n      \"text\": \"Hi Marcus! I'm ESPi...\",\n      \"isThoughts\": false,\n      \"imageStr\": null\n    },\n    {\n      \"id\": \"oywj8t\",\n      \"role\": \"user\",\n      \"text\": \"Yes\",\n      \"isThoughts\": false,\n      \"imageStr\": null\n    }\n  ]\n}\n",[484,1391,1392,1396,1416,1436,1456,1476,1489,1509,1529,1548,1595,1599,1619,1633,1639,1660,1681,1701,1716,1731,1737,1742,1762,1782,1802,1815,1828,1834,1840],{"__ignoreMap":715},[1056,1393,1394],{"class":1058,"line":1059},[1056,1395,1063],{"class":1062},[1056,1397,1398,1400,1403,1405,1407,1409,1412,1414],{"class":1058,"line":1066},[1056,1399,1069],{"class":1062},[1056,1401,1402],{"class":1072},"user_name",[1056,1404,1076],{"class":1062},[1056,1406,1079],{"class":1062},[1056,1408,1099],{"class":1062},[1056,1410,1411],{"class":1102},"Marcus Silva",[1056,1413,1076],{"class":1062},[1056,1415,1107],{"class":1062},[1056,1417,1418,1420,1423,1425,1427,1429,1432,1434],{"class":1058,"line":1085},[1056,1419,1069],{"class":1062},[1056,1421,1422],{"class":1072},"user_role",[1056,1424,1076],{"class":1062},[1056,1426,1079],{"class":1062},[1056,1428,1099],{"class":1062},[1056,1430,1431],{"class":1102},"Team Lead and L3 Engineer",[1056,1433,1076],{"class":1062},[1056,1435,1107],{"class":1062},[1056,1437,1438,1440,1443,1445,1447,1449,1452,1454],{"class":1058,"line":1110},[1056,1439,1069],{"class":1062},[1056,1441,1442],{"class":1072},"organisation_context",[1056,1444,1076],{"class":1062},[1056,1446,1079],{"class":1062},[1056,1448,1099],{"class":1062},[1056,1450,1451],{"class":1102},"Black Mesa is a diversified technology and research organization...",[1056,1453,1076],{"class":1062},[1056,1455,1107],{"class":1062},[1056,1457,1458,1460,1463,1465,1467,1469,1472,1474],{"class":1058,"line":1125},[1056,1459,1069],{"class":1062},[1056,1461,1462],{"class":1072},"source_type",[1056,1464,1076],{"class":1062},[1056,1466,1079],{"class":1062},[1056,1468,1099],{"class":1062},[1056,1470,1471],{"class":1102},"PROD",[1056,1473,1076],{"class":1062},[1056,1475,1107],{"class":1062},[1056,1477,1478,1480,1483,1485,1487],{"class":1058,"line":1140},[1056,1479,1069],{"class":1062},[1056,1481,1482],{"class":1072},"source_info",[1056,1484,1076],{"class":1062},[1056,1486,1079],{"class":1062},[1056,1488,1082],{"class":1062},[1056,1490,1491,1493,1496,1498,1500,1502,1505,1507],{"class":1058,"line":1146},[1056,1492,1088],{"class":1062},[1056,1494,1495],{"class":1091},"id",[1056,1497,1076],{"class":1062},[1056,1499,1079],{"class":1062},[1056,1501,1099],{"class":1062},[1056,1503,1504],{"class":1102},"02e4e876-4c33-b27b-9c8e-82cd402f0f85",[1056,1506,1076],{"class":1062},[1056,1508,1107],{"class":1062},[1056,1510,1511,1513,1516,1518,1520,1522,1525,1527],{"class":1058,"line":1244},[1056,1512,1088],{"class":1062},[1056,1514,1515],{"class":1091},"name",[1056,1517,1076],{"class":1062},[1056,1519,1079],{"class":1062},[1056,1521,1099],{"class":1062},[1056,1523,1524],{"class":1102},"Akamai App & API Protector",[1056,1526,1076],{"class":1062},[1056,1528,1107],{"class":1062},[1056,1530,1531,1533,1536,1538,1540,1542,1544,1546],{"class":1058,"line":1257},[1056,1532,1088],{"class":1062},[1056,1534,1535],{"class":1091},"type",[1056,1537,1076],{"class":1062},[1056,1539,1079],{"class":1062},[1056,1541,1099],{"class":1062},[1056,1543,1471],{"class":1102},[1056,1545,1076],{"class":1062},[1056,1547,1107],{"class":1062},[1056,1549,1550,1552,1555,1557,1559,1561,1563,1565,1567,1569,1571,1574,1576,1578,1580,1582,1584,1586,1588,1591,1593],{"class":1058,"line":1278},[1056,1551,1088],{"class":1062},[1056,1553,1554],{"class":1091},"vendor",[1056,1556,1076],{"class":1062},[1056,1558,1079],{"class":1062},[1056,1560,1325],{"class":1062},[1056,1562,1099],{"class":1062},[1056,1564,1495],{"class":1330},[1056,1566,1076],{"class":1062},[1056,1568,1079],{"class":1062},[1056,1570,1099],{"class":1062},[1056,1572,1573],{"class":1102},"...",[1056,1575,1076],{"class":1062},[1056,1577,1300],{"class":1062},[1056,1579,1099],{"class":1062},[1056,1581,1515],{"class":1330},[1056,1583,1076],{"class":1062},[1056,1585,1079],{"class":1062},[1056,1587,1099],{"class":1062},[1056,1589,1590],{"class":1102},"Akamai",[1056,1592,1076],{"class":1062},[1056,1594,1341],{"class":1062},[1056,1596,1597],{"class":1058,"line":1313},[1056,1598,1207],{"class":1062},[1056,1600,1601,1603,1606,1608,1610,1612,1615,1617],{"class":1058,"line":1344},[1056,1602,1069],{"class":1062},[1056,1604,1605],{"class":1072},"task_directive",[1056,1607,1076],{"class":1062},[1056,1609,1079],{"class":1062},[1056,1611,1099],{"class":1062},[1056,1613,1614],{"class":1102},"You are conducting a structured interview to gather information about a product...",[1056,1616,1076],{"class":1062},[1056,1618,1107],{"class":1062},[1056,1620,1621,1623,1626,1628,1630],{"class":1058,"line":1349},[1056,1622,1069],{"class":1062},[1056,1624,1625],{"class":1072},"interview_transcript",[1056,1627,1076],{"class":1062},[1056,1629,1079],{"class":1062},[1056,1631,1632],{"class":1062}," [\n",[1056,1634,1636],{"class":1058,"line":1635},14,[1056,1637,1638],{"class":1062},"    {\n",[1056,1640,1642,1645,1647,1649,1651,1653,1656,1658],{"class":1058,"line":1641},15,[1056,1643,1644],{"class":1062},"      \"",[1056,1646,1495],{"class":1091},[1056,1648,1076],{"class":1062},[1056,1650,1079],{"class":1062},[1056,1652,1099],{"class":1062},[1056,1654,1655],{"class":1102},"rufsaa",[1056,1657,1076],{"class":1062},[1056,1659,1107],{"class":1062},[1056,1661,1663,1665,1668,1670,1672,1674,1677,1679],{"class":1058,"line":1662},16,[1056,1664,1644],{"class":1062},[1056,1666,1667],{"class":1091},"role",[1056,1669,1076],{"class":1062},[1056,1671,1079],{"class":1062},[1056,1673,1099],{"class":1062},[1056,1675,1676],{"class":1102},"assistant",[1056,1678,1076],{"class":1062},[1056,1680,1107],{"class":1062},[1056,1682,1684,1686,1688,1690,1692,1694,1697,1699],{"class":1058,"line":1683},17,[1056,1685,1644],{"class":1062},[1056,1687,714],{"class":1091},[1056,1689,1076],{"class":1062},[1056,1691,1079],{"class":1062},[1056,1693,1099],{"class":1062},[1056,1695,1696],{"class":1102},"Hi Marcus! I'm ESPi...",[1056,1698,1076],{"class":1062},[1056,1700,1107],{"class":1062},[1056,1702,1704,1706,1709,1711,1713],{"class":1058,"line":1703},18,[1056,1705,1644],{"class":1062},[1056,1707,1708],{"class":1091},"isThoughts",[1056,1710,1076],{"class":1062},[1056,1712,1079],{"class":1062},[1056,1714,1715],{"class":1062}," false,\n",[1056,1717,1719,1721,1724,1726,1728],{"class":1058,"line":1718},19,[1056,1720,1644],{"class":1062},[1056,1722,1723],{"class":1091},"imageStr",[1056,1725,1076],{"class":1062},[1056,1727,1079],{"class":1062},[1056,1729,1730],{"class":1062}," null\n",[1056,1732,1734],{"class":1058,"line":1733},20,[1056,1735,1736],{"class":1062},"    },\n",[1056,1738,1740],{"class":1058,"line":1739},21,[1056,1741,1638],{"class":1062},[1056,1743,1745,1747,1749,1751,1753,1755,1758,1760],{"class":1058,"line":1744},22,[1056,1746,1644],{"class":1062},[1056,1748,1495],{"class":1091},[1056,1750,1076],{"class":1062},[1056,1752,1079],{"class":1062},[1056,1754,1099],{"class":1062},[1056,1756,1757],{"class":1102},"oywj8t",[1056,1759,1076],{"class":1062},[1056,1761,1107],{"class":1062},[1056,1763,1765,1767,1769,1771,1773,1775,1778,1780],{"class":1058,"line":1764},23,[1056,1766,1644],{"class":1062},[1056,1768,1667],{"class":1091},[1056,1770,1076],{"class":1062},[1056,1772,1079],{"class":1062},[1056,1774,1099],{"class":1062},[1056,1776,1777],{"class":1102},"user",[1056,1779,1076],{"class":1062},[1056,1781,1107],{"class":1062},[1056,1783,1785,1787,1789,1791,1793,1795,1798,1800],{"class":1058,"line":1784},24,[1056,1786,1644],{"class":1062},[1056,1788,714],{"class":1091},[1056,1790,1076],{"class":1062},[1056,1792,1079],{"class":1062},[1056,1794,1099],{"class":1062},[1056,1796,1797],{"class":1102},"Yes",[1056,1799,1076],{"class":1062},[1056,1801,1107],{"class":1062},[1056,1803,1805,1807,1809,1811,1813],{"class":1058,"line":1804},25,[1056,1806,1644],{"class":1062},[1056,1808,1708],{"class":1091},[1056,1810,1076],{"class":1062},[1056,1812,1079],{"class":1062},[1056,1814,1715],{"class":1062},[1056,1816,1818,1820,1822,1824,1826],{"class":1058,"line":1817},26,[1056,1819,1644],{"class":1062},[1056,1821,1723],{"class":1091},[1056,1823,1076],{"class":1062},[1056,1825,1079],{"class":1062},[1056,1827,1730],{"class":1062},[1056,1829,1831],{"class":1058,"line":1830},27,[1056,1832,1833],{"class":1062},"    }\n",[1056,1835,1837],{"class":1058,"line":1836},28,[1056,1838,1839],{"class":1062},"  ]\n",[1056,1841,1843],{"class":1058,"line":1842},29,[1056,1844,1149],{"class":1062},[455,1846,1847],{},[476,1848,1849,1851],{},[484,1850,490],{}," (guidance)",[709,1853,1855],{"className":1050,"code":1854,"language":1052,"meta":715,"style":715},"{\n  \"expected_summary_focus\": \"Summarize strategic state, material risks\u002Finsights, and decision-relevant takeaways. Surface only transcript-grounded points.\",\n  \"expected_min_findings\": 2,\n  \"expected_max_findings\": 11,\n  \"expected_required_finding_types\": [\"insight\", \"risk\", \"sentiment\"],\n  \"expected_report_sections\": [\n    \"Overview\",\n    \"Strategic State\",\n    \"Capabilities and Usage\",\n    \"Coverage and Controls\",\n    \"Sentiment and Operational Experience\",\n    \"Risks, Gaps and Opportunities\",\n    \"Stakeholders\",\n    \"Conclusion\"\n  ],\n  \"must_align_to_directive\": true,\n  \"must_be_grounded_in_transcript\": true,\n  \"schema_notes\": {\n    \"RISK_requires\": [\"severity\"],\n    \"INSIGHT_requires\": [\"category\"],\n    \"SENTIMENT_requires\": [\"category\", \"rating\"]\n  }\n}\n",[484,1856,1857,1861,1881,1897,1913,1953,1966,1977,1988,1999,2010,2021,2032,2043,2052,2057,2070,2083,2096,2118,2140,2171,2175],{"__ignoreMap":715},[1056,1858,1859],{"class":1058,"line":1059},[1056,1860,1063],{"class":1062},[1056,1862,1863,1865,1868,1870,1872,1874,1877,1879],{"class":1058,"line":1066},[1056,1864,1069],{"class":1062},[1056,1866,1867],{"class":1072},"expected_summary_focus",[1056,1869,1076],{"class":1062},[1056,1871,1079],{"class":1062},[1056,1873,1099],{"class":1062},[1056,1875,1876],{"class":1102},"Summarize strategic state, material risks\u002Finsights, and decision-relevant takeaways. Surface only transcript-grounded points.",[1056,1878,1076],{"class":1062},[1056,1880,1107],{"class":1062},[1056,1882,1883,1885,1888,1890,1892,1895],{"class":1058,"line":1085},[1056,1884,1069],{"class":1062},[1056,1886,1887],{"class":1072},"expected_min_findings",[1056,1889,1076],{"class":1062},[1056,1891,1079],{"class":1062},[1056,1893,1894],{"class":1330}," 2",[1056,1896,1107],{"class":1062},[1056,1898,1899,1901,1904,1906,1908,1911],{"class":1058,"line":1110},[1056,1900,1069],{"class":1062},[1056,1902,1903],{"class":1072},"expected_max_findings",[1056,1905,1076],{"class":1062},[1056,1907,1079],{"class":1062},[1056,1909,1910],{"class":1330}," 11",[1056,1912,1107],{"class":1062},[1056,1914,1915,1917,1920,1922,1924,1926,1928,1931,1933,1935,1937,1940,1942,1944,1946,1949,1951],{"class":1058,"line":1125},[1056,1916,1069],{"class":1062},[1056,1918,1919],{"class":1072},"expected_required_finding_types",[1056,1921,1076],{"class":1062},[1056,1923,1079],{"class":1062},[1056,1925,1290],{"class":1062},[1056,1927,1076],{"class":1062},[1056,1929,1930],{"class":1102},"insight",[1056,1932,1076],{"class":1062},[1056,1934,1300],{"class":1062},[1056,1936,1099],{"class":1062},[1056,1938,1939],{"class":1102},"risk",[1056,1941,1076],{"class":1062},[1056,1943,1300],{"class":1062},[1056,1945,1099],{"class":1062},[1056,1947,1948],{"class":1102},"sentiment",[1056,1950,1076],{"class":1062},[1056,1952,1310],{"class":1062},[1056,1954,1955,1957,1960,1962,1964],{"class":1058,"line":1140},[1056,1956,1069],{"class":1062},[1056,1958,1959],{"class":1072},"expected_report_sections",[1056,1961,1076],{"class":1062},[1056,1963,1079],{"class":1062},[1056,1965,1632],{"class":1062},[1056,1967,1968,1970,1973,1975],{"class":1058,"line":1146},[1056,1969,1088],{"class":1062},[1056,1971,1972],{"class":1102},"Overview",[1056,1974,1076],{"class":1062},[1056,1976,1107],{"class":1062},[1056,1978,1979,1981,1984,1986],{"class":1058,"line":1244},[1056,1980,1088],{"class":1062},[1056,1982,1983],{"class":1102},"Strategic State",[1056,1985,1076],{"class":1062},[1056,1987,1107],{"class":1062},[1056,1989,1990,1992,1995,1997],{"class":1058,"line":1257},[1056,1991,1088],{"class":1062},[1056,1993,1994],{"class":1102},"Capabilities and Usage",[1056,1996,1076],{"class":1062},[1056,1998,1107],{"class":1062},[1056,2000,2001,2003,2006,2008],{"class":1058,"line":1278},[1056,2002,1088],{"class":1062},[1056,2004,2005],{"class":1102},"Coverage and Controls",[1056,2007,1076],{"class":1062},[1056,2009,1107],{"class":1062},[1056,2011,2012,2014,2017,2019],{"class":1058,"line":1313},[1056,2013,1088],{"class":1062},[1056,2015,2016],{"class":1102},"Sentiment and Operational Experience",[1056,2018,1076],{"class":1062},[1056,2020,1107],{"class":1062},[1056,2022,2023,2025,2028,2030],{"class":1058,"line":1344},[1056,2024,1088],{"class":1062},[1056,2026,2027],{"class":1102},"Risks, Gaps and Opportunities",[1056,2029,1076],{"class":1062},[1056,2031,1107],{"class":1062},[1056,2033,2034,2036,2039,2041],{"class":1058,"line":1349},[1056,2035,1088],{"class":1062},[1056,2037,2038],{"class":1102},"Stakeholders",[1056,2040,1076],{"class":1062},[1056,2042,1107],{"class":1062},[1056,2044,2045,2047,2050],{"class":1058,"line":1635},[1056,2046,1088],{"class":1062},[1056,2048,2049],{"class":1102},"Conclusion",[1056,2051,1202],{"class":1062},[1056,2053,2054],{"class":1058,"line":1641},[1056,2055,2056],{"class":1062},"  ],\n",[1056,2058,2059,2061,2064,2066,2068],{"class":1058,"line":1662},[1056,2060,1069],{"class":1062},[1056,2062,2063],{"class":1072},"must_align_to_directive",[1056,2065,1076],{"class":1062},[1056,2067,1079],{"class":1062},[1056,2069,1122],{"class":1062},[1056,2071,2072,2074,2077,2079,2081],{"class":1058,"line":1683},[1056,2073,1069],{"class":1062},[1056,2075,2076],{"class":1072},"must_be_grounded_in_transcript",[1056,2078,1076],{"class":1062},[1056,2080,1079],{"class":1062},[1056,2082,1122],{"class":1062},[1056,2084,2085,2087,2090,2092,2094],{"class":1058,"line":1703},[1056,2086,1069],{"class":1062},[1056,2088,2089],{"class":1072},"schema_notes",[1056,2091,1076],{"class":1062},[1056,2093,1079],{"class":1062},[1056,2095,1082],{"class":1062},[1056,2097,2098,2100,2103,2105,2107,2109,2111,2114,2116],{"class":1058,"line":1718},[1056,2099,1088],{"class":1062},[1056,2101,2102],{"class":1091},"RISK_requires",[1056,2104,1076],{"class":1062},[1056,2106,1079],{"class":1062},[1056,2108,1290],{"class":1062},[1056,2110,1076],{"class":1062},[1056,2112,2113],{"class":1102},"severity",[1056,2115,1076],{"class":1062},[1056,2117,1310],{"class":1062},[1056,2119,2120,2122,2125,2127,2129,2131,2133,2136,2138],{"class":1058,"line":1733},[1056,2121,1088],{"class":1062},[1056,2123,2124],{"class":1091},"INSIGHT_requires",[1056,2126,1076],{"class":1062},[1056,2128,1079],{"class":1062},[1056,2130,1290],{"class":1062},[1056,2132,1076],{"class":1062},[1056,2134,2135],{"class":1102},"category",[1056,2137,1076],{"class":1062},[1056,2139,1310],{"class":1062},[1056,2141,2142,2144,2147,2149,2151,2153,2155,2157,2159,2161,2163,2166,2168],{"class":1058,"line":1739},[1056,2143,1088],{"class":1062},[1056,2145,2146],{"class":1091},"SENTIMENT_requires",[1056,2148,1076],{"class":1062},[1056,2150,1079],{"class":1062},[1056,2152,1290],{"class":1062},[1056,2154,1076],{"class":1062},[1056,2156,2135],{"class":1102},[1056,2158,1076],{"class":1062},[1056,2160,1300],{"class":1062},[1056,2162,1099],{"class":1062},[1056,2164,2165],{"class":1102},"rating",[1056,2167,1076],{"class":1062},[1056,2169,2170],{"class":1062},"]\n",[1056,2172,2173],{"class":1058,"line":1744},[1056,2174,1143],{"class":1062},[1056,2176,2177],{"class":1058,"line":1764},[1056,2178,1149],{"class":1062},[455,2180,2181],{},[476,2182,2183,2185],{},[484,2184,493],{}," (example)",[709,2187,2189],{"className":1050,"code":2188,"language":1052,"meta":715,"style":715},"{\n  \"dataset\": \"findings\u002Fgate_test\",\n  \"tags\": [\"findings_agent\", \"black_mesa\", \"anonymized\"],\n  \"original_task_id_hash\": \"4ec6af6be8582669\",\n  \"product_name\": \"Akamai App & API Protector\",\n  \"expected_tools\": [\"platformSearch\"],\n  \"reference_output\": {\n    \"summary\": \"...\",\n    \"report\": \"...\",\n    \"findings\": []\n  },\n  \"anonymization\": {\n    \"tenant_replaced_with\": \"Black Mesa\",\n    \"people_synthetic\": true,\n    \"images_stripped\": true\n  }\n}\n",[484,2190,2191,2195,2213,2252,2271,2290,2312,2325,2344,2363,2377,2381,2393,2411,2423,2435,2439],{"__ignoreMap":715},[1056,2192,2193],{"class":1058,"line":1059},[1056,2194,1063],{"class":1062},[1056,2196,2197,2199,2201,2203,2205,2207,2209,2211],{"class":1058,"line":1066},[1056,2198,1069],{"class":1062},[1056,2200,1262],{"class":1072},[1056,2202,1076],{"class":1062},[1056,2204,1079],{"class":1062},[1056,2206,1099],{"class":1062},[1056,2208,592],{"class":1102},[1056,2210,1076],{"class":1062},[1056,2212,1107],{"class":1062},[1056,2214,2215,2217,2219,2221,2223,2225,2227,2230,2232,2234,2236,2239,2241,2243,2245,2248,2250],{"class":1058,"line":1085},[1056,2216,1069],{"class":1062},[1056,2218,1283],{"class":1072},[1056,2220,1076],{"class":1062},[1056,2222,1079],{"class":1062},[1056,2224,1290],{"class":1062},[1056,2226,1076],{"class":1062},[1056,2228,2229],{"class":1102},"findings_agent",[1056,2231,1076],{"class":1062},[1056,2233,1300],{"class":1062},[1056,2235,1099],{"class":1062},[1056,2237,2238],{"class":1102},"black_mesa",[1056,2240,1076],{"class":1062},[1056,2242,1300],{"class":1062},[1056,2244,1099],{"class":1062},[1056,2246,2247],{"class":1102},"anonymized",[1056,2249,1076],{"class":1062},[1056,2251,1310],{"class":1062},[1056,2253,2254,2256,2258,2260,2262,2264,2267,2269],{"class":1058,"line":1110},[1056,2255,1069],{"class":1062},[1056,2257,1032],{"class":1072},[1056,2259,1076],{"class":1062},[1056,2261,1079],{"class":1062},[1056,2263,1099],{"class":1062},[1056,2265,2266],{"class":1102},"4ec6af6be8582669",[1056,2268,1076],{"class":1062},[1056,2270,1107],{"class":1062},[1056,2272,2273,2275,2278,2280,2282,2284,2286,2288],{"class":1058,"line":1125},[1056,2274,1069],{"class":1062},[1056,2276,2277],{"class":1072},"product_name",[1056,2279,1076],{"class":1062},[1056,2281,1079],{"class":1062},[1056,2283,1099],{"class":1062},[1056,2285,1524],{"class":1102},[1056,2287,1076],{"class":1062},[1056,2289,1107],{"class":1062},[1056,2291,2292,2294,2297,2299,2301,2303,2305,2308,2310],{"class":1058,"line":1140},[1056,2293,1069],{"class":1062},[1056,2295,2296],{"class":1072},"expected_tools",[1056,2298,1076],{"class":1062},[1056,2300,1079],{"class":1062},[1056,2302,1290],{"class":1062},[1056,2304,1076],{"class":1062},[1056,2306,2307],{"class":1102},"platformSearch",[1056,2309,1076],{"class":1062},[1056,2311,1310],{"class":1062},[1056,2313,2314,2316,2319,2321,2323],{"class":1058,"line":1146},[1056,2315,1069],{"class":1062},[1056,2317,2318],{"class":1072},"reference_output",[1056,2320,1076],{"class":1062},[1056,2322,1079],{"class":1062},[1056,2324,1082],{"class":1062},[1056,2326,2327,2329,2332,2334,2336,2338,2340,2342],{"class":1058,"line":1244},[1056,2328,1088],{"class":1062},[1056,2330,2331],{"class":1091},"summary",[1056,2333,1076],{"class":1062},[1056,2335,1079],{"class":1062},[1056,2337,1099],{"class":1062},[1056,2339,1573],{"class":1102},[1056,2341,1076],{"class":1062},[1056,2343,1107],{"class":1062},[1056,2345,2346,2348,2351,2353,2355,2357,2359,2361],{"class":1058,"line":1257},[1056,2347,1088],{"class":1062},[1056,2349,2350],{"class":1091},"report",[1056,2352,1076],{"class":1062},[1056,2354,1079],{"class":1062},[1056,2356,1099],{"class":1062},[1056,2358,1573],{"class":1102},[1056,2360,1076],{"class":1062},[1056,2362,1107],{"class":1062},[1056,2364,2365,2367,2370,2372,2374],{"class":1058,"line":1278},[1056,2366,1088],{"class":1062},[1056,2368,2369],{"class":1091},"findings",[1056,2371,1076],{"class":1062},[1056,2373,1079],{"class":1062},[1056,2375,2376],{"class":1062}," []\n",[1056,2378,2379],{"class":1058,"line":1313},[1056,2380,1207],{"class":1062},[1056,2382,2383,2385,2387,2389,2391],{"class":1058,"line":1344},[1056,2384,1069],{"class":1062},[1056,2386,1073],{"class":1072},[1056,2388,1076],{"class":1062},[1056,2390,1079],{"class":1062},[1056,2392,1082],{"class":1062},[1056,2394,2395,2397,2399,2401,2403,2405,2407,2409],{"class":1058,"line":1349},[1056,2396,1088],{"class":1062},[1056,2398,1092],{"class":1091},[1056,2400,1076],{"class":1062},[1056,2402,1079],{"class":1062},[1056,2404,1099],{"class":1062},[1056,2406,995],{"class":1102},[1056,2408,1076],{"class":1062},[1056,2410,1107],{"class":1062},[1056,2412,2413,2415,2417,2419,2421],{"class":1058,"line":1635},[1056,2414,1088],{"class":1062},[1056,2416,1115],{"class":1091},[1056,2418,1076],{"class":1062},[1056,2420,1079],{"class":1062},[1056,2422,1122],{"class":1062},[1056,2424,2425,2427,2429,2431,2433],{"class":1058,"line":1641},[1056,2426,1088],{"class":1062},[1056,2428,1130],{"class":1091},[1056,2430,1076],{"class":1062},[1056,2432,1079],{"class":1062},[1056,2434,1137],{"class":1062},[1056,2436,2437],{"class":1058,"line":1662},[1056,2438,1143],{"class":1062},[1056,2440,2441],{"class":1058,"line":1683},[1056,2442,1149],{"class":1062},[808,2444,2447,2493,2650],{"icon":2445,"label":2446},"i-lucide-code","1.4.7 Python SDK upload (nested JSON)",[709,2448,2452],{"className":2449,"code":2450,"language":2451,"meta":715,"style":715},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","export LANGFUSE_PUBLIC_KEY=pk-lf-...\nexport LANGFUSE_SECRET_KEY=sk-lf-...\nexport LANGFUSE_HOST=https:\u002F\u002Fcloud.langfuse.com\n","bash",[484,2453,2454,2469,2481],{"__ignoreMap":715},[1056,2455,2456,2459,2463,2466],{"class":1058,"line":1059},[1056,2457,2458],{"class":1072},"export",[1056,2460,2462],{"class":2461},"sTEyZ"," LANGFUSE_PUBLIC_KEY",[1056,2464,2465],{"class":1062},"=",[1056,2467,2468],{"class":2461},"pk-lf-...\n",[1056,2470,2471,2473,2476,2478],{"class":1058,"line":1066},[1056,2472,2458],{"class":1072},[1056,2474,2475],{"class":2461}," LANGFUSE_SECRET_KEY",[1056,2477,2465],{"class":1062},[1056,2479,2480],{"class":2461},"sk-lf-...\n",[1056,2482,2483,2485,2488,2490],{"class":1058,"line":1085},[1056,2484,2458],{"class":1072},[1056,2486,2487],{"class":2461}," LANGFUSE_HOST",[1056,2489,2465],{"class":1062},[1056,2491,2492],{"class":2461},"https:\u002F\u002Fcloud.langfuse.com\n",[709,2494,2498],{"className":2495,"code":2496,"language":2497,"meta":715,"style":715},"language-python shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","from langfuse import get_client\n\nlangfuse = get_client()\n\ndataset_name = \"conversation-namer\u002Fsmoke_test\"\n\nlangfuse.create_dataset(\n    name=dataset_name,\n    description=\"Smoke test set for Conversation Namer\",\n)\n\nitems = [\n    {\n        \"input\": {\n            \"message\": \"Can you compare our CrowdStrike and Wiz renewals for next quarter?\"\n        },\n        \"expected_output\": {\"title\": \"CrowdStrike vs Wiz renewals\"},\n        \"metadata\": {\n            \"dataset\": dataset_name,\n            \"tags\": [\"conversation_namer\", \"synthetic\"],\n        },\n    },\n]\n\nfor item in items:\n    langfuse.create_dataset_item(\n        dataset_name=dataset_name,\n        input=item[\"input\"],\n        expected_output=item[\"expected_output\"],\n        metadata=item.get(\"metadata\"),\n    )\n","python",[484,2499,2500,2505,2511,2516,2520,2525,2529,2534,2539,2544,2549,2553,2558,2562,2567,2572,2577,2582,2587,2592,2597,2601,2605,2609,2613,2618,2623,2628,2633,2638,2644],{"__ignoreMap":715},[1056,2501,2502],{"class":1058,"line":1059},[1056,2503,2504],{},"from langfuse import get_client\n",[1056,2506,2507],{"class":1058,"line":1066},[1056,2508,2510],{"emptyLinePlaceholder":2509},true,"\n",[1056,2512,2513],{"class":1058,"line":1085},[1056,2514,2515],{},"langfuse = get_client()\n",[1056,2517,2518],{"class":1058,"line":1110},[1056,2519,2510],{"emptyLinePlaceholder":2509},[1056,2521,2522],{"class":1058,"line":1125},[1056,2523,2524],{},"dataset_name = \"conversation-namer\u002Fsmoke_test\"\n",[1056,2526,2527],{"class":1058,"line":1140},[1056,2528,2510],{"emptyLinePlaceholder":2509},[1056,2530,2531],{"class":1058,"line":1146},[1056,2532,2533],{},"langfuse.create_dataset(\n",[1056,2535,2536],{"class":1058,"line":1244},[1056,2537,2538],{},"    name=dataset_name,\n",[1056,2540,2541],{"class":1058,"line":1257},[1056,2542,2543],{},"    description=\"Smoke test set for Conversation Namer\",\n",[1056,2545,2546],{"class":1058,"line":1278},[1056,2547,2548],{},")\n",[1056,2550,2551],{"class":1058,"line":1313},[1056,2552,2510],{"emptyLinePlaceholder":2509},[1056,2554,2555],{"class":1058,"line":1344},[1056,2556,2557],{},"items = [\n",[1056,2559,2560],{"class":1058,"line":1349},[1056,2561,1638],{},[1056,2563,2564],{"class":1058,"line":1635},[1056,2565,2566],{},"        \"input\": {\n",[1056,2568,2569],{"class":1058,"line":1641},[1056,2570,2571],{},"            \"message\": \"Can you compare our CrowdStrike and Wiz renewals for next quarter?\"\n",[1056,2573,2574],{"class":1058,"line":1662},[1056,2575,2576],{},"        },\n",[1056,2578,2579],{"class":1058,"line":1683},[1056,2580,2581],{},"        \"expected_output\": {\"title\": \"CrowdStrike vs Wiz renewals\"},\n",[1056,2583,2584],{"class":1058,"line":1703},[1056,2585,2586],{},"        \"metadata\": {\n",[1056,2588,2589],{"class":1058,"line":1718},[1056,2590,2591],{},"            \"dataset\": dataset_name,\n",[1056,2593,2594],{"class":1058,"line":1733},[1056,2595,2596],{},"            \"tags\": [\"conversation_namer\", \"synthetic\"],\n",[1056,2598,2599],{"class":1058,"line":1739},[1056,2600,2576],{},[1056,2602,2603],{"class":1058,"line":1744},[1056,2604,1736],{},[1056,2606,2607],{"class":1058,"line":1764},[1056,2608,2170],{},[1056,2610,2611],{"class":1058,"line":1784},[1056,2612,2510],{"emptyLinePlaceholder":2509},[1056,2614,2615],{"class":1058,"line":1804},[1056,2616,2617],{},"for item in items:\n",[1056,2619,2620],{"class":1058,"line":1817},[1056,2621,2622],{},"    langfuse.create_dataset_item(\n",[1056,2624,2625],{"class":1058,"line":1830},[1056,2626,2627],{},"        dataset_name=dataset_name,\n",[1056,2629,2630],{"class":1058,"line":1836},[1056,2631,2632],{},"        input=item[\"input\"],\n",[1056,2634,2635],{"class":1058,"line":1842},[1056,2636,2637],{},"        expected_output=item[\"expected_output\"],\n",[1056,2639,2641],{"class":1058,"line":2640},30,[1056,2642,2643],{},"        metadata=item.get(\"metadata\"),\n",[1056,2645,2647],{"class":1058,"line":2646},31,[1056,2648,2649],{},"    )\n",[455,2651,2652],{},"Same pattern for Findings — pass the full nested objects; no need to flatten.",[808,2654,2657],{"icon":2655,"label":2656},"i-lucide-square-check","1.4.8 Checklist before upload",[567,2658,2661,2669,2677,2688,2699,2705,2716],{"className":2659},[2660],"contains-task-list",[473,2662,2665,2668],{"className":2663},[2664],"task-list-item",[486,2666],{"disabled":2509,"type":2667},"checkbox"," Agent under test is clear",[473,2670,2672,794,2674,2676],{"className":2671},[2664],[486,2673],{"disabled":2509,"type":2667},[484,2675,486],{}," keys match prompt variables 1:1",[473,2678,2680,794,2682,2684,2685,2687],{"className":2679},[2664],[486,2681],{"disabled":2509,"type":2667},[484,2683,490],{}," is a golden answer ",[476,2686,642],{}," judge guidance (not mixed without intent)",[473,2689,2691,2693,2694,2696,2697],{"className":2690},[2664],[486,2692],{"disabled":2509,"type":2667}," For open-ended agents: guidance in ",[484,2695,490],{},"; full goldens (if kept) in ",[484,2698,865],{},[473,2700,2702,2704],{"className":2701},[2664],[486,2703],{"disabled":2509,"type":2667}," Production data is anonymized",[473,2706,2708,2710,2711,2713,2714],{"className":2707},[2664],[486,2709],{"disabled":2509,"type":2667}," Name is ",[484,2712,739],{}," or ",[484,2715,752],{},[473,2717,2719,2721],{"className":2718},[2664],[486,2720],{"disabled":2509,"type":2667}," Spot-check 2–3 items in the Langfuse UI",[598,2723],{},[601,2725,2727],{"id":2726},"_2-creating-evaluators","2. Creating Evaluators",[455,2729,2730],{},"Evaluators are the scoring definitions you attach when you run prompt experiments.",[455,2732,2733,2734,2737],{},"Today we ",[476,2735,2736],{},"mostly set up LLM-as-judge evaluators"," in Langfuse.",[455,2739,2740,2743,2744,2747,2748,1370,2751,2753,2754,2759],{},[476,2741,2742],{},"Code-based checks"," still matter for deterministic rules, but we typically implement them in the ",[476,2745,2746],{},"application"," and\u002For ",[476,2749,2750],{},"unit\u002Fintegration tests",[484,2752,837],{},", rather than as the primary Langfuse experiment evaluators. Langfuse also supports ",[459,2755,2758],{"href":2756,"rel":2757},"https:\u002F\u002Flangfuse.com\u002Fdocs\u002Fevaluation\u002Fevaluation-methods\u002Fcode-evaluators",[463],"Code Evaluators"," if needed later.",[455,2761,2762],{},"Rule of thumb:",[567,2764,2765,2771],{},[473,2766,2767,2768,2770],{},"Needs reading comprehension \u002F judgment → ",[476,2769,918],{}," (Langfuse)",[473,2772,2773,2774,2776],{},"A junior engineer could assert it in a test → ",[476,2775,484],{}," (app or unit tests)",[654,2778,2780],{"id":2779},"_21-create-your-first-llm-as-judge-5-steps","2.1 Create your first LLM-as-judge (5 steps)",[470,2782,2783,2789,2795,2801,2818],{},[473,2784,2785,2788],{},[476,2786,2787],{},"Pick one dimension"," — e.g. “grounded in transcript” or “aligns with directive”.",[473,2790,2791,2794],{},[476,2792,2793],{},"Choose score shape"," — boolean \u002F categorical \u002F numeric.",[473,2796,2797,2800],{},[476,2798,2799],{},"Write the judge prompt"," — explicit pass\u002Ffail or category rules; one job only.",[473,2802,2803,2806,2807,487,2810,2813,2814,2817],{},[476,2804,2805],{},"Create it in Langfuse"," — map ",[484,2808,2809],{},"{{input}}",[484,2811,2812],{},"{{output}}"," (and ",[484,2815,2816],{},"{{expected_output}}"," only if needed).",[473,2819,2820,2823],{},[476,2821,2822],{},"Verify mapping"," — use Prompt Preview; spot-check that variables populate as expected.",[455,2825,2826,2827,2830],{},"Start with ",[476,2828,2829],{},"2–3 focused judges"," per agent. Add more only when debugging a specific failure class.",[455,2832,2833],{},"Findings starter pack:",[470,2835,2836,2842],{},[473,2837,2838,2841],{},[484,2839,2840],{},"findings_agent.grounding"," — Boolean",[473,2843,2844,2847,2848,487,2851,487,2854,494],{},[484,2845,2846],{},"findings_agent.directive_alignment"," — Categorical (",[484,2849,2850],{},"fail",[484,2852,2853],{},"partial",[484,2855,2856],{},"pass",[654,2858,2860],{"id":2859},"_22-naming","2.2 Naming",[709,2862,2865],{"className":2863,"code":2864,"language":714,"meta":715},[712],"{agentName}.{dimension}\n",[484,2866,2864],{"__ignoreMap":715},[455,2868,761,2869,765,2872,765,2874,668],{},[484,2870,2871],{},"conversation_namer.title_quality",[484,2873,2840],{},[484,2875,2846],{},[654,2877,2879],{"id":2878},"_23-create-in-langfuse-ui","2.3 Create in Langfuse (UI)",[470,2881,2882,2893,2903,2913,2920,2923],{},[473,2883,2884,2885,2888,2889,2892],{},"Ensure an ",[476,2886,2887],{},"LLM Connection"," exists (",[476,2890,2891],{},"Settings → LLM Connections","). The judge model must support structured output.",[473,2894,2895,2896,2899,2900,668],{},"Open ",[476,2897,2898],{},"Evaluators"," → ",[476,2901,2902],{},"+ Set up Evaluator",[473,2904,2905,2906,2909,2910,668],{},"Pick a managed template, or ",[476,2907,2908],{},"Custom"," and paste your judge prompt with ",[484,2911,2912],{},"{{variables}}",[473,2914,2915,2916,2919],{},"Choose ",[476,2917,2918],{},"score type"," (boolean \u002F categorical \u002F numeric). For categorical, define labels and numeric mapping.",[473,2921,2922],{},"Map variables to Input \u002F Output \u002F Expected output (add JSONPath if needed).",[473,2924,2925,2926,668],{},"Save. Attach these evaluators when ",[459,2927,2928],{"href":510},"running experiments",[455,2930,2931,2932,668],{},"Official guide: ",[459,2933,2936],{"href":2934,"rel":2935},"https:\u002F\u002Flangfuse.com\u002Fdocs\u002Fevaluation\u002Fevaluation-methods\u002Fllm-as-a-judge",[463],"LLM-as-a-Judge",[654,2938,2940],{"id":2939},"_24-evaluator-reference-expand-as-needed","2.4 Evaluator reference (expand as needed)",[805,2942,2943,3000,3088,3134,3211,3292,3324,3374],{},[808,2944,2947],{"icon":2945,"label":2946},"i-lucide-layout-list","2.4.1 Where evaluators fit in Langfuse",[514,2948,2949,2959],{},[517,2950,2951],{},[520,2952,2953,2956],{},[523,2954,2955],{},"If you want to...",[523,2957,2958],{},"Langfuse \u002F approach we use",[530,2960,2961,2970,2980,2989],{},[520,2962,2963,2966],{},[535,2964,2965],{},"Build a reusable set of test cases",[535,2967,2968],{},[459,2969,575],{"href":480},[520,2971,2972,2975],{},[535,2973,2974],{},"Compare prompt or model changes",[535,2976,2977],{},[459,2978,2979],{"href":510},"Experiments",[520,2981,2982,2985],{},[535,2983,2984],{},"Automatically score quality (grounding, alignment, tone, …)",[535,2986,2987],{},[476,2988,2936],{},[520,2990,2991,2994],{},[535,2992,2993],{},"Run deterministic checks (schema, length, exact match)",[535,2995,2996,2999],{},[476,2997,2998],{},"Code checks"," in the app or unit\u002Fintegration tests",[808,3001,3004,3066,3085],{"icon":3002,"label":3003},"i-lucide-gauge","2.4.2 Score types",[514,3005,3006,3019],{},[517,3007,3008],{},[520,3009,3010,3013,3016],{},[523,3011,3012],{},"Score type",[523,3014,3015],{},"Use when",[523,3017,3018],{},"Typical use",[530,3020,3021,3034,3053],{},[520,3022,3023,3028,3031],{},[535,3024,3025],{},[476,3026,3027],{},"Boolean",[535,3029,3030],{},"Clear pass\u002Ffail",[535,3032,3033],{},"Hard checks (e.g. grounding)",[520,3035,3036,3041,3050],{},[535,3037,3038],{},[476,3039,3040],{},"Categorical",[535,3042,3043,3044,487,3046,487,3048,494],{},"Small fixed tiers (",[484,3045,2850],{},[484,3047,2853],{},[484,3049,2856],{},[535,3051,3052],{},"Soft quality bands",[520,3054,3055,3060,3063],{},[535,3056,3057],{},[476,3058,3059],{},"Numeric",[535,3061,3062],{},"Fine-grained ranking",[535,3064,3065],{},"When tiers are too coarse",[455,3067,3068,3069,2899,3071,765,3074,2899,3076,765,3079,2899,3081,3084],{},"Map categorical labels to numbers when useful (e.g. ",[484,3070,2850],{},[484,3072,3073],{},"0",[484,3075,2853],{},[484,3077,3078],{},"0.5",[484,3080,2856],{},[484,3082,3083],{},"1",").",[455,3086,3087],{},"Prefer boolean for hard correctness; categorical (3-tier) for softer quality.",[808,3089,3092],{"icon":3090,"label":3091},"i-lucide-pencil","2.4.3 Design principles",[470,3093,3094,3100,3106,3112,3118,3124],{},[473,3095,3096,3099],{},[476,3097,3098],{},"One job per evaluator"," — do not mix grounding + writing quality in one score.",[473,3101,3102,3105],{},[476,3103,3104],{},"Say what to check"," — avoid long “do not score X” lists.",[473,3107,3108,3111],{},[476,3109,3110],{},"Define pass\u002Ffail or categories explicitly"," with short examples.",[473,3113,3114,3117],{},[476,3115,3116],{},"Derive criteria from the use case"," — do not hard-code one product’s interview branch unless it is universal.",[473,3119,3120,3123],{},[476,3121,3122],{},"Map only the data needed"," — use JSONPath when you only need a nested field.",[473,3125,3126,3129,3130,3133],{},[476,3127,3128],{},"Name and version stably"," — keep ",[484,3131,3132],{},"{agent}.{dimension}"," names unchanged so later experiment comparisons stay readable.",[808,3135,3138,3179,3193],{"icon":3136,"label":3137},"i-lucide-link","2.4.4 Variable mapping",[514,3139,3140,3150],{},[517,3141,3142],{},[520,3143,3144,3147],{},[523,3145,3146],{},"Common variable",[523,3148,3149],{},"Typical source",[530,3151,3152,3161,3170],{},[520,3153,3154,3158],{},[535,3155,3156],{},[484,3157,2809],{},[535,3159,3160],{},"Dataset item input",[520,3162,3163,3167],{},[535,3164,3165],{},[484,3166,2812],{},[535,3168,3169],{},"Run output (filled when an experiment executes)",[520,3171,3172,3176],{},[535,3173,3174],{},[484,3175,2816],{},[535,3177,3178],{},"Dataset expected output (optional)",[455,3180,3181,3182,3185,3186,3189,3190,668],{},"Use ",[476,3183,3184],{},"JSONPath"," when you only need a nested field (e.g. Output → ",[484,3187,3188],{},"$.findings","). Confirm in Langfuse ",[476,3191,3192],{},"Prompt Preview",[455,3194,3195,3201,3202,3084,3204,3207,3210],{},[476,3196,3197,3198],{},"Skip ",[484,3199,3200],{},"expected_output"," when the rubric lives fully in the judge prompt, or the check is reference-free (e.g. grounding against transcript in ",[484,3203,486],{},[3205,3206],"br",{},[476,3208,3209],{},"Use it"," when item-specific guidance lives in the dataset.",[808,3212,3215,3218,3266,3272,3281],{"icon":3213,"label":3214},"i-lucide-sparkles","2.4.5 Simple example: Conversation Namer title quality",[455,3216,3217],{},"A light judge — good first custom evaluator to practise the shape.",[514,3219,3220,3229],{},[517,3221,3222],{},[520,3223,3224,3226],{},[523,3225,616],{},[523,3227,3228],{},"Value",[530,3230,3231,3242,3251],{},[520,3232,3233,3238],{},[535,3234,3235],{},[476,3236,3237],{},"Name",[535,3239,3240],{},[484,3241,2871],{},[520,3243,3244,3249],{},[535,3245,3246],{},[476,3247,3248],{},"Score",[535,3250,3027],{},[520,3252,3253,3258],{},[535,3254,3255],{},[476,3256,3257],{},"Maps",[535,3259,3260,3262,3263,3265],{},[484,3261,2809],{}," → Input, ",[484,3264,2812],{}," → Output",[709,3267,3270],{"className":3268,"code":3269,"language":714,"meta":715},[712],"You evaluate whether the OUTPUT title is a good short label for the INPUT user message.\n\nPass (true) if ALL are true:\n- Title is non-empty\n- Title is at most 5 words\n- Title reflects the main topic of the user message (no unrelated subject)\n\nFail (false) otherwise.\nIf OUTPUT is empty or malformed, return false.\n\nINPUT:\n{{input}}\n\nOUTPUT:\n{{output}}\n",[484,3271,3269],{"__ignoreMap":715},[455,3273,3274,3275,2713,3278,668],{},"Score output: return ONLY ",[484,3276,3277],{},"true",[484,3279,3280],{},"false",[455,3282,3283,3284,3287,3288,3291],{},"For a pure length rule (",[484,3285,3286],{},"≤ 5 words","), prefer a ",[476,3289,3290],{},"unit test"," or in-app validation instead of a judge.",[808,3293,3296,3308,3314],{"icon":3294,"label":3295},"i-lucide-shield-check","2.4.6 Findings: grounding judge (Boolean)",[455,3297,3298,794,3301,3262,3303,3305,3306,494],{},[476,3299,3300],{},"Maps:",[484,3302,2809],{},[484,3304,2812],{}," → Output (no ",[484,3307,3200],{},[709,3309,3312],{"className":3310,"code":3311,"language":714,"meta":715},[712],"You evaluate whether the agent OUTPUT is grounded in the interview transcript from INPUT.\n\nTask:\nDecide if material claims in the summary, report, and findings are supported by the transcript.\n\nRules:\n- Every material claim must be supported by the transcript\n- Paraphrase is allowed; invention is not\n- Minor omissions are OK; fabrication is not\n\nFail (false) if any material claim is invented, over-precise beyond the transcript, contradicts the transcript without uncertainty, or invents rationale\u002Fabbreviation expansions not stated.\n\nPass (true) only if all material claims are transcript-supported.\nIf OUTPUT is empty or malformed, return false.\n\nList any ungrounded claims briefly before deciding.\n\nINPUT:\n{{input}}\n\nOUTPUT:\n{{output}}\n",[484,3313,3311],{"__ignoreMap":715},[455,3315,3316,3317,3319,3320,2713,3322,668],{},"Score reasoning: 1–3 sentences; if false, name the worst ungrounded claim(s).",[3205,3318],{},"\nScore output: return ONLY ",[484,3321,3277],{},[484,3323,3280],{},[808,3325,3328,3356,3362],{"icon":3326,"label":3327},"i-lucide-list-checks","2.4.7 Findings: directive alignment judge (Categorical)",[455,3329,3330,794,3333,2899,3335,765,3337,2899,3339,765,3341,2899,3343,3345,3347,794,3349,3262,3351,3353,3354,494],{},[476,3331,3332],{},"Categories:",[484,3334,2850],{},[484,3336,3073],{},[484,3338,2853],{},[484,3340,3078],{},[484,3342,2856],{},[484,3344,3083],{},[3205,3346],{},[476,3348,3300],{},[484,3350,2809],{},[484,3352,2812],{}," → Output (directive usually inside ",[484,3355,486],{},[709,3357,3360],{"className":3358,"code":3359,"language":714,"meta":715},[712],"You evaluate whether the Findings\u002FReport agent OUTPUT aligns with the interview DIRECTIVE in INPUT.\n\nTask:\nJudge how well the REPORT and FINDINGS deliver the directive’s information goals, based on what the transcript actually captured.\n\nDo not assume a fixed interview structure. Derive success criteria from the directive itself.\n\nCheck only:\n1) Main directive objective(s) appear in report and\u002For findings\n2) Key requested topics are covered when the transcript has answers\n3) Findings are decision-useful for the directive\n4) Important directive-relevant transcript content is not systematically missing\n\nIf the transcript lacked answers for a topic, do not penalize missing content for that topic.\n\nCategories:\n- fail: largely misses the directive, or major goals missing\n- partial: some alignment, important gaps remain\n- pass: strong alignment; minor gaps only\n\nINPUT:\n{{input}}\n\nOUTPUT:\n{{output}}\n",[484,3361,3359],{"__ignoreMap":715},[455,3363,3364,3365,3319,3367,765,3369,3371,3372,668],{},"Score reasoning: 2–4 sentences with strongest coverage and most important gaps.",[3205,3366],{},[484,3368,2850],{},[484,3370,2853],{},", or ",[484,3373,2856],{},[808,3375,3377],{"icon":2655,"label":3376},"2.4.8 Checklist before saving an evaluator",[567,3378,3380,3386,3392,3398,3404,3410,3416,3424],{"className":3379},[2660],[473,3381,3383,3385],{"className":3382},[2664],[486,3384],{"disabled":2509,"type":2667}," One clear dimension",[473,3387,3389,3391],{"className":3388},[2664],[486,3390],{"disabled":2509,"type":2667}," Score type matches the check (boolean \u002F categorical \u002F numeric)",[473,3393,3395,3397],{"className":3394},[2664],[486,3396],{"disabled":2509,"type":2667}," Categorical labels + numeric mapping defined (if used)",[473,3399,3401,3403],{"className":3400},[2664],[486,3402],{"disabled":2509,"type":2667}," Judge prompt states positive checks",[473,3405,3407,3409],{"className":3406},[2664],[486,3408],{"disabled":2509,"type":2667}," Variable mapping verified in Prompt Preview",[473,3411,3413,3415],{"className":3412},[2664],[486,3414],{"disabled":2509,"type":2667}," JSONPath used when only a nested field is needed",[473,3417,3419,794,3421,3423],{"className":3418},[2664],[486,3420],{"disabled":2509,"type":2667},[484,3422,3200],{}," only when item-specific guidance is required",[473,3425,3427,3429,3430],{"className":3426},[2664],[486,3428],{"disabled":2509,"type":2667}," Name follows ",[484,3431,3132],{},[598,3433],{},[601,3435,3437],{"id":3436},"_3-running-experiments","3. Running Experiments",[455,3439,3440,3441,3443,3444,3447,3448,3451],{},"Use a ",[476,3442,1262],{}," and ",[476,3445,3446],{},"evaluators"," together in a Langfuse ",[476,3449,3450],{},"Prompt Experiment"," to compare prompt versions and decide whether to ship a change.",[455,3453,3454,3455,668],{},"Example completed runs: ",[459,3456,3458,3460],{"href":587,"rel":3457},[463],[484,3459,592],{}," experiments",[455,3462,3463,3464,3469,3470],{},"Official docs: ",[459,3465,3468],{"href":3466,"rel":3467},"https:\u002F\u002Flangfuse.com\u002Fdocs\u002Fevaluation\u002Fexperiments\u002Fexperiments-via-ui",[463],"Experiments via UI"," · ",[459,3471,3474],{"href":3472,"rel":3473},"https:\u002F\u002Flangfuse.com\u002Fdocs\u002Fevaluation\u002Fexperiments\u002Fdata-model",[463],"Experiments data model",[654,3476,3478],{"id":3477},"_31-what-a-langfuse-experiment-is","3.1 What a Langfuse experiment is",[514,3480,3481,3491],{},[517,3482,3483],{},[520,3484,3485,3488],{},[523,3486,3487],{},"Concept",[523,3489,3490],{},"Meaning",[530,3492,3493,3509,3519,3529,3538],{},[520,3494,3495,3499],{},[535,3496,3497],{},[476,3498,539],{},[535,3500,3501,3502,3504,3505,765,3507,494],{},"Frozen test cases (",[484,3503,486],{},", optional ",[484,3506,490],{},[484,3508,493],{},[520,3510,3511,3516],{},[535,3512,3513],{},[476,3514,3515],{},"Prompt",[535,3517,3518],{},"Versioned prompt from Prompt Management",[520,3520,3521,3526],{},[535,3522,3523],{},[476,3524,3525],{},"Experiment (Prompt Experiment)",[535,3527,3528],{},"Runs the selected prompt on each dataset item",[520,3530,3531,3535],{},[535,3532,3533],{},[476,3534,549],{},[535,3536,3537],{},"Scores each experiment item output (LLM-as-judge and\u002For code)",[520,3539,3540,3545],{},[535,3541,3542],{},[476,3543,3544],{},"Experiment comparison",[535,3546,3547],{},"Side-by-side aggregate + item-level score comparison across runs",[709,3549,3552],{"className":3550,"code":3551,"language":714,"meta":715},[712],"Dataset item input\n        │\n        ▼\nPrompt version (variables filled from input)\n        │\n        ▼\nModel output\n        │\n        ▼\nEvaluators attach scores\n        │\n        ▼\nCompare runs → promote or reject prompt\n",[484,3553,3551],{"__ignoreMap":715},[455,3555,3556,3559,3560,3563],{},[476,3557,3558],{},"Important:"," one experiment can attach ",[476,3561,3562],{},"multiple evaluators",". Do not create one experiment per score.",[654,3565,3567],{"id":3566},"_32-prerequisites","3.2 Prerequisites",[455,3569,3570],{},"Before running an experiment, confirm:",[470,3572,3573,3584,3597,3603],{},[473,3574,3575,3577,3578,3580,3581,3583],{},[476,3576,3515],{}," in Prompt Management with ",[484,3579,2912],{}," matching dataset ",[484,3582,486],{}," keys",[473,3585,3586,3588,3589,2713,3591,3593,3594],{},[476,3587,539],{}," uploaded (",[484,3590,960],{},[484,3592,956],{},") — see ",[459,3595,3596],{"href":480},"§1",[473,3598,3599,3602],{},[476,3600,3601],{},"LLM connection"," configured; default evaluation model supports structured output for judges",[473,3604,3605,3607,3608],{},[476,3606,2898],{}," created and able to target Experiments — see ",[459,3609,3610],{"href":501},"§2",[654,3612,3614],{"id":3613},"_33-run-a-prompt-experiment-ui","3.3 Run a Prompt Experiment (UI)",[470,3616,3617,3626,3640,3687],{},[473,3618,3619,3620,3622,3623,3625],{},"Go to ",[476,3621,575],{}," → open the dataset (e.g. ",[484,3624,592],{},") → spot-check 1–2 items",[473,3627,3628,3629,487,3632,2899,3635,2899,3637],{},"Click ",[476,3630,3631],{},"Start Experiment",[476,3633,3634],{},"Run Experiment",[476,3636,3450],{},[476,3638,3639],{},"Create",[473,3641,3642,3643],{},"Configure:\n",[567,3644,3645,3651,3659,3664,3669,3682],{},[473,3646,3647,3650],{},[476,3648,3649],{},"Experiment name"," (see naming below)",[473,3652,3653,3655,3656],{},[476,3654,3515],{}," + ",[476,3657,3658],{},"prompt version",[473,3660,3661,3663],{},[476,3662,3601],{}," \u002F model settings",[473,3665,3666,3668],{},[476,3667,539],{}," (usually already selected)",[473,3670,3671,3672,3675,3676,765,3678,765,3680,494],{},"Optional: ",[476,3673,3674],{},"structured output"," schema (recommended for Findings: ",[484,3677,2331],{},[484,3679,2350],{},[484,3681,2369],{},[473,3683,3684,3686],{},[476,3685,2898],{}," to attach (all gate evaluators)",[473,3688,3628,3689],{},[476,3690,3639],{},[455,3692,3693],{},"Langfuse runs the prompt per item, stores outputs, runs evaluators asynchronously, and shows aggregate scores. Runtime depends on dataset size, prompt length, and judge count.",[654,3695,3697],{"id":3696},"_34-experiment-naming","3.4 Experiment naming",[709,3699,3702],{"className":3700,"code":3701,"language":714,"meta":715},[712],"{agent}-{role}-{promptVersion}-{yyyymmdd}\n",[484,3703,3701],{"__ignoreMap":715},[455,3705,3706],{},"Examples:",[567,3708,3709,3714],{},[473,3710,3711],{},[484,3712,3713],{},"findings-baseline-v12-20260729",[473,3715,3716],{},[484,3717,3718],{},"findings-candidate-v13-20260729",[455,3720,3721,3722,3725,3726,3729,3730,3733],{},"For prompt gates: ",[476,3723,3724],{},"baseline"," = current production prompt; ",[476,3727,3728],{},"candidate"," = proposed version; ",[476,3731,3732],{},"same dataset + same evaluators"," for both.",[654,3735,3737],{"id":3736},"_35-compare-experiments","3.5 Compare experiments",[455,3739,3740],{},"After runs complete:",[470,3742,3743,3748,3751],{},[473,3744,2895,3745,3747],{},[476,3746,2979],{}," (or the dataset’s Experiments tab)",[473,3749,3750],{},"Select baseline and candidate runs",[473,3752,3753],{},"Compare aggregate scores, item-level regressions (especially boolean fails), and judge comments on failures",[455,3755,3756],{},"Always spot-check a few failed items manually before promoting.",[654,3758,3760],{"id":3759},"_36-experiment-reference-expand-as-needed","3.6 Experiment reference (expand as needed)",[805,3762,3763,3872,4009,4059,4100,4134,4224,4296],{},[808,3764,3766,3775,3869],{"icon":3136,"label":3765},"3.6.1 Prompt ↔ dataset variable mapping (Findings)",[455,3767,3768,3769,3771,3772,668],{},"A prompt is usable for Prompt Experiments when its ",[484,3770,2912],{}," match dataset item ",[476,3773,3774],{},"input keys",[514,3776,3777,3790],{},[517,3778,3779],{},[520,3780,3781,3784],{},[523,3782,3783],{},"Prompt variable",[523,3785,3786,3787,3789],{},"Dataset ",[484,3788,486],{}," key",[530,3791,3792,3803,3814,3825,3836,3847,3858],{},[520,3793,3794,3799],{},[535,3795,3796],{},[484,3797,3798],{},"{{user_name}}",[535,3800,3801],{},[484,3802,1402],{},[520,3804,3805,3810],{},[535,3806,3807],{},[484,3808,3809],{},"{{user_role}}",[535,3811,3812],{},[484,3813,1422],{},[520,3815,3816,3821],{},[535,3817,3818],{},[484,3819,3820],{},"{{organisation_context}}",[535,3822,3823],{},[484,3824,1442],{},[520,3826,3827,3832],{},[535,3828,3829],{},[484,3830,3831],{},"{{source_type}}",[535,3833,3834],{},[484,3835,1462],{},[520,3837,3838,3843],{},[535,3839,3840],{},[484,3841,3842],{},"{{source_info}}",[535,3844,3845],{},[484,3846,1482],{},[520,3848,3849,3854],{},[535,3850,3851],{},[484,3852,3853],{},"{{task_directive}}",[535,3855,3856],{},[484,3857,1605],{},[520,3859,3860,3865],{},[535,3861,3862],{},[484,3863,3864],{},"{{interview_transcript}}",[535,3866,3867],{},[484,3868,1625],{},[455,3870,3871],{},"If variables and input keys do not match, the experiment will fail or run with empty fields.",[808,3873,3876,3881,3912,3917,3946,3951,3954,3957,3963,3968],{"icon":3874,"label":3875},"i-lucide-git-branch","3.6.2 Findings Agent workflow (baseline → candidate → decide)",[455,3877,3878],{},[476,3879,3880],{},"A) Baseline (current production prompt)",[470,3882,3883,3887,3890,3896,3903,3909],{},[473,3884,2895,3885],{},[484,3886,592],{},[473,3888,3889],{},"Start Prompt Experiment",[473,3891,3892,3893],{},"Select ",[476,3894,3895],{},"current production prompt version",[473,3897,3898,3899,765,3901,494],{},"Attach gate evaluators (",[484,3900,2840],{},[484,3902,2846],{},[473,3904,3905,3906],{},"Name: ",[484,3907,3908],{},"findings-baseline-\u003Cprod-version>",[473,3910,3911],{},"Wait for scores",[455,3913,3914],{},[476,3915,3916],{},"B) Candidate (new prompt)",[470,3918,3919,3926,3935,3941],{},[473,3920,3921,3922,3925],{},"Save prompt edits as a ",[476,3923,3924],{},"new prompt version"," in Prompt Management",[473,3927,3928,3929,794,3932,3934],{},"Run another Prompt Experiment on the ",[476,3930,3931],{},"same",[484,3933,592],{}," dataset",[473,3936,3937,3938,3940],{},"Attach the ",[476,3939,3931],{}," evaluators",[473,3942,3905,3943],{},[484,3944,3945],{},"findings-candidate-\u003Cnew-version>",[455,3947,3948],{},[476,3949,3950],{},"C) Decide",[455,3952,3953],{},"Promote only if grounding and directive alignment do not regress materially, and no new systemic failure pattern appears in item review.",[455,3955,3956],{},"If it fails: revise the candidate prompt and rerun candidate only (keep baseline fixed).",[709,3958,3961],{"className":3959,"code":3960,"language":714,"meta":715},[712],"Prod prompt vN\n   │\n   ▼\nBaseline experiment on findings\u002Fgate_test + gate evaluators\n   │\n   ▼\nEdit prompt → save vN+1\n   │\n   ▼\nCandidate experiment on SAME dataset + SAME evaluators\n   │\n   ▼\nCompare in Langfuse Experiments\n   │\n   ├─ Pass → promote vN+1\n   └─ Fail → revise prompt, rerun candidate\n",[484,3962,3960],{"__ignoreMap":715},[455,3964,3965],{},[476,3966,3967],{},"What to look for",[514,3969,3970,3979],{},[517,3971,3972],{},[520,3973,3974,3976],{},[523,3975,549],{},[523,3977,3978],{},"Prefer",[530,3980,3981,3991],{},[520,3982,3983,3988],{},[535,3984,3985],{},[484,3986,3987],{},"grounding",[535,3989,3990],{},"pass-rate ≥ baseline",[520,3992,3993,3998],{},[535,3994,3995],{},[484,3996,3997],{},"directive_alignment",[535,3999,4000,4001,4004,4005,4008],{},"higher ",[484,4002,4003],{},"% pass",", lower ",[484,4006,4007],{},"% fail",", mean mapped score ≥ baseline − tolerance",[808,4010,4013,4046,4053],{"icon":4011,"label":4012},"i-lucide-app-window","3.6.3 UI vs SDK experiments",[514,4014,4015,4024],{},[517,4016,4017],{},[520,4018,4019,4022],{},[523,4020,4021],{},"Approach",[523,4023,3015],{},[530,4025,4026,4036],{},[520,4027,4028,4033],{},[535,4029,4030],{},[476,4031,4032],{},"Experiments via UI (Prompt Experiments)",[535,4034,4035],{},"Prompt-only changes; variables map cleanly from dataset input",[520,4037,4038,4043],{},[535,4039,4040],{},[476,4041,4042],{},"Experiments via SDK",[535,4044,4045],{},"Full app\u002Fagent logic, tools, retrieval, custom runtime config",[455,4047,4048,4049,4052],{},"Findings Agent ",[476,4050,4051],{},"prompt iteration"," fits UI Prompt Experiments well when structured output is enforced.",[455,4054,4055,4056,4058],{},"If the flow depends heavily on tool calls \u002F multi-step orchestration that Prompt Experiments cannot reproduce, use ",[476,4057,4042],{}," (or hybrid: UI for prompt drafts, SDK for full-agent realism).",[808,4060,4063,4066,4091,4097],{"icon":4061,"label":4062},"i-lucide-snowflake","3.6.4 Dataset freeze rules during experiments",[455,4064,4065],{},"To keep comparisons fair:",[470,4067,4068,4077,4085],{},[473,4069,4070,4071,4073,4074,4076],{},"Do ",[476,4072,875],{}," edit\u002Fadd\u002Fdelete ",[484,4075,956],{}," items while comparing prompts",[473,4078,4079,4080,4082,4083],{},"Put fresh cases into ",[484,4081,960],{}," (or a scratch set), not into ",[484,4084,956],{},[473,4086,4087,4088,4090],{},"Only expand ",[484,4089,956],{}," with reviewed items after the current comparison cycle",[455,4092,4093,4094,4096],{},"Langfuse experiments run against the dataset state at experiment time. Treat ",[484,4095,956],{}," as frozen for the duration of a promotion decision.",[455,4098,4099],{},"Optional: use dataset versioning (Items tab → version view) when available, so you can re-run against a historical snapshot.",[808,4101,4104,4110,4126,4131],{"icon":4102,"label":4103},"i-lucide-braces","3.6.5 Structured output tip (Findings)",[455,4105,4106,4107,4109],{},"For Findings experiments, enable ",[476,4108,3674],{}," with a schema requiring:",[567,4111,4112,4117,4121],{},[473,4113,4114,4116],{},[484,4115,2331],{}," (string)",[473,4118,4119,4116],{},[484,4120,2350],{},[473,4122,4123,4125],{},[484,4124,2369],{}," (array)",[455,4127,4128,4129,3084],{},"This improves parseability for judges, consistency across items, and JSONPath mapping (e.g. ",[484,4130,3188],{},[455,4132,4133],{},"Schemas can be created\u002Fsaved in Langfuse Playground and reused in experiments.",[808,4135,4138],{"icon":4136,"label":4137},"i-lucide-bug","3.6.6 Debugging failed or empty scores",[514,4139,4140,4153],{},[517,4141,4142],{},[520,4143,4144,4147,4150],{},[523,4145,4146],{},"Symptom",[523,4148,4149],{},"Likely cause",[523,4151,4152],{},"Fix",[530,4154,4155,4166,4177,4188,4199,4210],{},[520,4156,4157,4160,4163],{},[535,4158,4159],{},"Experiment fails immediately",[535,4161,4162],{},"Prompt variables ≠ dataset input keys",[535,4164,4165],{},"Align names exactly",[520,4167,4168,4171,4174],{},[535,4169,4170],{},"Empty outputs",[535,4172,4173],{},"LLM connection \u002F model issue",[535,4175,4176],{},"Check project LLM connection + logs",[520,4178,4179,4182,4185],{},[535,4180,4181],{},"No evaluator scores",[535,4183,4184],{},"Evaluator not attached \u002F wrong target",[535,4186,4187],{},"Attach evaluators; target Experiments",[520,4189,4190,4193,4196],{},[535,4191,4192],{},"Judge mapping empty",[535,4194,4195],{},"Wrong source or JSONPath",[535,4197,4198],{},"Fix mapping; use Prompt Preview",[520,4200,4201,4204,4207],{},[535,4202,4203],{},"Noisy scores",[535,4205,4206],{},"Judge prompt too broad",[535,4208,4209],{},"Split into one-dimension evaluators",[520,4211,4212,4215,4218],{},[535,4213,4214],{},"Need judge internals",[535,4216,4217],{},"—",[535,4219,4220,4221],{},"Filter traces by environment ",[484,4222,4223],{},"langfuse-llm-as-a-judge",[808,4225,4227],{"icon":2655,"label":4226},"3.6.7 Checklist before promoting a prompt",[567,4228,4230,4236,4242,4253,4260,4266,4272,4278,4284,4290],{"className":4229},[2660],[473,4231,4233,4235],{"className":4232},[2664],[486,4234],{"disabled":2509,"type":2667}," Prompt version is saved in Prompt Management (not an unsaved playground edit)",[473,4237,4239,4241],{"className":4238},[2664],[486,4240],{"disabled":2509,"type":2667}," Baseline experiment exists for current production prompt",[473,4243,4245,4247,4248,4250,4251,494],{"className":4244},[2664],[486,4246],{"disabled":2509,"type":2667}," Candidate experiment used the ",[476,4249,3931],{}," dataset (e.g. ",[484,4252,592],{},[473,4254,4256,4247,4258,3940],{"className":4255},[2664],[486,4257],{"disabled":2509,"type":2667},[476,4259,3931],{},[473,4261,4263,4265],{"className":4262},[2664],[486,4264],{"disabled":2509,"type":2667}," Structured output schema enabled (if required by agent)",[473,4267,4269,4271],{"className":4268},[2664],[486,4270],{"disabled":2509,"type":2667}," Aggregate scores reviewed",[473,4273,4275,4277],{"className":4274},[2664],[486,4276],{"disabled":2509,"type":2667}," Item-level failures reviewed (especially grounding fails)",[473,4279,4281,4283],{"className":4280},[2664],[486,4282],{"disabled":2509,"type":2667}," No dataset edits happened between baseline and candidate",[473,4285,4287,4289],{"className":4286},[2664],[486,4288],{"disabled":2509,"type":2667}," Promotion decision documented (pass \u002F fail + reason)",[473,4291,4293,4295],{"className":4292},[2664],[486,4294],{"disabled":2509,"type":2667}," Production prompt pointer updated only after pass",[808,4297,4300,4377,4382],{"icon":4298,"label":4299},"i-lucide-bookmark","3.6.8 Quick reference — Findings Agent",[514,4301,4302,4311],{},[517,4303,4304],{},[520,4305,4306,4308],{},[523,4307,525],{},[523,4309,4310],{},"Langfuse object",[530,4312,4313,4322,4332,4339,4350,4358,4369],{},[520,4314,4315,4318],{},[535,4316,4317],{},"Gate dataset",[535,4319,4320],{},[484,4321,592],{},[520,4323,4324,4327],{},[535,4325,4326],{},"Smoke dataset",[535,4328,4329],{},[484,4330,4331],{},"findings\u002Fsmoke_test",[520,4333,4334,4336],{},[535,4335,3515],{},[535,4337,4338],{},"Findings Agent prompt in Prompt Management",[520,4340,4341,4344],{},[535,4342,4343],{},"Gate evaluators",[535,4345,4346,765,4348],{},[484,4347,2840],{},[484,4349,2846],{},[520,4351,4352,4355],{},[535,4353,4354],{},"Experiment type",[535,4356,4357],{},"Prompt Experiment (UI)",[520,4359,4360,4363],{},[535,4361,4362],{},"Example runs",[535,4364,4365],{},[459,4366,4368],{"href":587,"rel":4367},[463],"findings\u002Fgate_test experiments",[520,4370,4371,4374],{},[535,4372,4373],{},"Decision",[535,4375,4376],{},"Compare baseline vs candidate → promote only on gate pass",[455,4378,4379],{},[476,4380,4381],{},"Tips",[470,4383,4384,4392,4398,4401],{},[473,4385,4386,4387,487,4389,4391],{},"Keep experiment names searchable (",[484,4388,3724],{},[484,4390,3728],{}," + prompt version).",[473,4393,4394,4395,4397],{},"Prefer foldered datasets (",[484,4396,592],{},") for clarity in the Datasets UI.",[473,4399,4400],{},"Attach all gate evaluators on every promotion run — do not compare incomplete score sets.",[473,4402,3181,4403,4405,4406,668],{},[484,4404,960],{}," for quick checks; promote prompts using ",[484,4407,956],{},[598,4409],{},[601,4411,4413],{"id":4412},"_4-related-docs","4. Related docs",[567,4415,4416,4420,4427,4433,4439,4444,4449],{},[473,4417,4418],{},[459,4419,401],{"href":402},[473,4421,4422],{},[459,4423,4426],{"href":4424,"rel":4425},"https:\u002F\u002Flangfuse.com\u002Fdocs\u002Fevaluation\u002Foverview",[463],"Langfuse evaluation overview",[473,4428,4429],{},[459,4430,4432],{"href":2934,"rel":4431},[463],"Langfuse LLM-as-a-Judge",[473,4434,4435],{},[459,4436,4438],{"href":2756,"rel":4437},[463],"Langfuse Code Evaluators",[473,4440,4441],{},[459,4442,799],{"href":797,"rel":4443},[463],[473,4445,4446],{},[459,4447,3468],{"href":3466,"rel":4448},[463],[473,4450,4451],{},[459,4452,4455],{"href":4453,"rel":4454},"https:\u002F\u002Flangfuse.com\u002Fdocs\u002Fevaluation\u002Fscores\u002Fdata-model",[463],"Scores data model",[4457,4458,4459],"style",{},"html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .spNyl, html code.shiki .spNyl{--shiki-light:#9C3EDA;--shiki-default:#C792EA;--shiki-dark:#C792EA}html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}",{"title":715,"searchDepth":1066,"depth":1066,"links":4461},[4462,4468,4474,4482],{"id":603,"depth":1066,"text":604,"children":4463},[4464,4465,4466,4467],{"id":656,"depth":1085,"text":657},{"id":706,"depth":1085,"text":707},{"id":773,"depth":1085,"text":774},{"id":802,"depth":1085,"text":803},{"id":2726,"depth":1066,"text":2727,"children":4469},[4470,4471,4472,4473],{"id":2779,"depth":1085,"text":2780},{"id":2859,"depth":1085,"text":2860},{"id":2878,"depth":1085,"text":2879},{"id":2939,"depth":1085,"text":2940},{"id":3436,"depth":1066,"text":3437,"children":4475},[4476,4477,4478,4479,4480,4481],{"id":3477,"depth":1085,"text":3478},{"id":3566,"depth":1085,"text":3567},{"id":3613,"depth":1085,"text":3614},{"id":3696,"depth":1085,"text":3697},{"id":3736,"depth":1085,"text":3737},{"id":3759,"depth":1085,"text":3760},{"id":4412,"depth":1066,"text":4413},"How to evaluate LLM agents with Langfuse — datasets, evaluators, and experiments.","md",null,{},{"title":409,"description":4483},"0EnGvDfgEMon3yvnwyp4Na6BavnAN1Xc_jwDy73oPok",[4490,4492],{"title":405,"path":406,"stem":407,"description":4491,"children":-1},"This document provides a high-level overview, a component breakdown, and detailed diagrams of how ESPi (ESPROFILER Intelligence), the AI co-pilot and agent orchestrator built into ESPROFILER, handles and processes user queries.",{"title":413,"path":414,"stem":415,"description":4493,"children":-1},"How to assess the performance of ESPi across platform changes.",1795106631361]