Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.
Create an evaluation
Click + next to Evaluations in the sidebar, or use the Evaluations dropdown in a prompt’s or workflow’s toolbar to create one directly from what you’re testing.An evaluation is a workflow with two extra features: a dataset and SUT marking.
Build your dataset
Switch to the Dataset tab in the bottom pane. This is a spreadsheet-style editor: add rows and columns, edit cells directly, or import from CSV.Each column can map to an input on your evaluation workflow. When the evaluation runs, each row becomes a separate run with those values as inputs.
Multi-turn testing
For testing conversational prompts and workflows, mark a column as a thread identifier using the column header’s kebab menu. Rows with the same thread value will run against the same conversation thread, simulating a multi-turn interaction.
Mark what you’re testing
On the canvas, select a Prompt or Workflow node and mark it as the System Under Test (look for the beaker icon in the node’s hover toolbar). Token usage, cost, and duration metrics will only count SUT nodes, so your measurements reflect what you’re actually testing rather than the evaluation scaffolding around it.
Add assertions
Wire Assertion nodes into your workflow to define what “good” looks like. Pick an operator, set the expected condition, and assign a weight (0 to 1). The evaluation’s aggregate score is a weighted average of all assertion outcomes.
Run the evaluation
Hit the Run button in the bottom pane. Every dataset row becomes a workflow run. Watch the results appear in the Run tab: inputs, outputs, assertion results, and the aggregate score.Switch between past evaluation batches using the dropdown next to the tabs. Compare how different prompt versions perform against the same dataset.
⌘I
{"docsConfig":{"theme":"maple","$schema":"https://mintlify.com/docs.json","name":"PromptJuggler","description":"PromptJuggler – AI Prompt, Workflow \u0026 Agent Platform","colors":{"primary":"#ff5500","light":"#ff5500","dark":"#8033cc"},"logo":{"light":"https://mintcdn.com/promptjuggler/4dRIri76Lp5nCQVC/logo/promptjuggler_light.svg?fit=max\u0026auto=format\u0026n=4dRIri76Lp5nCQVC\u0026q=85\u0026s=d5d02983378ce48bde8601f4f8cd65a2","dark":"https://mintcdn.com/promptjuggler/4dRIri76Lp5nCQVC/logo/promptjuggler_dark.svg?fit=max\u0026auto=format\u0026n=4dRIri76Lp5nCQVC\u0026q=85\u0026s=3cf6384bd4f9988e3592e2efb58a94d0","href":"https://promptjuggler.com"},"favicon":"https://mintcdn.com/promptjuggler/P5HZRwsUrSR6Wrzv/favicon.svg?fit=max\u0026auto=format\u0026n=P5HZRwsUrSR6Wrzv\u0026q=85\u0026s=56df151d100385b4478595880c38e181","api":{"openapi":"openapi.json","params":{"expanded":"closed"},"playground":{"display":"interactive","proxy":true}},"appearance":{"default":"system"},"navbar":{"primary":{"type":"button","label":"Go to App","href":"https://promptjuggler.com"}},"search":{"prompt":"Search docs…"},"icons":{"library":"lucide"},"contextual":{"options":["copy","claude","chatgpt","perplexity","cursor","vscode","aistudio"],"display":"header"},"metadata":{"timestamp":false},"styling":{"latex":false},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"promptjuggler","hasAudioNarration":false,"actualSubdomain":"promptjuggler","customDomains":["docs.promptjuggler.com"],"gitSource":{"type":"github","owner":"motleyhand","repo":"promptjuggler","deployBranch":"master","contentDirectory":"docs","isPrivate":true,"mountPath":"","label":""},"entitlements":{},"buildId":"6b0051d5ec650d4393974d71","clientVersion":"0.0.4270","searchFilterCounts":[{"language":"default","tag":"API Reference","count":15},{"language":"default","tag":"Core Concepts","count":10},{"language":"default","tag":"Workflow Nodes","count":10},{"language":"default","tag":"Tools","count":8},{"language":"default","tag":"Guides","count":6},{"language":"default","tag":"Knowledge Bases","count":6},{"language":"default","tag":"Backend SDKs","count":5},{"language":"default","tag":"Getting Started","count":4},{"language":"default","tag":"Prompt Runs","count":3},{"language":"default","tag":"Workflow Runs","count":3}]},"assistantConfig":{"enableAskAiButton":false,"enableStarterQuestions":false,"enableContextualStarterQuestions":false,"enableAskAiSearchBar":true,"deflection":{"enabled":true,"email":"hello@promptjuggler.com","showHelpButton":true}},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-rbac-roles","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"rbac-expansion","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"site-roles","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-metadata","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"sso-setup-deadline-banner","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-serve","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"live-agents","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"starter-pricing","enabled":true,"variant":{"name":"control","enabled":true,"payload":{"type":"string","value":"control"},"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"wildcard-email-auth","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"googleFontsInlined":false},"customCss":[]}
{"page":{"pagePath":"guides/setting-up-evaluations","slug":"/guides/setting-up-evaluations","pageMetadata":{"title":"Setting Up Evaluations","description":"Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.","icon":"flask-conical","href":"/guides/setting-up-evaluations"},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsx(_components.p, {\n children: \"Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.\"\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={a:`a`,p:`p`,strong:`strong`,..._provideComponents(),...props.components},{Heading}=_components;if(!Heading)_missingMdxReference(`Heading`,true);return _jsxs(_Fragment,{children:[_jsx(Heading,{level:`2`,id:`create-an-evaluation`,children:`Create an evaluation`}),`\n`,_jsxs(_components.p,{children:[`Click `,_jsx(_components.strong,{children:`+`}),` next to `,_jsx(_components.strong,{children:`Evaluations`}),` in the sidebar, or use the `,_jsx(_components.strong,{children:`Evaluations`}),` dropdown in a prompt’s or workflow’s toolbar to create one directly from what you’re testing.`]}),`\n`,_jsxs(_components.p,{children:[`An evaluation is a `,_jsx(_components.a,{href:`/concepts/workflows`,children:`workflow`}),` with two extra features: a dataset and SUT marking.`]}),`\n`,_jsx(Heading,{level:`2`,id:`build-your-dataset`,children:`Build your dataset`}),`\n`,_jsxs(_components.p,{children:[`Switch to the `,_jsx(_components.strong,{children:`Dataset`}),` tab in the bottom pane. This is a spreadsheet-style editor: add rows and columns, edit cells directly, or import from CSV.`]}),`\n`,_jsx(_components.p,{children:`Each column can map to an input on your evaluation workflow. When the evaluation runs, each row becomes a separate run with those values as inputs.`}),`\n`,_jsx(Heading,{level:`3`,id:`multi-turn-testing`,children:`Multi-turn testing`}),`\n`,_jsxs(_components.p,{children:[`For testing conversational prompts and workflows, mark a column as a `,_jsx(_components.strong,{children:`thread identifier`}),` using the column header’s kebab menu. Rows with the same thread value will run against the same conversation thread, simulating a multi-turn interaction.`]}),`\n`,_jsx(Heading,{level:`2`,id:`mark-what-you’re-testing`,children:`Mark what you’re testing`}),`\n`,_jsxs(_components.p,{children:[`On the canvas, select a Prompt or Workflow node and mark it as the `,_jsx(_components.strong,{children:`System Under Test`}),` (look for the beaker icon in the node’s hover toolbar). Token usage, cost, and duration metrics will only count SUT nodes, so your measurements reflect what you’re actually testing rather than the evaluation scaffolding around it.`]}),`\n`,_jsx(Heading,{level:`2`,id:`add-assertions`,children:`Add assertions`}),`\n`,_jsxs(_components.p,{children:[`Wire `,_jsx(_components.a,{href:`/workflow-nodes/assertion`,children:`Assertion nodes`}),` into your workflow to define what “good” looks like. Pick an operator, set the expected condition, and assign a weight (0 to 1). The evaluation’s aggregate score is a weighted average of all assertion outcomes.`]}),`\n`,_jsx(Heading,{level:`2`,id:`run-the-evaluation`,children:`Run the evaluation`}),`\n`,_jsxs(_components.p,{children:[`Hit the `,_jsx(_components.strong,{children:`Run`}),` button in the bottom pane. Every dataset row becomes a workflow run. Watch the results appear in the `,_jsx(_components.strong,{children:`Run`}),` tab: inputs, outputs, assertion results, and the aggregate score.`]}),`\n`,_jsx(_components.p,{children:`Switch between past evaluation batches using the dropdown next to the tabs. Compare how different prompt versions perform against the same dataset.`})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Setting Up Evaluations","description":"Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.","icon":"flask-conical","href":"/guides/setting-up-evaluations"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[{"title":"Create an evaluation","slug":"create-an-evaluation","depth":2,"children":[]},{"title":"Build your dataset","slug":"build-your-dataset","depth":2,"children":[{"title":"Multi-turn testing","slug":"multi-turn-testing","depth":3,"children":[]}]},{"title":"Mark what you’re testing","slug":"mark-what-you’re-testing","depth":2,"children":[]},{"title":"Add assertions","slug":"add-assertions","depth":2,"children":[]},{"title":"Run the evaluation","slug":"run-the-evaluation","depth":2,"children":[]}],"codeExamples":{}},"jsLinks":["/_mintlify/custom/promptjuggler/cd62668c24c5b1f7/umami.js"],"subdomain":"promptjuggler","siteName":"PromptJuggler","title":"Setting Up Evaluations - PromptJuggler","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"lastModified":"2026-11-12T08:47:43.638Z","originalFileLocation":"guides/setting-up-evaluations.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.\">\u003cmeta data-mint-head name=\"application-name\" content=\"PromptJuggler\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon/android-chrome-192x192.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon/apple-touch-icon.png\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon/favicon-16x16.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon/favicon-32x32.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon/favicon.ico\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon-dark/android-chrome-192x192.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon-dark/favicon-16x16.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon-dark/favicon-32x32.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon-dark/favicon.ico\" media=\"(prefers-color-scheme: dark)\">\u003cmeta data-mint-head property=\"og:title\" content=\"Setting Up Evaluations - PromptJuggler\">\u003cmeta data-mint-head property=\"og:description\" content=\"Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"PromptJuggler\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.promptjuggler.com/guides/setting-up-evaluations\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://promptjuggler.mintlify.site/_mintlify/api/og/v2?division=Guides\u0026amp;title=Setting+Up+Evaluations\u0026amp;description=Create+datasets%2C+wire+assertions%2C+and+measure+how+your+prompts%2C+workflows+and+agents+perform.\u0026amp;theme=1b766d4779f836636abfc877\u0026amp;signature=8d621e9621957b154fe7de8f3c78600200b448500e513706fa75b677c3ca9132\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Setting Up Evaluations - PromptJuggler\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://promptjuggler.mintlify.site/_mintlify/api/og/v2?division=Guides\u0026amp;title=Setting+Up+Evaluations\u0026amp;description=Create+datasets%2C+wire+assertions%2C+and+measure+how+your+prompts%2C+workflows+and+agents+perform.\u0026amp;theme=1b766d4779f836636abfc877\u0026amp;signature=8d621e9621957b154fe7de8f3c78600200b448500e513706fa75b677c3ca9132\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"/mintlify-assets/_mintlify/favicons/promptjuggler/xximN_89kXP6W21J/_generated/favicon/browserconfig.xml\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"PromptJuggler\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#ff5500\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.promptjuggler.com/guides/setting-up-evaluations\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/guides/setting-up-evaluations.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.promptjuggler.com/#organization\",\"name\":\"PromptJuggler\",\"url\":\"https://docs.promptjuggler.com\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintcdn.com/promptjuggler/4dRIri76Lp5nCQVC/logo/promptjuggler_light.svg?fit=max\u0026auto=format\u0026n=4dRIri76Lp5nCQVC\u0026q=85\u0026s=d5d02983378ce48bde8601f4f8cd65a2\"}},{\"@type\":\"WebSite\",\"@id\":\"https://docs.promptjuggler.com#website\",\"name\":\"PromptJuggler\",\"url\":\"https://docs.promptjuggler.com\",\"publisher\":{\"@id\":\"https://docs.promptjuggler.com/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.promptjuggler.com/guides/setting-up-evaluations#webpage\",\"url\":\"https://docs.promptjuggler.com/guides/setting-up-evaluations\",\"name\":\"Setting Up Evaluations\",\"description\":\"Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.\",\"dateModified\":\"2026-11-12T08:47:43.638Z\",\"isPartOf\":{\"@id\":\"https://docs.promptjuggler.com#website\"},\"breadcrumb\":{\"@id\":\"https://docs.promptjuggler.com/guides/setting-up-evaluations#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://docs.promptjuggler.com/guides/setting-up-evaluations#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Guides\",\"item\":\"https://docs.promptjuggler.com/guides/running-prompts\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Setting Up Evaluations\",\"item\":\"https://docs.promptjuggler.com/guides/setting-up-evaluations\"}]},{\"@type\":[\"Article\",\"TechArticle\"],\"@id\":\"https://docs.promptjuggler.com/guides/setting-up-evaluations#article\",\"headline\":\"Setting Up Evaluations\",\"name\":\"Setting Up Evaluations\",\"description\":\"Create datasets, wire assertions, and measure how your prompts, workflows and agents perform.\",\"url\":\"https://docs.promptjuggler.com/guides/setting-up-evaluations\",\"mainEntityOfPage\":{\"@id\":\"https://docs.promptjuggler.com/guides/setting-up-evaluations#webpage\"},\"dateModified\":\"2026-11-12T08:47:43.638Z\",\"publisher\":{\"@id\":\"https://docs.promptjuggler.com/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.promptjuggler.com#website\"}}]}\u003c/script>\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"PromptJuggler\",\"publisher\":{\"@type\":\"Organization\",\"name\":\"PromptJuggler\"},\"provider\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","agentPointerHref":"/guides/setting-up-evaluations.md","siteName":"PromptJuggler","nav":{"global":null,"groups":[{"group":"Getting Started","pages":[{"title":"What is PromptJuggler?","icon":"rocket","href":"/getting-started/introduction"},{"title":"Quickstart","icon":"zap","href":"/getting-started/quickstart"},{"title":"Build with your AI assistant","icon":"sparkles","href":"/getting-started/ai-assistants"},{"title":"Settings","icon":"settings","href":"/getting-started/settings"}]},{"group":"Roadmap","pages":[{"title":"Roadmap","icon":"map","href":"/roadmap"}]},{"group":"Core Concepts","pages":[{"title":"Naming \u0026 Versioning","icon":"git-branch","href":"/concepts/naming-and-versioning"},{"title":"Prompts","icon":"message-square","href":"/concepts/prompts"},{"title":"Workflows","icon":"workflow","href":"/concepts/workflows"},{"title":"Tools","icon":"wrench","href":"/concepts/tools"},{"title":"Knowledge Bases","icon":"database","href":"/concepts/knowledge-bases"},{"title":"Memory \u0026 Threads","icon":"messages-square","href":"/concepts/memory-and-threads"},{"title":"Streaming","icon":"radio","href":"/concepts/streaming"},{"title":"Transcript","icon":"list","href":"/concepts/transcript"},{"title":"Reliability","icon":"shield-check","href":"/concepts/reliability"},{"title":"Evaluations","icon":"flask-conical","href":"/concepts/evaluations"}]},{"group":"Workflow Nodes","pages":[{"title":"Workflow Nodes","icon":"blocks","href":"/workflow-nodes/overview"},{"title":"Data Nodes","icon":"database","href":"/workflow-nodes/data-nodes"},{"title":"Execution Nodes","icon":"play","href":"/workflow-nodes/execution-nodes"},{"title":"Knowledge Search Node","icon":"search","href":"/workflow-nodes/knowledge-search"},{"title":"Script Node","icon":"code","href":"/workflow-nodes/script"},{"title":"Transform Nodes","icon":"shuffle","href":"/workflow-nodes/transform-nodes"},{"title":"Fanout Node","icon":"split","href":"/workflow-nodes/fanout"},{"title":"Control Flow Nodes","icon":"git-branch","href":"/workflow-nodes/control-flow"},{"title":"Loop Node","icon":"repeat","href":"/workflow-nodes/loop"},{"title":"Assertion Node","icon":"circle-check","href":"/workflow-nodes/assertion"}]},{"group":"Tools","pages":[{"title":"Tools","icon":"wrench","href":"/tools/overview"},{"title":"Prompt \u0026 Workflow Tools","icon":"layers","href":"/tools/prompt-and-workflow"},{"title":"Knowledge Search Tool","icon":"search","href":"/tools/knowledge-search"},{"title":"HTTP Call Tool","icon":"globe","href":"/tools/http-call"},{"title":"Script Tool","icon":"code","href":"/tools/script"},{"title":"Web Search Tool","icon":"search","href":"/tools/web-search"},{"title":"MCP Tool","icon":"plug","href":"/tools/mcp"},{"title":"Emit Tool","icon":"share","href":"/tools/emit"}]},{"group":"Guides","pages":[{"title":"Running Prompts","icon":"play","href":"/guides/running-prompts"},{"title":"Building a Chatbot","icon":"bot-message-square","href":"/guides/building-a-chatbot"},{"title":"Building Workflows","description":"From a blank canvas to a working pipeline.","icon":"workflow","href":"/guides/building-workflows"},{"title":"Setting Up Evaluations","icon":"flask-conical","href":"/guides/setting-up-evaluations"},{"title":"Using Knowledge Bases","description":"Upload documents, wait for indexing, and connect them to your prompts and workflows.","icon":"database","href":"/guides/using-knowledge-bases"},{"title":"Integration Guide","icon":"plug","href":"/guides/integration"}]},{"group":"Backend SDKs","pages":[{"title":"Python SDK","icon":"python","href":"/sdks/python/overview"},{"title":"TypeScript SDK","icon":"js","href":"/sdks/typescript/overview"},{"title":"Java SDK","icon":"java","href":"/sdks/java/overview"},{"title":"PHP SDK","icon":"php","href":"/sdks/php/overview"},{"title":"Go SDK","icon":"golang","href":"/sdks/go/overview"}]},{"group":"Browser SDK","pages":[{"title":"Browser SDK","icon":"globe","href":"/sdks/browser/overview"}]},{"group":"API Reference","pages":[{"title":"Overview","icon":"code","href":"/api/overview"},{"group":"Prompts","pages":[{"title":"Get a prompt revision by the prompt’s slug and version","deprecated":null,"version":null,"openapi":"/openapi.json get /api/v1/prompts/{slug}/{version}","href":"/api-reference/prompts/get-a-prompt-revision-by-the-prompt’s-slug-and-version"}]},{"group":"Prompt Runs","pages":[{"title":"Create and trigger a prompt run","deprecated":null,"version":null,"openapi":"/openapi.json post /api/v1/prompts/{slug}/{version}/runs","href":"/api-reference/prompt-runs/create-and-trigger-a-prompt-run"},{"title":"Get a prompt run by ID","deprecated":null,"version":null,"openapi":"/openapi.json get /api/v1/promptruns/{id}","href":"/api-reference/prompt-runs/get-a-prompt-run-by-id"},{"title":"Prompt run finished","version":null,"deprecated":null,"openapi":"/openapi.json webhook promptrun.finished","href":"/api-reference/prompt-runs/prompt-run-finished"}]},{"group":"Knowledge Bases","pages":[{"title":"Get a knowledge base by slug","deprecated":null,"version":null,"openapi":"/openapi.json get /api/v1/knowledge-bases/{slug}","href":"/api-reference/knowledge-bases/get-a-knowledge-base-by-slug"},{"title":"Upload documents to a knowledge base","deprecated":null,"version":null,"openapi":"/openapi.json post /api/v1/knowledge-bases/{slug}/documents","href":"/api-reference/knowledge-bases/upload-documents-to-a-knowledge-base"},{"title":"Get a knowledge document by ID","deprecated":null,"version":null,"openapi":"/openapi.json get /api/v1/knowledge-documents/{id}","href":"/api-reference/knowledge-bases/get-a-knowledge-document-by-id"},{"title":"Delete a knowledge document","deprecated":null,"version":null,"openapi":"/openapi.json delete /api/v1/knowledge-documents/{id}","href":"/api-reference/knowledge-bases/delete-a-knowledge-document"},{"title":"Knowledge document finished","version":null,"deprecated":null,"openapi":"/openapi.json webhook knowledgedocument.finished","href":"/api-reference/knowledge-bases/knowledge-document-finished"},{"title":"Knowledge base finished","version":null,"deprecated":null,"openapi":"/openapi.json webhook knowledgebase.finished","href":"/api-reference/knowledge-bases/knowledge-base-finished"}]},{"group":"Streaming","pages":[{"title":"Create a streaming credential for a thread","deprecated":null,"version":null,"openapi":"/openapi.json post /api/v1/threads/{thread}/stream-token","href":"/api-reference/streaming/create-a-streaming-credential-for-a-thread"}]},{"group":"Workflow Runs","pages":[{"title":"Create and trigger a workflow run","deprecated":null,"version":null,"openapi":"/openapi.json post /api/v1/workflows/{slug}/{version}/runs","href":"/api-reference/workflow-runs/create-and-trigger-a-workflow-run"},{"title":"Get a workflow run by ID","deprecated":null,"version":null,"openapi":"/openapi.json get /api/v1/workflowruns/{id}","href":"/api-reference/workflow-runs/get-a-workflow-run-by-id"},{"title":"Workflow run finished","version":null,"deprecated":null,"openapi":"/openapi.json webhook workflowrun.finished","href":"/api-reference/workflow-runs/workflow-run-finished"}]}]}]},"basePath":"","title":"Setting Up Evaluations - PromptJuggler","deploymentId":"dpl_DyXGnKeCjZyA3sUdfMJ7FjkdBefg","buildId":"dpl_DyXGnKeCjZyA3sUdfMJ7FjkdBefg","dynamics":[],"chunks":[]}