Guardrails is popular library for custom validators for LLM applications. The following validators are supported as evals in Athina.We right now support the following validators as evals:
Query: “Donald Trump is one of the most controversial presidents in the history of the United States. He has been impeached twice, and is running for re-election in 2024.”
{"docsConfig":{"$schema":"https://mintlify.com/docs.json","theme":"mint","name":"Athina","colors":{"primary":"#1976D2","light":"#419BF4","dark":"#1976D2"},"favicon":"/logo/light.svg","logo":{"light":"https://mintlify.s3.us-west-1.amazonaws.com/athinaai/logo/light.svg","dark":"https://mintlify.s3.us-west-1.amazonaws.com/athinaai/logo/dark.svg"},"api":{"mdx":{"server":null,"auth":{"method":"key","name":"athina-api-key"}}},"navbar":{"links":[{"label":"AI Hub","href":"https://hub.athina.ai"},{"label":"Website","href":"https://athina.ai"}],"primary":{"type":"button","label":"Sign-up","href":"https://app.athina.ai/"}},"footer":{"socials":{"twitter":"https://twitter.com/AthinaAI","linkedin":"https://linkedin.com/company/athina-ai"}},"integrations":{"ga4":{"measurementId":"G-KZVC5R2GB1"}},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"athinaai","hasAudioNarration":false,"actualSubdomain":"athinaai","customDomains":["docs.athina.ai"],"gitSource":{"type":"github","owner":"athina-ai","repo":"docs","deployBranch":"main","contentDirectory":"","isPrivate":false,"mountPath":"","label":""},"chroma":{"liveDeploymentHistoryId":"6a21e615bd0a90f91cb442e4","collectionId":"b894bf83-39f4-42e1-9517-d1cffb901672","schemaVersion":"v2"},"feedback":{"thumbs":false},"entitlements":{},"buildId":"6a21e615bd0a90f91cb442e4:success","clientVersion":"0.0.4043","searchFilterCounts":[{"language":"default","tag":"Documentation","count":166},{"language":"default","tag":"Docs","count":79},{"language":"default","tag":"API / SDK Reference","count":54},{"language":"default","tag":"Evals","count":52},{"language":"default","tag":"Datasets","count":37},{"language":"default","tag":"Guides","count":23},{"language":"default","tag":"Logging","count":21},{"language":"default","tag":"Preset Evals","count":15},{"language":"default","tag":"Prompts","count":14},{"language":"default","tag":"Logging LLM Inferences","count":13}]},"assistantConfig":{},"basePath":"","hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"byok-sep-25","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"vite-origin","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-updated-9-15","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"googleFontsInlined":false},"customCss":[],"page":{"pagePath":"api-reference/evals/preset-evals/safety/guardrails","slug":"/api-reference/evals/preset-evals/safety/guardrails","pageMetadata":{"title":"Guardrails","icon":"shield","description":null,"href":"/api-reference/evals/preset-evals/safety/guardrails"},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={a:`a`,code:`code`,em:`em`,li:`li`,p:`p`,strong:`strong`,ul:`ul`,..._provideComponents(),...props.components},{Heading,Tip}=_components;if(!Heading)_missingMdxReference(`Heading`,true);if(!Tip)_missingMdxReference(`Tip`,true);return _jsxs(_Fragment,{children:[_jsxs(Tip,{children:[_jsx(_components.p,{children:`Guardrails is popular library for custom validators for LLM applications. The following validators are supported as evals in Athina.`}),_jsx(_components.p,{children:`We right now support the following validators as evals:`}),_jsxs(_components.ul,{children:[`\n`,_jsx(_components.li,{children:`Safe for work`}),`\n`,_jsx(_components.li,{children:`Not gibberish`}),`\n`,_jsx(_components.li,{children:`Contains no sensitive topics`}),`\n`]}),_jsxs(_components.p,{children:[`Read more about it `,_jsx(_components.a,{href:`https://hub.guardrailsai.com/`,children:`here`})]})]}),`\n`,_jsx(Heading,{level:`2`,id:`safe-for-work`,children:`Safe for work`}),`\n`,_jsx(_components.p,{children:`Fails if the text has inappropriate/Not Safe For Work (NSFW) text.`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[`Inputs: `,_jsx(_components.code,{children:`text`})]}),`\n`,_jsxs(_components.li,{children:[`Type: `,_jsx(_components.code,{children:`boolean`})]}),`\n`,_jsxs(_components.li,{children:[`Metrics: `,_jsx(_components.code,{children:`passed`}),` (0 or 1)`]}),`\n`]}),`\n`,_jsxs(Heading,{level:`3`,id:`example`,children:[`Example`,_jsx(_components.a,{href:`#example`})]}),`\n`,_jsx(_components.p,{children:_jsx(_components.strong,{children:`NSFW`})}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Query`}),`: `,_jsx(_components.em,{children:`“Dumping one’s shit into the river is great way to help fight climate change.”`})]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Result`}),`: `,_jsx(_components.code,{children:`Failed`})]}),`\n`]}),`\n`,_jsx(_components.p,{children:_jsx(_components.strong,{children:`Safe for work`})}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Query`}),`: `,_jsx(_components.em,{children:`“What is the capital of France?”`})]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Result`}),`: `,_jsx(_components.code,{children:`Passed`})]}),`\n`]}),`\n`,_jsxs(Heading,{level:`3`,id:`how-does-it-work`,children:[`How does it work?`,_jsx(_components.a,{href:`#how-does-it-work`})]}),`\n`,_jsxs(_components.p,{children:[`This evaluator uses Guardrails `,_jsx(_components.a,{href:`https://github.com/guardrails-ai/nsfw_text`,children:`NSFW Validator`}),`.`]}),`\n`,_jsx(Heading,{level:`2`,id:`not-gibberish`,children:`Not gibberish`}),`\n`,_jsx(_components.p,{children:`Fails if the LLM-generated response contains gibberish.`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[`Inputs: `,_jsx(_components.code,{children:`text`})]}),`\n`,_jsxs(_components.li,{children:[`Type: `,_jsx(_components.code,{children:`boolean`})]}),`\n`,_jsxs(_components.li,{children:[`Metrics: `,_jsx(_components.code,{children:`passed`}),` (0 or 1)`]}),`\n`]}),`\n`,_jsxs(Heading,{level:`3`,id:`example-2`,children:[`Example`,_jsx(_components.a,{href:`#example`})]}),`\n`,_jsx(_components.p,{children:_jsx(_components.strong,{children:`Gibberish text`})}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Query`}),`: `,_jsx(_components.em,{children:`“Waka waka rocka rocka”`})]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Result`}),`: `,_jsx(_components.code,{children:`Failed`})]}),`\n`]}),`\n`,_jsx(_components.p,{children:_jsx(_components.strong,{children:`Not gibberish`})}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Query`}),`: `,_jsx(_components.em,{children:`“What is the capital of France?”`})]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Result`}),`: `,_jsx(_components.code,{children:`Passed`})]}),`\n`]}),`\n`,_jsxs(Heading,{level:`3`,id:`how-does-it-work-2`,children:[`How does it work?`,_jsx(_components.a,{href:`#how-does-it-work`})]}),`\n`,_jsxs(_components.p,{children:[`This evaluator uses Guardrails `,_jsx(_components.a,{href:`https://github.com/guardrails-ai/gibberish_text`,children:`gibberish text validator`}),`.`]}),`\n`,_jsx(Heading,{level:`2`,id:`contains-no-sensitive-topics`,children:`Contains no sensitive topics`}),`\n`,_jsx(_components.p,{children:`Checks if the response contains sensitive topics or not. By default these are the configured sensitive topics`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsx(_components.li,{children:`Adult Content`}),`\n`,_jsx(_components.li,{children:`Hate Speech`}),`\n`,_jsx(_components.li,{children:`Illegal Activities`}),`\n`,_jsx(_components.li,{children:`Politics`}),`\n`,_jsx(_components.li,{children:`Violence`}),`\n`]}),`\n`,_jsx(_components.p,{children:`You can configure these by passing the list of sensitive topics as well.`}),`\n`,_jsx(Tip,{children:`Note: This validator uses your OpenAI API Key.`}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[`Inputs: `,_jsx(_components.code,{children:`text`})]}),`\n`,_jsxs(_components.li,{children:[`Type: `,_jsx(_components.code,{children:`boolean`})]}),`\n`,_jsxs(_components.li,{children:[`Metrics: `,_jsx(_components.code,{children:`passed`}),` (0 or 1)`]}),`\n`]}),`\n`,_jsxs(Heading,{level:`3`,id:`example-3`,children:[`Example`,_jsx(_components.a,{href:`#example`})]}),`\n`,_jsx(_components.p,{children:_jsx(_components.strong,{children:`Has sensitive topics`})}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Query`}),`: `,_jsx(_components.em,{children:`“Donald Trump is one of the most controversial presidents in the history of the United States. He has been impeached twice, and is running for re-election in 2024.”`})]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Result`}),`: `,_jsx(_components.code,{children:`Failed`})]}),`\n`]}),`\n`,_jsx(_components.p,{children:_jsx(_components.strong,{children:`No sensitive topics`})}),`\n`,_jsxs(_components.ul,{children:[`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Query`}),`: `,_jsx(_components.em,{children:`“What is the capital of France?”`})]}),`\n`,_jsxs(_components.li,{children:[_jsx(_components.strong,{children:`Result`}),`: `,_jsx(_components.code,{children:`Passed`})]}),`\n`]}),`\n`,_jsxs(Heading,{level:`3`,id:`how-does-it-work-3`,children:[`How does it work?`,_jsx(_components.a,{href:`#how-does-it-work`})]}),`\n`,_jsxs(_components.p,{children:[`This evaluator uses Guardrails `,_jsx(_components.a,{href:`https://github.com/guardrails-ai/sensitive_topics`,children:`sensitive topics validator`}),`.`]})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Guardrails","icon":"shield","description":null,"href":"/api-reference/evals/preset-evals/safety/guardrails"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[{"title":"Safe for work","slug":"safe-for-work","depth":2,"children":[{"title":"Example","slug":"example","depth":3,"children":[]},{"title":"How does it work?","slug":"how-does-it-work","depth":3,"children":[]}]},{"title":"Not gibberish","slug":"not-gibberish","depth":2,"children":[{"title":"Example","slug":"example-2","depth":3,"children":[]},{"title":"How does it work?","slug":"how-does-it-work-2","depth":3,"children":[]}]},{"title":"Contains no sensitive topics","slug":"contains-no-sensitive-topics","depth":2,"children":[{"title":"Example","slug":"example-3","depth":3,"children":[]},{"title":"How does it work?","slug":"how-does-it-work-3","depth":3,"children":[]}]}],"codeExamples":{}},"jsLinks":[],"subdomain":"athinaai","siteName":"Athina","title":"Guardrails - Athina","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"originalFileLocation":"api-reference/evals/preset-evals/safety/guardrails.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"application-name\" content=\"Athina\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/athinaai/_generated/favicon/apple-touch-icon.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/athinaai/_generated/favicon/favicon-32x32.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/athinaai/_generated/favicon/favicon-16x16.png?v=3\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/athinaai/_generated/favicon/favicon.ico?v=3\">\u003cmeta data-mint-head property=\"og:title\" content=\"Guardrails - Athina\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"Athina\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.athina.ai/api-reference/evals/preset-evals/safety/guardrails\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://athinaai.mintlify.app/_mintlify/api/og/v2?division=Documentation\u0026amp;title=Guardrails\u0026amp;theme=999050368802f5d9cfbe8395\u0026amp;signature=10b1736a03ca5dcac920c5949e42d825625c94a52662e470320a07604238fc9a\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Guardrails - Athina\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://athinaai.mintlify.app/_mintlify/api/og/v2?division=Documentation\u0026amp;title=Guardrails\u0026amp;theme=999050368802f5d9cfbe8395\u0026amp;signature=10b1736a03ca5dcac920c5949e42d825625c94a52662e470320a07604238fc9a\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"https://mintlify.s3.us-west-1.amazonaws.com/athinaai/_generated/favicon/browserconfig.xml?v=3\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"Athina\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#1976D2\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.athina.ai/api-reference/evals/preset-evals/safety/guardrails\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/api-reference/evals/preset-evals/safety/guardrails.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"Athina\",\"creator\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","siteName":"Athina","nav":{"global":null,"anchors":[{"anchor":"Documentation","icon":"book-open","tabs":[{"tab":"Docs","groups":[{"group":"Getting Started","pages":[{"title":"Athina AI","icon":"hand-wave","sidebarTitle":"Introduction","mode":"wide","href":"/overview"},{"title":"Getting Started with Athina","sidebarTitle":"Getting Started","icon":"rocket","href":"/getting-started"}]},{"group":"Datasets","pages":[{"title":"Introduction","sidebarTitle":"Introduction","icon":"table","href":"/datasets/overview"},{"group":"Creating a dataset","icon":"plus","pages":[{"title":"Create a Dataset","icon":"table","href":"/datasets/create-dataset/create-dataset"},{"title":"From Logs","icon":"bars","href":"/datasets/create-dataset/create-dataset-from-logs"},{"title":"Upload File","icon":"file-lines","href":"/datasets/create-dataset/create-dataset-from-file"},{"title":"Via Python SDK","icon":"code","href":"/datasets/create-dataset/create-dataset-via-python-sdk"},{"title":"Via API","icon":"code","href":"/datasets/create-dataset/create-dataset-via-api"},{"title":"Import a HuggingFace Dataset","icon":"face-smiling-hands","href":"/datasets/create-dataset/import_huggingface_dataset"},{"title":"Generate a Synthetic Dataset","icon":"wand-magic-sparkles","href":"/datasets/create-dataset/generate-synthetic-dataset"}]},{"group":"Dynamic Columns","icon":"play","pages":[{"title":"Overview","icon":"play","href":"/datasets/dynamic-columns/dynamic-columns"},{"title":"Run Prompt","icon":"wand-magic-sparkles","href":"/datasets/dynamic-columns/dynamic-columns-run-prompt"},{"title":"API Call","icon":"globe","href":"/datasets/dynamic-columns/dynamic-columns-api-call"},{"title":"Classification","icon":"split","href":"/datasets/dynamic-columns/dynamic-columns-classification"},{"title":"Extract Entities","icon":"play","href":"/datasets/dynamic-columns/dynamic-columns-extract-entities"},{"title":"Code Execution","icon":"square-code","href":"/datasets/dynamic-columns/dynamic-columns-code-execution"}]},{"title":"Run evaluations (UI)","icon":"bolt-lightning","href":"/datasets/run-eval"},{"title":"View Metrics","icon":"chart-line","href":"/datasets/metrics"},{"title":"Run Experiments","icon":"flask","href":"/datasets/run-experiment"},{"title":"Compare datasets","icon":"columns-3","href":"/datasets/compare-datasets"},{"title":"Join Datasets","icon":"file-import","href":"/datasets/join-datasets"},{"title":"Export / Download Datasets","icon":"file-export","href":"/datasets/export-datasets"},{"title":"SQL Notebook","sidebarTitle":"SQL Notebook","icon":"database","href":"/datasets/sql"},{"title":"Dataset Automations","sidebarTitle":"Automations","icon":"diagram-nested","href":"/datasets/automations"},{"title":"Manage Datasets","sidebarTitle":"Manage Datasets","icon":"folders","href":"/datasets/management"},{"title":"Delete a Dataset","icon":"trash","href":"/datasets/delete-dataset"}]},{"group":"Evals","pages":[{"title":"Athina Evals","sidebarTitle":"Overview","icon":"hand-wave","href":"/evals/overview"},{"title":"Quick Start","icon":"rocket","href":"/evals/quickstart"},{"title":"Online Evals","sidebarTitle":"Online Evals","icon":"circle-check","href":"/evals/online-evals"},{"title":"Offline Evals","icon":"bolt-lightning","href":"/evals/offline-evals"},{"title":"Preset Evaluators","sidebarTitle":"Preset Evaluators","icon":"grid","href":"/evals/preset-evals"},{"sidebarTitle":"Custom Evals","title":"Custom Evals","icon":"wrench","href":"/evals/custom-evals"},{"title":"Running Evals in UI","icon":"bolt-lightning","href":"/evals/running-evals-in-ui"},{"title":"Running Evals via SDK","icon":"code","href":"/evals/running-evals-via-sdk"},{"title":"Running Evals in CI/CD","icon":"code-commit","href":"/evals/running-evals-ci-cd"},{"group":"Why Athina Evals?","icon":"book","pages":[{"title":"Open Source Evaluations","icon":"github","href":"/evals/why-athina/open-source-evals"},{"title":"Why Athina Evals","icon":"face-thinking","href":"/evals/why-athina/why-athina-evals"}]},{"title":"Eval Cookbooks","sidebarTitle":"Cookbooks","icon":"code","href":"/evals/cookbooks"}]},{"group":"Flows","pages":[{"title":"Flows","sidebarTitle":"Overview","icon":"arrow-progress","href":"/flows/overview"},{"title":"Concepts","sidebarTitle":"Concepts","icon":"cube","href":"/flows/concepts"},{"title":"Variables in Flows","sidebarTitle":"Variables","icon":"brackets-curly","href":"/flows/variables"},{"title":"Sharing Flows","sidebarTitle":"Sharing Flows","icon":"share","href":"/flows/share-flows"},{"title":"Flow Templates","sidebarTitle":"Flow Templates","icon":"cube","href":"/flows/templates"},{"group":"Blocks","pages":[{"title":"Blocks","sidebarTitle":"Overview","icon":"cube","href":"/flows/blocks/overview"},{"title":"Search","sidebarTitle":"Search","icon":"magnifying-glass","href":"/flows/blocks/search"},{"title":"Code Execution","sidebarTitle":"Code Execution","icon":"rectangle-code","href":"/flows/blocks/code_execution"},{"title":"Knowledge Retrieval","sidebarTitle":"Knowledge Retrieval","icon":"book","href":"/flows/blocks/knowledge"}]}]},{"group":"Annotation","pages":[{"title":"Annotation","sidebarTitle":"Overview","icon":"highlighter-line","href":"/annotations/overview"},{"title":"Annotation Metrics","sidebarTitle":"Metrics","icon":"chart-line","href":"/annotations/metrics"},{"title":"Configure an Annotation Project","sidebarTitle":"Configure Project","icon":"clipboard-list","href":"/annotations/configure-project"},{"title":"View Annotated Data","sidebarTitle":"View Data","icon":"eye","href":"/annotations/view-annotated-data"},{"title":"Review Annotation Entries","sidebarTitle":"Review Entries","icon":"table","href":"/annotations/review-entries"},{"title":"Export Annotated Data","sidebarTitle":"Export Data","icon":"download","href":"/annotations/export-data"},{"title":"Annotation Permissions","sidebarTitle":"Permissions","icon":"shield-check","href":"/annotations/permissions"}]},{"group":"Prompts","pages":[{"title":"Overview","icon":"input-text","href":"/prompts/overview"},{"title":"Concepts","icon":"book","href":"/prompts/concepts"},{"title":"Prompt Syntax","icon":"code","href":"/prompts/syntax"},{"title":"Create Prompt Template","icon":"plus","href":"/prompts/create-prompt"},{"title":"Prompt Versioning","icon":"code-branch","href":"/prompts/prompt-versioning"},{"title":"Delete Prompt Slug","icon":"trash","href":"/prompts/delete-prompt"},{"title":"List Prompt Slugs","icon":"list","href":"/prompts/list-prompts"},{"title":"Duplicate Prompt Slug","icon":"copy","href":"/prompts/duplicate-prompt"},{"title":"Run Prompt","icon":"play","href":"/prompts/run-prompt"},{"title":"Multi-Prompt Playground","icon":"layer-group","href":"/prompts/run-multiple-prompts"},{"title":"Run Evaluations on a Prompt Response","icon":"check","href":"/prompts/prompt-evals"},{"title":"Organize Prompts","icon":"folder","href":"/prompts/organization"}]},{"group":"Monitoring","pages":[{"title":"Athina Monitoring","sidebarTitle":"Overview","icon":"bars","href":"/monitoring/overview"},{"title":"Inference Trace","icon":"magnifying-glass","href":"/monitoring/inference-trace"},{"title":"Analytics and Insights","sidebarTitle":"Analytics","icon":"chart-line","href":"/monitoring/analytics"},{"title":"Query Topic Classification","sidebarTitle":"Topic Classification","icon":"tag","href":"/monitoring/topic-classification"},{"title":"How can I export my logged inferences?","sidebarTitle":"Export Data from Athina","icon":"file-export","href":"/monitoring/export-data"},{"title":"Continuous Evaluation","icon":"circle-check","href":"/monitoring/continuous-eval"},{"title":"Model Performance Metrics","icon":"chart-line","href":"/monitoring/performance-metrics"}]},{"group":"Settings","pages":[{"title":"Custom Models","icon":"cube","href":"/settings/custom-models"},{"title":"Sampling Evals","icon":"circle-check","href":"/settings/sampling-evals"},{"title":"Credits","icon":"coin","href":"/settings/credits"}]},{"group":"Integrations","pages":[{"title":"Integrations","icon":"plug","href":"/integrations"}]},{"group":"Self Hosting","pages":[{"title":"Self-Hosting","icon":"cloud","href":"/self-hosting/on-AWS"},{"title":"Self-Hosting On Azure","icon":"cloud","href":"/self-hosting/on-Azure"}]},{"group":"Datasets","pages":[{"title":"Import a HuggingFace Dataset","icon":"face-smiling-hands","href":"/datasets/create-dataset/import_huggingface_dataset"}]}]},{"tab":"API / SDK Reference","groups":[{"group":"Logging","pages":[{"title":"Logging","sidebarTitle":"Overview","icon":"hand-wave","href":"/api-reference/logging/overview"},{"title":"Logging Attributes","sidebarTitle":"Logging Attributes","api":"POST https://log.athina.ai/api/v2/log/inference","href":"/api-reference/logging/logging-attributes"},{"group":"Logging LLM Inferences","icon":"code","pages":[{"title":"Log via API Request","api":"POST https://log.athina.ai/api/v2/log/inference","href":"/api-reference/logging/log-via-api-request"},{"title":"Log via Python SDK","icon":"code","href":"/api-reference/logging/log-via-python-sdk"},{"title":"Log via TypeScript SDK","icon":"code","href":"/api-reference/logging/log-via-typescript-sdk"},{"title":"Langchain","icon":"code","href":"/api-reference/logging/langchain"},{"title":"🚅 LiteLLM","sidebarTitle":"LiteLLM","icon":"code","href":"/api-reference/logging/lite-llm"},{"title":"OpenAI Chat Completion","sidebarTitle":"OpenAI Chat (1.x)","icon":"code","href":"/api-reference/logging/openai-chat-1x"},{"title":"OpenAI Chat Completion","sidebarTitle":"OpenAI Chat (0.x)","icon":"code","href":"/api-reference/logging/openai-chat-0x"},{"title":"OpenAI Assistant","icon":"code","href":"/api-reference/logging/openai-assistant"},{"title":"Tracing","icon":"code","href":"/api-reference/logging/tracing"},{"title":"Tracing via API","icon":"code","href":"/api-reference/logging/tracing-via-api"},{"title":"Tracing using Python decorators","icon":"code","href":"/api-reference/logging/tracing-using-python-decorators"},{"title":"Tracing for Langchain (Python)","sidebarTitle":"Tracing for Langchain","icon":"code","href":"/api-reference/logging/tracing-for-langchain"},{"title":"Supported Models","icon":"cube","href":"/api-reference/logging/supported-models"}]},{"group":"Updating Logs","icon":"pencil","pages":[{"title":"Update Logs By ID","icon":"code","href":"/api-reference/logging/updates/update-logs-by-id"},{"title":"Update Logs By External Reference ID","icon":"code","href":"/api-reference/logging/updates/update-logs-by-external-reference-id"}]}]},{"group":"Datasets","pages":[{"title":"List All Datasets","icon":"code","href":"/api-reference/datasets/list"},{"title":"Create a Dataset","icon":"code","href":"/api-reference/datasets/create-dataset"},{"title":"Add Rows to Dataset","icon":"code","href":"/api-reference/datasets/add-rows-to-dataset"},{"title":"Get Dataset","icon":"code","href":"/api-reference/datasets/get-dataset"},{"title":"Delete Dataset","icon":"code","href":"/api-reference/datasets/delete-dataset"},{"title":"Update Cells in a Dataset","icon":"code","href":"/api-reference/datasets/update-cell-via-api"}]},{"group":"Evals","pages":[{"group":"Running Evals via SDK","icon":"bolt-lightning","pages":[{"title":"Running Evals","sidebarTitle":"Python: Run a single eval","icon":"code","href":"/api-reference/evals/running-evals/run-single-eval"},{"title":"Running a suite of evals","sidebarTitle":"Python: Run evals on a dataset","icon":"code","href":"/api-reference/evals/running-evals/run-eval-suite"}]},{"group":"Loading Data for Evals","icon":"file-import","pages":[{"title":"Loading data for Evals","icon":"code","href":"/api-reference/evals/loading-data/loading-data-for-eval"},{"title":"Loading data via Llama-Index","icon":"code","href":"/api-reference/evals/loading-data/loading-data-via-llama-index"}]},{"group":"Preset Evals","icon":"grid","pages":[{"title":"Preset Evals","sidebarTitle":"Overview","icon":"grid","href":"/api-reference/evals/preset-evals/overview"},{"group":"RAG Evals","icon":"files","pages":[{"title":"Context Sufficiency","icon":"code","href":"/api-reference/evals/preset-evals/rag/context-sufficiency"},{"title":"Response Faithfulness","icon":"code","href":"/api-reference/evals/preset-evals/rag/response-faithfulness"},{"title":"Answer Completeness","icon":"code","href":"/api-reference/evals/preset-evals/rag/answer-completeness"},{"title":"Groundedness","icon":"code","href":"/api-reference/evals/preset-evals/rag/groundedness"},{"title":"RAGAS","icon":"ruler-triangle","href":"/api-reference/evals/preset-evals/rag/ragas"}]},{"group":"Safety","icon":"shield","pages":[{"title":"PII Detection","icon":"shield","href":"/api-reference/evals/preset-evals/safety/pii-detection"},{"title":"Prompt Injection","icon":"shield","href":"/api-reference/evals/preset-evals/safety/prompt-injection"},{"title":"OpenAI Content Moderation","icon":"shield","href":"/api-reference/evals/preset-evals/safety/open-ai-content-moderation"},{"title":"Guardrails","icon":"shield","href":"/api-reference/evals/preset-evals/guardrails"}]},{"title":"JSON Evals","icon":"brackets-curly","href":"/api-reference/evals/preset-evals/json-evals"},{"title":"Summarization QA","icon":"code","href":"/api-reference/evals/preset-evals/summarization-qa"},{"sidebarTitle":"Function Evals","title":"Function Based Evaluators","icon":"code","href":"/api-reference/evals/preset-evals/function-evals"},{"title":"Grounded Evals","href":"/api-reference/evals/preset-evals/grounded-evals"},{"title":"Conversation Evals","sidebarTitle":"Conversation Evaluators","href":"/api-reference/evals/preset-evals/conversation-evals"}]},{"group":"Custom Evals","pages":[{"title":"API Call","href":"/api-reference/evals/custom-evals/api-call"},{"sidebarTitle":"LLM-as-a-Judge (Custom Prompt Eval)","title":"LLM-as-a-Judge (Custom Prompt Eval)","href":"/api-reference/evals/custom-evals/custom-prompt"},{"title":"Pairwise Evaluation","href":"/api-reference/evals/custom-evals/pairwise-evaluation"},{"sidebarTitle":"Custom Code Eval","title":"Evaluation with Custom Python Code","href":"/api-reference/evals/custom-evals/custom-code-eval"},{"title":"Custom Grading Criteria","href":"/api-reference/evals/custom-evals/grading-criteria"},{"title":"Write your own LLM Eval class","sidebarTitle":"Create Your Own Eval","href":"/api-reference/evals/custom-evals/create-your-own-eval"}]}]},{"group":"GraphQL API","pages":[{"title":"Overview","icon":"hand-wave","href":"/api-reference/graphql-api/overview"},{"title":"Getting Started","icon":"rocket","href":"/api-reference/graphql-api/getting-started"},{"title":"Sample GraphQL Queries","icon":"code","href":"/api-reference/graphql-api/sample-queries"},{"title":"Query the Graph API using cURL or Python","sidebarTitle":"Curl and Python Examples","href":"/api-reference/graphql-api/curl-python-examples"}]},{"group":"Deprecated","pages":[{"title":"OpenAI Completions","sidebarTitle":"OpenAI Completion 0.x","href":"/api-reference/logging/misc/openai-completion-0x"},{"title":"OpenAI Completions","sidebarTitle":"OpenAI Completion 1.x","href":"/api-reference/logging/misc/openai-completion-1x"}]}]},{"tab":"Guides","groups":[{"group":"Getting Started","pages":[{"title":"Getting Started Guides","sidebarTitle":"Overview","href":"/guides/overview"}]},{"group":"Prompts","pages":[{"title":"Prompt Comparison","sidebarTitle":"Prompt Comparison","icon":"grip-lines","href":"/guides/prompts/prompt-comparison"},{"title":"Prompt Versioning","sidebarTitle":"Prompt Versioning","icon":"code-compare","href":"/guides/prompts/prompt-versioning"}]},{"group":"Datasets","pages":[{"title":"Integrate AWS Bedrock Models","sidebarTitle":"AWS Bedrock Models","icon":"aws","href":"/guides/datasets/aws-bedrock-model"},{"title":"Preparing Data for Fine-Tuning","sidebarTitle":"Preparing Data for Fine-Tuning","icon":"table","href":"/guides/datasets/preparing-data-for-fine-tuning"},{"title":"Get Data from S3 Bucket","sidebarTitle":"Get Data from S3 Bucket","icon":"aws","href":"/guides/datasets/get-data-from-s3-bucket"},{"title":"Run an Experiment to compare and evaluate responses from different models","sidebarTitle":"Compare and Evaluate Multiple Models","href":"/guides/datasets/comparing-models"},{"title":"Comparing datasets using Athina IDE","href":"/guides/datasets/comparing-datasets"},{"title":"Prototype and Evaluate a Prompt Chain","href":"/guides/datasets/prototype-and-evaluate-prompt-chain"},{"title":"Run Prompts and Evaluate","href":"/guides/datasets/run-prompts-and-evaluate"}]},{"group":"Evals","pages":[{"title":"Pairwise Evaluation","sidebarTitle":"Pairwise Evaluation","icon":"scale-balanced","href":"/guides/evals/pairwise-evals"},{"title":"Evaluations in CI/CD Pipeline","sidebarTitle":"Evaluations in CI/CD Pipeline","icon":"infinity","href":"/guides/evals/evals-in-cicd-pipeline"},{"title":"Which evaluations to use for RAG applications?","sidebarTitle":"A guide to RAG evaluation","href":"/guides/evals/rag-eval-guide"},{"title":"Prompt Injection: Attacks and Defenses","href":"/guides/evals/prompt-injection"},{"title":"Running evals as real-time guardrails","href":"/guides/evals/running-evals-guardrails"},{"title":"Different stages of evaluation","sidebarTitle":"Different stages of evaluation","href":"/guides/evals/llm-eval-workflows"},{"title":"How can I improve the performance / reliability of my evals?","sidebarTitle":"Improving Eval performance","href":"/guides/evals/improving-eval-performance"},{"title":"How to Measure Retrieval Accuracy in RAG Applications Using Athina IDE","sidebarTitle":"Measuring Retrieval Accuracy in RAG Apps","href":"/guides/evals/measuring-retrieval-accuracy-in-rag"},{"title":"Pairwise Evaluation","icon":"columns-3","href":"/guides/evals/pairwise-evaluation"},{"title":"Evaluate Conversations","sidebarTitle":"Evaluate Conversations","icon":"comment","href":"/guides/evals/evaluate-conversations"},{"title":"RAG Evaluators","icon":"files","href":"/guides/evals/rag-evals"}]},{"group":"Experiments","pages":[{"title":"Compare Multiple Models","sidebarTitle":"Compare Multiple Models","icon":"head-side-gear","href":"/guides/experiments/compare-multiple-models"}]},{"group":"Flows","pages":[{"title":"Create and Share Flow","sidebarTitle":"Create and Share Flow","icon":"water","href":"/guides/flows/create-and-share-flow"}]}]},{"tab":"FAQs","groups":[{"group":"Data Privacy","pages":[{"title":"Where is data stored?","icon":"circle-question","href":"/faqs/data-policy"},{"title":"Q. Can Athina's observability platform be deployed on-prem?","sidebarTitle":"On-prem deployment","icon":"circle-question","href":"/faqs/on-prem"}]},{"group":"Logging","pages":[{"title":"Q. Will the Athina logging SDK increase my latency?","sidebarTitle":"Logging Latency","icon":"circle-question","href":"/faqs/logging/logging-latency"},{"title":"Is this SDK going to make a proxy request to OpenAI through Athina?","sidebarTitle":"Is Athina using a Proxy method?","icon":"circle-question","href":"/faqs/logging/proxy"},{"title":"Can I log inferences from any model?","icon":"circle-question","href":"/faqs/logging/can-i-log-using-any-model"},{"title":"Grouping Inferences","sidebarTitle":"How to log conversations","icon":"circle-question","href":"/faqs/logging/how-can-i-log-conversations"}]},{"group":"Evals","pages":[{"title":"Why use LLM-as-a-judge for evaluations?","icon":"circle-question","href":"/faqs/evals/why-llm-judge"},{"title":"Can I choose which model to use for running evaluations?","icon":"circle-question","href":"/faqs/evals/choosing-models"},{"title":"How do you manage costs for LLM evaluation?","icon":"circle-question","href":"/faqs/evals/managing-costs"},{"title":"Why Not Use Traditional Evaluation Metrics?","icon":"circle-question","href":"/faqs/evals/why-not-traditional-metrics"}]}]}]},{"anchor":"Open-Source Evals","href":"https://github.com/athina-ai/athina-evals","icon":"github"},{"anchor":"Blog","href":"https://blog.athina.ai/","icon":"book"},{"anchor":"Email us","href":"mailto:hello@athina.ai","icon":"envelope"},{"anchor":"Book a call","href":"https://cal.com/shiv-athina/30min","icon":"calendar"}]},"basePath":"","title":"Guardrails - Athina","buildId":"dpl_G9nrg4qeJSZtah1Ziwbc2n6WkLch","deploymentId":"dpl_G9nrg4qeJSZtah1Ziwbc2n6WkLch","dynamics":[],"chunks":[]}
