[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"portal-settings:stajic:de":3,"public-menus:all":37,"post:mlops-vs-llmops-what-changes-when-the-model-is-an-llm:de":204,"related:post:mlops-vs-llmops-what-changes-when-the-model-is-an-llm:de:1":3394},{"statusCode":4,"data":5,"message":36},200,{"tenantId":6,"lang":7,"defaultLang":7,"siteUrl":8,"contactEmail":9,"brandName":10,"logoUrl":11,"siteName":10,"siteDescription":12,"ogImage":9,"robotsIndex":13,"socialLinks":9,"reservedSlugs":9,"seoPolicy":14},"stajic","de","https:\u002F\u002Fstajic.de",null,"Stajic Platform","\u002FLogo_Planet.svg","Stajic Portal",true,{"branding":15,"relatedContent":16,"crossDomainLinks":17},{"logoUrl":11},{"enabled":13},[18,21,24,27,30,33],{"url":19,"label":20,"isActive":13,"showInFooter":13,"includeInSameAs":13},"https:\u002F\u002Ffigure.rocks","figure.rocks",{"url":22,"label":23,"isActive":13,"showInFooter":13,"includeInSameAs":13},"https:\u002F\u002Floving.rocks","loving.rocks",{"url":25,"label":26,"isActive":13,"showInFooter":13,"includeInSameAs":13},"https:\u002F\u002Fbazify.com","bazify.com",{"url":28,"label":29,"isActive":13,"showInFooter":13,"includeInSameAs":13},"https:\u002F\u002Fbazify.de","bazify.de",{"url":31,"label":32,"isActive":13,"showInFooter":13,"includeInSameAs":13},"https:\u002F\u002Fbazify.at","bazify.at",{"url":34,"label":35,"isActive":13,"showInFooter":13,"includeInSameAs":13},"https:\u002F\u002Fbazify.ba","bazify.ba","Portal settings resolved",[38,44],{"id":39,"name":40,"location":41,"isActive":13,"isDefault":42,"items":43},1,"main-navigation","header",false,[],{"id":45,"name":46,"location":47,"isActive":13,"isDefault":13,"items":48},4,"main-menu","sidebar",[49,65,78,92,102,117,132],{"id":50,"title":51,"url":59,"target":60,"icon":61,"isActive":13,"type":62,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":63,"portfolioId":9,"children":64},"item-18",{"de":52,"en":53,"es":54,"fr":55,"it":53,"ru":56,"sr":57,"zh":58},"Startseite","Home","Inicio","Accueil","Главная","Почетна","首页","\u002Ffull-stack-web-developer-munich-performance-seo-and-maintainable-builds","_self","i-lucide-home","page",111,[],{"id":66,"title":67,"url":74,"target":60,"icon":75,"isActive":13,"type":62,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":76,"portfolioId":9,"children":77},"item-22",{"de":68,"en":68,"es":69,"fr":68,"it":70,"ru":71,"sr":72,"zh":73},"Vision","Visión","Visione","Видение","Визија","想象","\u002Fueber-uns-webdesign-muenchen-webaplikation","i-lucide-eye",113,[],{"id":79,"title":80,"url":88,"target":60,"icon":89,"isActive":13,"type":62,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":90,"portfolioId":9,"children":91},"item-19",{"de":81,"en":82,"es":83,"fr":82,"it":84,"ru":85,"sr":86,"zh":87},"Leistungen","Services","Servicios","Servizi","Услуги","Услуге","服务","\u002Fservices-dienstleistungen-muenchen","i-lucide-wrench",116,[],{"id":93,"title":94,"url":98,"target":60,"icon":99,"isActive":13,"type":62,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":100,"portfolioId":9,"children":101},"item-23",{"de":95,"en":95,"es":95,"fr":95,"it":95,"ru":96,"sr":96,"zh":97},"Blog","Блог","博客","\u002Fblog","i-lucide-book-open",112,[],{"id":103,"title":104,"url":113,"target":60,"icon":114,"isActive":13,"type":62,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":115,"portfolioId":9,"children":116},"item-32",{"de":105,"en":106,"es":107,"fr":108,"it":109,"ru":110,"sr":111,"zh":112},"Neue Technologien","New Technologies","Nuevas tecnologías","Nouvelles technologies","Nuove tecnologie","Новые технологии","Нове технологије","新技术！","\u002Fneue-webtechnologien","i-lucide-sparkles",122,[],{"id":118,"title":119,"url":128,"target":60,"icon":129,"isActive":13,"type":62,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":130,"portfolioId":9,"children":131},"item-20",{"de":120,"en":121,"es":122,"fr":123,"it":124,"ru":125,"sr":126,"zh":127},"Kontakt","Contact us!","Contacto","Contact","Contatto","Контакт","Контактирајте нас","联系我们！","\u002Fcontact","i-lucide-mail",115,[],{"id":133,"title":134,"url":143,"target":60,"icon":144,"isActive":13,"type":62,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":145,"portfolioId":9,"children":146},"item-21",{"de":135,"en":136,"es":137,"fr":138,"it":139,"ru":140,"sr":141,"zh":142},"Unsere Arbeit","Our Work","Nuestro trabajo","Nos réalisations","I nostri lavori","Наши работы","Наши радови","文件夹","\u002Fportfolio","i-lucide-briefcase",114,[147,160,174,180,192],{"id":148,"title":149,"url":143,"target":60,"icon":158,"isActive":13,"type":62,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":145,"portfolioId":9,"children":159},"item-24",{"de":150,"en":151,"es":152,"fr":153,"it":154,"ru":155,"sr":156,"zh":157},"Alle Projekte","All Projects","Todos los proyectos","Tous les projets","Tutti i progetti","Все проекты","Сви пројекти","所有项目","i-lucide-grid-3x3",[],{"id":161,"title":162,"url":170,"target":60,"icon":171,"isActive":13,"type":172,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":9,"portfolioId":9,"children":173},"item-29",{"de":163,"en":164,"es":165,"fr":166,"it":167,"ru":168,"sr":169,"zh":142},"Local Roots, Global Reach","Local Roots - Global Reach","Empresa local ","Entreprise locale","Azienda locale","Местная компания","Локално предузеће глобално тржиште","\u002Fportfolio\u002Flocal-roots-global-reach-communication-media-systems-for-modern-business","i-lucide-folder","custom",[],{"id":175,"title":176,"url":178,"target":60,"icon":171,"isActive":13,"type":172,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":9,"portfolioId":9,"children":179},"item-28",{"de":177,"en":177,"es":177,"fr":177,"it":177,"ru":177,"sr":177,"zh":177},"Solr Suggester","\u002Fportfolio\u002Fsolr-fuzzy-suggester-und-solr-infix-suggester-abfrage-ueber-ajax-und-filterung",[],{"id":181,"title":182,"url":190,"target":60,"icon":171,"isActive":13,"type":172,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":9,"portfolioId":9,"children":191},"item-27",{"de":183,"en":184,"es":185,"fr":186,"it":187,"ru":188,"sr":189,"zh":184},"Firmenwebseite SEO","Company Website SEO","Sitio web corporativo SEO","Site web d’entreprise SEO","Sito web aziendale SEO","Корпоративный сайт SEO","Пословна веб-страница SEO","\u002Fportfolio\u002Fseo-sem-branding-mobile-webseite-muenchen",[],{"id":193,"title":194,"url":202,"target":60,"icon":171,"isActive":13,"type":172,"productId":9,"categoryId":9,"shopCategoryId":9,"articleId":9,"pageId":9,"portfolioId":9,"children":203},"item-31",{"de":195,"en":196,"es":197,"fr":198,"it":199,"ru":200,"sr":201,"zh":196},"Digitalisierungsportal","Digitalization Portal","Portal de digitalización","Portail de numérisation","Portale di digitalizzazione","Портал цифровизации","Портал за дигитализацију","\u002Fportfolio\u002Fdigitalisierungsportal-archiv-museum-bibliothek-ead-lido-mets-mods",[],{"statusCode":4,"data":205,"message":3393},{"id":206,"title":207,"slug":208,"content":209,"contentJson":210,"excerpt":1558,"featuredImage":1559,"featuredImageAlt":1560,"featuredImageCaption":9,"featuredImageTitle":9,"featuredImageCopyright":9,"featuredImageAuthor":9,"featuredImageSourceUrl":9,"featuredImageLicense":9,"featuredImageIsAiGenerated":42,"status":1561,"publishedAt":1562,"createdAt":1563,"updatedAt":1564,"seoLocalePaths":1565,"categories":1574,"author":1591,"translations":1596},"493","MLOps vs. LLMOps: Was sich ändert, wenn das Modell ein LLM ist","mlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u003Cp>MLOps ist die Ingenieursdisziplin für die zuverlässige Entwicklung, Bereitstellung, Versionierung und den Betrieb von Machine-Learning-Systemen; LLMOps erweitert diese Disziplin auf Anwendungen, die auf großen Sprachmodellen basieren, bei denen das Produktionsverhalten nicht nur von einem Modellartefakt abhängt, sondern auch von Prompts, Kontext, Retrieval, Anbieter-\u002FModellversionen, Tool-Aufrufen, Sicherheitskontrollen und Evaluierungspipelines. LLMOps ersetzt MLOps nicht. Es verändert die operative Einheit von „einem Modell plus Serving-Pipeline“ hin zu „einer sich entwickelnden LLM-Anwendung, deren Verhalten aus mehreren unabhängig voneinander veränderlichen Komponenten entsteht“.\u003C\u002Fp>\n\u003Caside class=\"editorjs-callout editorjs-callout--info my-6 rounded-xl border p-5 border-blue-300 bg-blue-50 dark:border-blue-900 dark:bg-blue-950\u002F20\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Direkte Antwort\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">\u003Cstrong>MLOps betreibt ML-Systeme. LLMOps betreibt LLM-Anwendungen.\u003C\u002Fstrong>\u003Cbr>\u003Cbr>Klassisches MLOps konzentriert sich üblicherweise auf Datenpipelines, Training, Validierung, Modellregistrierung, Bereitstellung, Drift und Neutraining. LLMOps behält diese Disziplinen bei, wo sie relevant sind, fügt aber oft Prompt-\u002FKontextversionierung, Modell-\u002FAnbieterabstraktion, RAG-Indizes, Agent-\u002FTool-Traces, semantische Evaluierungen, Sicherheitstests, Token-\u002FKostenüberwachung und Regressionstests über sich schnell ändernde Modell-Snapshots hinzu.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Caside class=\"editorjs-callout editorjs-callout--warning my-6 rounded-xl border p-5 border-amber-300 bg-amber-50 dark:border-amber-900 dark:bg-amber-950\u002F20\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">LLMOps ist nicht nur Prompt-Management\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">Eine Produktions-LLM-Anwendung kann selbst dann fehlschlagen, wenn der Prompt unverändert bleibt: Der Anbieter kann einen Modell-Snapshot ändern, ein RAG-Korpus kann veralten, ein Reranker kann sich verschlechtern, Tool-Berechtigungen können sich ändern, die Kontextzusammenstellung kann Belege auslassen oder ein Agent kann eine falsche Trajektorie einschlagen. LLMOps muss daher das System um das Modell herum beobachten und versionieren, nicht nur den Prompt-Text.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Caside class=\"editorjs-callout editorjs-callout--note my-6 rounded-xl border p-5 border-gray-300 bg-gray-50 dark:border-gray-700 dark:bg-gray-900\u002F40\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Terminologische Abgrenzung\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">\u003Cstrong>LLMOps\u003C\u002Fstrong>, \u003Cstrong>GenAIOps\u003C\u002Fstrong> und verwandte Begriffe sind weit verbreitete Engineering-Bezeichnungen, aber sie sind kein einheitlicher formaler Standard mit einem einzigen kanonischen Lebenszyklus. Microsoft beschreibt GenAIOps derzeit als „manchmal LLMOps genannt“, während MLflow operative Tooling um Agenten und LLM-Anwendungen gruppiert. Dieser Artikel verwendet LLMOps als praktischen Architekturbegriff für den Betrieb von Produktionssystemen, deren Verhalten maßgeblich von LLMs abhängt.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Caside class=\"editorjs-callout editorjs-callout--note my-6 rounded-xl border p-5 border-gray-300 bg-gray-50 dark:border-gray-700 dark:bg-gray-900\u002F40\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Hinweis zu aktuellen Quellen — 8. Oktober 2026\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">Die operative Oberfläche verändert sich schnell. OpenAI empfiehlt derzeit, Modell-Snapshots zu pinnen und Evals auszuführen, weil sich das Prompting-Verhalten zwischen Snapshots ändern kann, und mehrere ältere plattformspezifische Prompt-\u002FEval-Oberflächen werden 2026 eingestellt. Die stabile architektonische Lehre ist, Prompts, Tests und Evals portabel zu halten und mit der Anwendung zu versionieren, anstatt von einem Dashboard-Objektmodell eines Anbieters abhängig zu sein.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Cnav class=\"editorjs-toc\" data-editorjs-toc=\"true\" aria-label=\"Inhalt\">\u003Cstrong class=\"editorjs-toc__title\">Inhalt\u003C\u002Fstrong>\u003Col class=\"editorjs-toc__list editorjs-toc__list--depth-0\">\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-7\" class=\"editorjs-toc__link\">Was MLOps wirklich bedeutet\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-11\" class=\"editorjs-toc__link\">Was sich ändert, wenn das Modell ein LLM ist\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-15\" class=\"editorjs-toc__link\">Das einfachste Beispiel\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-20\" class=\"editorjs-toc__link\">Wo das einfache Beispiel endet\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-24\" class=\"editorjs-toc__link\">MLOps vs. LLMOps\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-26\" class=\"editorjs-toc__link\">LLMOps erweitert MLOps, statt es zu ersetzen\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-30\" class=\"editorjs-toc__link\">Was muss in LLMOps versioniert werden?\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-32\" class=\"editorjs-toc__link\">Modell-Snapshots werden zu Release-Abhängigkeiten\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-36\" class=\"editorjs-toc__link\">Der Anbieter-Lebenszyklus wird Teil des Betriebs\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-40\" class=\"editorjs-toc__link\">Prompts verhalten sich wie Produktionscode\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-44\" class=\"editorjs-toc__link\">Context Engineering wird zu einem operativen Anliegen\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-48\" class=\"editorjs-toc__link\">RAG schafft einen eigenen operativen Lebenszyklus\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-53\" class=\"editorjs-toc__link\">Evals ersetzen „sieht gut aus für mich“ durch Release-Beweise\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-58\" class=\"editorjs-toc__link\">LLM-as-a-judge ist nützlich, aber keine Ground Truth\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-62\" class=\"editorjs-toc__link\">Tracing wird wichtiger als Endpoint-Logs\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-66\" class=\"editorjs-toc__link\">Agenten erweitern LLMOps zu Runtime-Operationen\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-71\" class=\"editorjs-toc__link\">Tokens, Modellaufrufe und Kontext werden zu Kostenvariablen\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-75\" class=\"editorjs-toc__link\">Caching wird semantisch, nicht nur technisch\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-79\" class=\"editorjs-toc__link\">Sicherheit und Berechtigungen werden zu Release-Kriterien\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-83\" class=\"editorjs-toc__link\">Wie CI in LLMOps aussieht\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-85\" class=\"editorjs-toc__link\">Wie CD in LLMOps aussieht\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-89\" class=\"editorjs-toc__link\">Kontinuierliches Training wird optional; kontinuierliche Evaluierung wird zentral\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-93\" class=\"editorjs-toc__link\">Was sollte in der Produktion überwacht werden?\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-95\" class=\"editorjs-toc__link\">Produktions-Traces können zu Evaluierungsdaten werden\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-99\" class=\"editorjs-toc__link\">Reproduzierbarkeit wird bedingt statt exakt\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-103\" class=\"editorjs-toc__link\">Lineage erweitert sich von Modell-Lineage zu Anwendungs-Lineage\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-107\" class=\"editorjs-toc__link\">Multi-Provider- und Modell-Routing erzeugen Betriebsrichtlinien\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-111\" class=\"editorjs-toc__link\">Belege aus der ursprünglichen Implementierung\u003C\u002Fa>\u003Col class=\"editorjs-toc__list editorjs-toc__list--depth-1\">\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-112\" class=\"editorjs-toc__link\">Aaasaasa AI Client: Anbieter, Modell und Laufzeitumgebung sind separate operative Objekte\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-117\" class=\"editorjs-toc__link\">Source of Truth Research Engine: Der Zustand einer LLM-Anwendung reicht über das Modell hinaus\u003C\u002Fa>\u003C\u002Fli>\u003C\u002Fol>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-123\" class=\"editorjs-toc__link\">Häufige LLMOps-Fehlermodi\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-125\" class=\"editorjs-toc__link\">Häufige Missverständnisse\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-127\" class=\"editorjs-toc__link\">Eine praktische LLMOps-Designsequenz\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-129\" class=\"editorjs-toc__link\">LLMOps-Architektur-Checkliste\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-131\" class=\"editorjs-toc__link\">Randfälle und Einschränkungen\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-137\" class=\"editorjs-toc__link\">Was würde diese Antwort ändern?\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-141\" class=\"editorjs-toc__link\">Verwandtes kanonisches Wissen\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-147\" class=\"editorjs-toc__link\">Häufig gestellte Fragen\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-149\" class=\"editorjs-toc__link\">Glossar\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-151\" class=\"editorjs-toc__link\">Fazit\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-155\" class=\"editorjs-toc__link\">Primärquellen und aktuelle Dokumentation\u003C\u002Fa>\u003C\u002Fli>\u003C\u002Fol>\u003C\u002Fnav>\n\u003Ch2 id=\"section-7\">Was MLOps wirklich bedeutet\u003C\u002Fh2>\n\u003Cp>MLOps wendet Software-Engineering- und Betriebsdisziplin auf Machine-Learning-Systeme an. Die Produktionsherausforderung ist breiter als das Trainieren eines Modells: Datenerfassung, Datenvalidierung, Experimentieren, Reproduzierbarkeit, Modellbewertung, Bereitstellung, Infrastruktur und Monitoring müssen alle zusammenwirken.\u003C\u002Fp>\n\u003Cp>Googles MLOps-Architekturleitfaden rahmt die Disziplin um kontinuierliche Integration, kontinuierliche Bereitstellung und kontinuierliches Training. CI validiert nicht nur Code, sondern auch Daten, Schemata und Modelle; CD stellt ML-Pipelines und Vorhersagedienste bereit; CT kann Modelle neu trainieren und erneut bereitstellen, wenn sich Daten oder Implementierungen ändern.\u003C\u002Fp>\n\u003Cp>AWS-Leitfaden ergänzt dieselben operativen Belange aus einem anderen Blickwinkel: Modellherkunft, Modell-\u002FVersionsnachverfolgbarkeit, Drift-Monitoring und Produktionsqualitätsüberwachung sind Kernbestandteile, um ML-Systeme nach der Bereitstellung zuverlässig zu halten.\u003C\u002Fp>\n\u003Ch2 id=\"section-11\">Was sich ändert, wenn das Modell ein LLM ist\u003C\u002Fh2>\n\u003Cp>Große Sprachmodelle verändern das Produktionsproblem, weil die Anwendung oft nicht den vollständigen Modelltrainingslebenszyklus besitzt. Ein Team kann eine gehostete Modell-API aufrufen, ein offenes Modell lokal ausführen, zwischen Anbietern wechseln oder mehrere Modelle für verschiedene Aufgaben verwenden.\u003C\u002Fp>\n\u003Cp>Das Modell ist daher nur eine versionierte Abhängigkeit innerhalb eines größeren Verhaltenssystems. Prompts, Retrieval-Ergebnisse, Kontextreihenfolge, Tools, Modell-Snapshot, Temperatur-\u002FReasoning-Einstellungen, Sicherheitsfilter und Laufzeitorchestrierung können alle die Ausgabe verändern.\u003C\u002Fp>\n\u003Cp>Dies schafft eine breitere operative Frage: Welche Kombination aus Modell, Kontext, Daten, Prompt, Tools und Laufzeit hat dieses Verhalten erzeugt? LLMOps existiert, um diese Frage beantwortbar und die Antwort reproduzierbar genug für Ingenieursarbeit zu machen.\u003C\u002Fp>\n\u003Ch2 id=\"section-15\">Das einfachste Beispiel\u003C\u002Fh2>\n\u003Cp>Angenommen, eine Anwendung beantwortet interne Richtlinienfragen.\u003C\u002Fp>\n\u003Cp>In einer klassischen ML-Rahmung könnten Sie einen trainierten Klassifikator versionieren, ihn bereitstellen und die Vorhersagequalität überwachen. In einer LLM-Anwendung könnte die Antwort von einem gehosteten Modell-Snapshot, einem System-Prompt, einem Embedding-Modell, einem Vektorindex, Retrieval-Filtern, einem Reranker und dem schließlich ausgewählten Kontext abhängen.\u003C\u002Fp>\n\u003Cp>Das Ändern einer dieser Komponenten kann die endgültige Antwort verändern, selbst wenn der Anwendungsendpunkt und die Benutzerfrage identisch bleiben.\u003C\u002Fp>\n\u003Csection class=\"editorjs-process my-6\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">Ein typischer LLMOps-Release-Pfad\u003C\u002Fh3>\u003Cdiv class=\"grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4\">\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">1\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">1. Eine Komponente ändern\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Prompt, Modell, Anbieter, Retrieval-Einstellung, Tool-Schema oder Anwendungscode ändern sich.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">2\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">2. Deterministische Tests ausführen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Schemata, Berechtigungen, Tool-Verträge, Retrieval-Filter und Anwendungsverhalten validieren.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">3\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">3. Verhaltens-Evals ausführen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Repräsentative Ausgaben, Retrieval-Qualität und Agent-\u002FTool-Trajektorien mit Akzeptanzkriterien vergleichen.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">4\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">4. Kosten und Latenz vergleichen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Token-Nutzung, Modellaufrufe, Retrieval-\u002FTool-Overhead und Antwortlatenz messen.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">5\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">5. Kontrollierte Version bereitstellen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Die konkrete Anwendungskonfiguration mit aufgezeichneten Modell-\u002FAnbieter-Versionen ausliefern.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">6\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">6. Produktionsverhalten nachverfolgen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Relevante Modell-, Retrieval-, Tool- und Laufzeit-Spans erfassen.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">7\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">7. Produktions-Traces auswerten\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Echte Ausführungen auf Qualität, Fundierung, Sicherheit und Aufgabenerfolg stichprobenartig prüfen.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">8\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">8. Zurückrollen oder iterieren\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Regressionsnachweise und Betriebssignale nutzen, um das nächste Release zu entscheiden.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fsection>\n\u003Ch2 id=\"section-20\">Wo das einfache Beispiel endet\u003C\u002Fh2>\n\u003Cp>Einige LLM-Systeme trainieren oder fine-tunen noch ihre eigenen Modelle, sodass traditionelle MLOps-Praktiken wie Trainingspipelines, Modell-Registry und Datenherkunft weiterhin direkt relevant bleiben.\u003C\u002Fp>\n\u003Cp>Andere Systeme verwenden nur externe Foundation-Model-APIs und führen nie kontinuierliches Training durch. Ihre Hauptbetriebsaufgabe ist Anwendungsbewertung, Modell-\u002FAnbieter-Änderungsmanagement, Prompt-\u002FKontext-Versionierung, Retrieval-Qualität und Observability.\u003C\u002Fp>\n\u003Cp>Es gibt daher keine einzige universelle „LLMOps-Pipeline“. Der genaue Lebenszyklus hängt davon ab, ob Sie trainieren, fine-tunen, selbst hosten, externes Wissen abrufen, Agents ausführen oder von verwalteten Modell-APIs abhängen.\u003C\u002Fp>\n\u003Ch2 id=\"section-24\">MLOps vs. LLMOps\u003C\u002Fh2>\n\u003Csection class=\"editorjs-comparison my-6\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">Was gleich bleibt und was sich erweitert\u003C\u002Fh3>\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 bg-gray-50 px-4 py-3 text-left dark:border-gray-700 dark:bg-gray-900\">\u003C\u002Fth>\u003Cth class=\"border border-gray-300 bg-gray-50 px-4 py-3 text-left font-semibold dark:border-gray-700 dark:bg-gray-900\">MLOps\u003C\u002Fth>\u003Cth class=\"border border-gray-300 bg-gray-50 px-4 py-3 text-left font-semibold dark:border-gray-700 dark:bg-gray-900\">LLMOps\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Primäre operative Einheit\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Modell-Eigentümerschaft\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Typische Änderung\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Bewertung\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Produktionsüberwachung\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Kontinuierliches Training\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Versionierte Artefakte\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Rollback-Ziel\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\u003C\u002Fsection>\n\u003Ch2 id=\"section-26\">LLMOps erweitert MLOps, statt es zu ersetzen\u003C\u002Fh2>\n\u003Cp>Die Kernbetriebsprinzipien verschwinden nicht: Quellcodeverwaltung, CI\u002FCD, Reproduzierbarkeit, Herkunft, Deployment-Kontrollen, Monitoring, Rollback und messbare Akzeptanzkriterien bleiben unerlässlich.\u003C\u002Fp>\n\u003Cp>Die Erweiterung besteht darin, dass sich mehr verhaltensbestimmende Artefakte nun außerhalb der Modellgewichte befinden. Ein verwaltetes Foundation-Modell kann sein Verhalten durch Snapshot-Upgrades ändern, während sich die Anwendungsausgabe durch Prompt- oder Retrieval-Änderungen ohne jegliches Modell-Retraining ändern kann.\u003C\u002Fp>\n\u003Cp>Deshalb ist die nützliche Hierarchie normalerweise DevOps → MLOps → LLMOps\u002FGenAIOps als zunehmend spezialisierte operative Anliegen, nicht drei sich gegenseitig ausschließende Praktiken.\u003C\u002Fp>\n\u003Ch2 id=\"section-30\">Was muss in LLMOps versioniert werden?\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Artefakt\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Warum es wichtig ist\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Anwendungscode\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Definiert Orchestrierung, Validierung, Wiederholungen und Geschäftsverhalten\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modellfamilie + Snapshot\u002FVersion\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Unterschiedliche Snapshots können unterschiedliches Verhalten erzeugen\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Anbieter \u002F Endpunkt\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändert Datenfluss, Latenz, Limits, Preisgestaltung und Verfügbarkeit\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Prompt-\u002FInstruktionscode\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändert das Modellverhalten auch bei gleichem Modell\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Generierungs-\u002FReasoning-Parameter\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Können Determinismus, Latenz, Tiefe und Kosten verändern\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Eval-Datensatz\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Definiert, wogegen „gut genug“ getestet wird\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Scorer \u002F Bewerter\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Definieren, wie Qualität gemessen wird\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Embedding-Modell\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändert Vektorrepräsentation und Retrieval-Verhalten\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Chunking-\u002FIndex-Konfiguration\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändert, was abgerufen werden kann\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Reranker \u002F Retrieval-Fusion\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändert die Ergebnisreihenfolge\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Tool-Schemata\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändern, was das Modell anfordern kann und wie\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Berechtigungsprofil\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändert, welche Tool-Aktionen tatsächlich ausgeführt werden dürfen\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kontext-Assemblierungsregeln\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändern, welche Evidenz und welcher Zustand das Modell erreichen\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Sicherheits-\u002FGuardrail-Konfiguration\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ändert erlaubtes oder blockiertes Verhalten\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-32\">Modell-Snapshots werden zu Release-Abhängigkeiten\u003C\u002Fh2>\n\u003Cp>Bei gehosteten LLMs kontrolliert das Team möglicherweise nicht das Modelltraining, aber es kontrolliert weiterhin, welches Modell oder welchen Snapshot die Anwendung aufruft.\u003C\u002Fp>\n\u003Cp>Die aktuelle API-Richtlinie von OpenAI warnt ausdrücklich, dass sich das Prompting-Verhalten zwischen Modell-Snapshots ändern kann, und empfiehlt, Produktionsanwendungen auf bestimmte Snapshots festzulegen, wo Konsistenz wichtig ist, und dann beim Upgrade Evals auszuführen.\u003C\u002Fp>\n\u003Cp>Die operative Konsequenz ist eindeutig: Modell-Upgrades sollten als Anwendungs-Releases behandelt werden, nicht als unsichtbare Infrastrukturwartung.\u003C\u002Fp>\n\u003Ch2 id=\"section-36\">Der Anbieter-Lebenszyklus wird Teil des Betriebs\u003C\u002Fh2>\n\u003Cp>LLM-Anwendungen hängen oft von Anbieter-Rate-Limits, Deprecation-Zeitplänen, API-Semantik, Kontextlimits, Datenverarbeitungsregeln und Preisen ab.\u003C\u002Fp>\n\u003Cp>Ein Anbieter kann ein Modell als veraltet einstufen, während Ihr Anwendungscode unverändert bleibt. Der aktuelle Deprecation-Zeitplan von OpenAI enthält beispielsweise Auslaufdaten für 2026 für ältere Modell-Snapshots und Plattformoberflächen.\u003C\u002Fp>\n\u003Cp>LLMOps benötigt daher zusätzlich zur Modellqualitätsüberwachung auch die Verfolgung des Anbieter-Lebenszyklus, Migrationstests und Fallback-Entscheidungen.\u003C\u002Fp>\n\u003Ch2 id=\"section-40\">Prompts verhalten sich wie Produktionscode\u003C\u002Fh2>\n\u003Cp>Prompts sind ausführbare Verhaltenskonfiguration. Kleine Änderungen können die Ausgabequalität, die Werkzeugauswahl und die Richtlinieninterpretation verändern.\u003C\u002Fp>\n\u003Cp>Die aktuelle Empfehlung von OpenAI rät dazu, Produktions-Prompts im Anwendungscode zu speichern, Prompt-Änderungen über Pull Requests zu überprüfen, typisierte Eingaben zu verwenden und Änderungen mit Tests und Evaluierungsprüfungen abzudecken.\u003C\u002Fp>\n\u003Cp>Dadurch ähnelt die Prompt-Versionierung weniger dem Bearbeiten von Marketingtexten und mehr dem Ändern einer Funktion, deren Ausgabe probabilistisch und modellabhängig ist.\u003C\u002Fp>\n\u003Ch2 id=\"section-44\">Context Engineering wird zu einem operativen Anliegen\u003C\u002Fh2>\n\u003Cp>Das Produktionsmodell erhält selten nur einen statischen Prompt. Es kann Gesprächsverlauf, abgerufene Dokumente, Werkzeugausgaben, Speicher, aktuellen Anwendungszustand und Richtlinienanweisungen erhalten.\u003C\u002Fp>\n\u003Cp>LLMOps muss daher die Kontextzusammenstellung beobachten: welche Belege ausgewählt wurden, welche Zustandsversion aktuell war, ob eine Kürzung stattfand und ob wichtige Anweisungen die Komprimierung überlebt haben.\u003C\u002Fp>\n\u003Cp>Eine Modellregression und eine Kontextregression können bei der endgültigen Antwort identisch aussehen. Das Nachverfolgen des tatsächlichen Kontextpfads ermöglicht es dem Team, sie zu unterscheiden.\u003C\u002Fp>\n\u003Ch2 id=\"section-48\">RAG schafft einen eigenen operativen Lebenszyklus\u003C\u002Fh2>\n\u003Cp>Ein RAG-System führt eine zweite Produktionspipeline neben der Modellinferenz ein: Ingestion, Extraktion, Chunking, Metadaten, Embeddings, Indizes, Retrieval, Reranking und Kontextauswahl.\u003C\u002Fp>\n\u003Cp>Der Wissenskorpus kann sich täglich ändern, selbst wenn Modell und Prompt unverändert bleiben. Ein veralteter Index oder ein defekter Metadatenfilter kann daher die Antwortqualität beeinträchtigen, ohne dass ein Modell-Drift vorliegt.\u003C\u002Fp>\n\u003Cp>LLMOps für RAG sollte Korpus-\u002FIndexversion, Embedding-Modell, Chunking-Richtlinie, Retrieval-Konfiguration, Quellenaktualität und Retrieval-Metriken getrennt von der Generierungsqualität verfolgen.\u003C\u002Fp>\n\u003Caside class=\"editorjs-referral my-6\">\u003Ca href=\"https:\u002F\u002Fstajic.de\u002Fde\u002Fblog\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method\" class=\"flex flex-col sm:flex-row gap-4 rounded-xl border border-gray-200 dark:border-gray-700 p-4 transition hover:border-primary-500\">\u003Cdiv class=\"min-w-0 flex-1\">\u003Cstrong class=\"block text-lg text-gray-900 dark:text-gray-100\">RAG fehlgeschlagen – aber welche Ebene ist tatsächlich fehlgeschlagen? Eine diagnostische Methode\u003C\u002Fstrong>\u003Cp class=\"mt-2 text-sm text-gray-600 dark:text-gray-300\">Eine Produktions-LLM-Pipeline benötigt separate Observability für Quellenabdeckung, Retrieval, Ranking, Kontextzusammenstellung und Generierung.\u003C\u002Fp>\u003Cspan class=\"mt-3 inline-flex text-sm font-medium text-primary-600 dark:text-primary-400\">Die RAG-Diagnosemethode lesen →\u003C\u002Fspan>\u003C\u002Fdiv>\u003C\u002Fa>\u003C\u002Faside>\n\u003Ch2 id=\"section-53\">Evals ersetzen „sieht gut aus für mich“ durch Release-Beweise\u003C\u002Fh2>\n\u003Cp>Generative Ausgaben sind oft offen, sodass Exact-Match-Tests für viele Aufgaben unzureichend sind. LLMOps fügt Evaluierungsdatensätze und Scorer hinzu, die Aufgabenerfolg, Korrektheit, Sicherheit, Fundiertheit, Stil oder domänenspezifische Akzeptanzkriterien messen können.\u003C\u002Fp>\n\u003Cp>Der aktuelle GenAI-Evaluierungsstack von MLflow unterstützt versionierte Evaluierungsdatensätze, Prompt-\u002FModellvergleiche, benutzerdefinierte Scorer und die Evaluierung über vollständige Traces.\u003C\u002Fp>\n\u003Cp>Die stärkste Praxis ist evaluierungsgetriebene Entwicklung: Definieren Sie repräsentative Fälle und Akzeptanzkriterien vor oder parallel zu Änderungen und vergleichen Sie dann Releases anhand derselben Evidenz.\u003C\u002Fp>\n\u003Caside class=\"editorjs-callout editorjs-callout--success my-6 rounded-xl border p-5 border-emerald-300 bg-emerald-50 dark:border-emerald-900 dark:bg-emerald-950\u002F20\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Verhaltensänderungen erfordern Verhaltenstests\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">Ein Deployment sollte nicht allein deshalb als gleichwertig betrachtet werden, weil der API-Vertrag noch funktioniert. Wenn sich Prompt, Modell, Retrieval oder Tools geändert haben, sollte die behavioral Regression Suite erneut ausgeführt werden.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Ch2 id=\"section-58\">LLM-as-a-judge ist nützlich, aber keine Ground Truth\u003C\u002Fh2>\n\u003Cp>LLM-Richter können die Evaluierung für Qualitäten skalieren, die teuer als deterministische Assertions zu kodieren sind, wie Relevanz, Tonalität oder Groundedness.\u003C\u002Fp>\n\u003Cp>Der Richter ist jedoch ein weiteres Modell mit eigenem Bias, eigener Version und eigenem Prompt. Die Richter-Konfiguration sollte daher versioniert und gegen menschliche oder deterministische Referenzfälle kalibriert werden, wo Konsequenzen wichtig sind.\u003C\u002Fp>\n\u003Cp>Eine Produktions-Evaluierung kann deterministische Checks, referenzbasierte Metriken, Modell-Richter und menschliche Überprüfung kombinieren, anstatt eine einzige Metrik alle Qualitätsdimensionen repräsentieren zu lassen.\u003C\u002Fp>\n\u003Ch2 id=\"section-62\">Tracing wird wichtiger als Endpoint-Logs\u003C\u002Fh2>\n\u003Cp>Traditionelle API-Logs können Ihnen sagen, dass eine Anfrage zwei Sekunden gedauert hat und HTTP 200 zurückgegeben wurde. Sie können Ihnen nicht sagen, welche abgerufenen Chunks ausgewählt wurden, welches Tool der Agent aufgerufen hat oder welcher Modell-Span die meisten Tokens verbraucht hat.\u003C\u002Fp>\n\u003Cp>Das aktuelle GenAI-Tracing von MLflow erfasst Prompts, Retrievals, Tool-Aufrufe und Anwendungs-Spans, und sein Produktions-Evaluierungsablauf kann Zwischeninformationen der Trajektorie bewerten, nicht nur den finalen Text.\u003C\u002Fp>\n\u003Cp>Dies ist ein bedeutender LLMOps-Wandel: Observability folgt dem Verhaltensgraphen der Anwendung, nicht nur dem Serving-Endpoint.\u003C\u002Fp>\n\u003Ch2 id=\"section-66\">Agenten erweitern LLMOps zu Runtime-Operationen\u003C\u002Fh2>\n\u003Cp>Eine agentische Anwendung kann mehrere Modellaufrufe, Tool-Aufrufe und Zustandsübergänge ausführen, bevor sie ein Ergebnis liefert.\u003C\u002Fp>\n\u003Cp>Der Betrieb von Agenten erfordert daher Schrittanzahlen, Tool-Call-Traces, Berechtigungsverweigerungen, Wiederholungen, Schleifenerkennung, menschliche Genehmigungen und verifizierten Endzustand zusätzlich zu gewöhnlichen Modell-Latenz- und Token-Metriken.\u003C\u002Fp>\n\u003Cp>Eine korrekte finale Antwort kann eine schlechte Trajektorie verbergen, daher muss die Agenten-Evaluierung sowohl den Pfad als auch das Ergebnis untersuchen.\u003C\u002Fp>\n\u003Caside class=\"editorjs-referral my-6\">\u003Ca href=\"https:\u002F\u002Fstajic.de\u002Fde\u002Fblog\u002Fai-agent-reliability-why-the-final-answer-is-not-enough\" class=\"flex flex-col sm:flex-row gap-4 rounded-xl border border-gray-200 dark:border-gray-700 p-4 transition hover:border-primary-500\">\u003Cdiv class=\"min-w-0 flex-1\">\u003Cstrong class=\"block text-lg text-gray-900 dark:text-gray-100\">Zuverlässigkeit von KI-Agenten: Warum die finale Antwort nicht ausreicht\u003C\u002Fstrong>\u003Cp class=\"mt-2 text-sm text-gray-600 dark:text-gray-300\">Warum die Produktions-Evaluierung von Agenten Tool-Aufrufe, Zustandsübergänge, Genehmigungen und Wiederherstellbarkeit umfassen muss.\u003C\u002Fp>\u003Cspan class=\"mt-3 inline-flex text-sm font-medium text-primary-600 dark:text-primary-400\">Lesen Sie den Artikel zur Agenten-Zuverlässigkeit →\u003C\u002Fspan>\u003C\u002Fdiv>\u003C\u002Fa>\u003C\u002Faside>\n\u003Ch2 id=\"section-71\">Tokens, Modellaufrufe und Kontext werden zu Kostenvariablen\u003C\u002Fh2>\n\u003Cp>Die klassischen ML-Inferenzkosten werden oft von der Serving-Infrastruktur oder der Compute pro Vorhersage dominiert. LLM-Anwendungen können Anbieter-Token-Preise, wiederholte Agentenaufrufe, Embedding-Aufrufe, Reranking und Tool-\u002FRuntime-Overhead hinzufügen.\u003C\u002Fp>\n\u003Cp>Kosten müssen daher einer Aufgabe oder einem Trace zugeordnet werden, nicht nur einem Endpunkt. Ein Workflow, der acht versteckte Modellaufrufe macht, kann funktional korrekt, aber operativ inakzeptabel sein.\u003C\u002Fp>\n\u003Cp>Latenz verhält sich genauso: Modelllatenz, Retrieval, Reranking und externe Tools setzen sich zu einer End-to-End-Benutzerlatenz zusammen.\u003C\u002Fp>\n\u003Ch2 id=\"section-75\">Caching wird semantisch, nicht nur technisch\u003C\u002Fh2>\n\u003Cp>LLM-Systeme können Prompts, Embeddings, Retrieval-Ergebnisse oder vollständige Antworten cachen, aber der Cache-Schlüssel muss die Semantik widerspiegeln, die das Ergebnis verändern kann.\u003C\u002Fp>\n\u003Cp>Ein Antwort-Cache, der Modellversion, Mandant, Berechtigungen oder Aktualität der Quelle ignoriert, kann eine technisch gültige, aber semantisch ungültige Antwort zurückgeben.\u003C\u002Fp>\n\u003Cp>LLMOps behandelt Cache-Invalidierung daher als Teil der Modell-\u002FKontext-\u002FDaten-Versionierung und nicht nur als Infrastruktur-Optimierung.\u003C\u002Fp>\n\u003Ch2 id=\"section-79\">Sicherheit und Berechtigungen werden zu Release-Kriterien\u003C\u002Fh2>\n\u003Cp>Generative Systeme können unbegrenzten Text erzeugen und Agenten können externe Aktionen auslösen. Sicherheitstests stehen daher näher an gewöhnlicher CI\u002FCD als in vielen klassischen prädiktiven ML-Systemen.\u003C\u002Fp>\n\u003Cp>Berechtigungsprüfungen, Prompt-Injection-Tests, Mandantenisolationstests und Genehmigungen für Nebenwirkungen sollten reproduzierbare Regressionstests sein, wo diese Risiken bestehen.\u003C\u002Fp>\n\u003Cp>Das Modell kann eine Operation vorschlagen, aber die Laufzeit muss weiterhin die Autorisierung durchsetzen. LLMOps verantwortet den Nachweis, dass diese Kontrollen nach Änderungen an Modell, Prompt oder Tools weiterhin funktionieren.\u003C\u002Fp>\n\u003Ch2 id=\"section-83\">Wie CI in LLMOps aussieht\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">CI-Ebene\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Beispielprüfungen\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Code\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Unit-Tests, Typprüfungen, Schema-Validierung\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Prompts\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Template-Rendering, erforderliche Variablen, Richtlinientext, Snapshot-Review\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modelle\u002FAnbieter\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kompatibilität, Ausgabeschema, Fähigkeits- und Regressionstests\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">RAG\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Chunking-Fixtures, Filtertests, Recall@k, Reranker-Regression\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Tools\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ein-\u002FAusgabeschema-Tests, Berechtigungstests, Idempotenz-Tests\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Agenten\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Trajektorien-Fixtures, Schleifenlimits, Handoff-\u002FTool-Auswahltests\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Sicherheit\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Prompt-Injection, nicht autorisierte Tools, mandantenübergreifende Negativtests\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verhaltensevaluierungen\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Aufgabenerfolg, Korrektheit, Fundierung, Sicherheit, Domänenkriterien\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Operativ\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Latenz, Token-\u002FKostenbudgets, Timeout-\u002FFallback-Verhalten\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-85\">Wie CD in LLMOps aussieht\u003C\u002Fh2>\n\u003Cp>Ein Produktions-Release kann überhaupt kein neues Modellartefakt bereitstellen. Es kann einfach einen neuen Prompt, eine Retrieval-Konfiguration, ein Toolset oder eine Anbieterzuordnung ausliefern.\u003C\u002Fp>\n\u003Cp>Das Release-Bundle sollte daher die vollständige verhaltensbestimmende Konfiguration identifizieren und nicht nur das Anwendungs-Container-Image.\u003C\u002Fp>\n\u003Cp>Feature-Flags, gestaffelte Rollouts, Shadow-Evaluierung, Canary-Traffic und Rollback sind nützlich, weil LLM-Verhalten auf Arten regressieren kann, die statische Vertragstests nicht erkennen.\u003C\u002Fp>\n\u003Ch2 id=\"section-89\">Kontinuierliches Training wird optional; kontinuierliche Evaluierung wird zentral\u003C\u002Fh2>\n\u003Cp>Traditionelles MLOps betont oft kontinuierliches Training, wenn neue Daten oder Drift ein Neutraining rechtfertigen.\u003C\u002Fp>\n\u003Cp>Viele LLM-Anwendungen trainieren niemals das Foundation-Modell. Ihr äquivalenter kontinuierlicher Kreislauf ist die kontinuierliche Evaluierung: Fehler und repräsentative Produktionsfälle sammeln, sie den Evaluierungsdatensätzen hinzufügen, Kandidatenänderungen an Prompt\u002FModell\u002FRetrieval testen und nur dann neu bereitstellen, wenn sich die Evidenz verbessert.\u003C\u002Fp>\n\u003Cp>Fine-Tuning kann einen Trainingslebenszyklus wieder einführen, sollte aber in denselben übergeordneten Evaluierungs- und Release-Prozess eingebettet sein.\u003C\u002Fp>\n\u003Ch2 id=\"section-93\">Was sollte in der Produktion überwacht werden?\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Signalklasse\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Beispiele\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Systemzustand\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Fehler, Timeouts, Endpunktverfügbarkeit\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modell\u002FAnbieter\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modell-ID, Snapshot, Ratenlimits, Anbieterfehler\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Latenz\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">End-to-End-, Modell-, Retrieval-, Tool- und Reranker-Spans\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kosten\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Eingabe-\u002FAusgabe-Token, Embeddings, Tool-\u002FAPI-Ausgaben\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Qualität\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Stichprobenartiger Aufgabenerfolg, Korrektheit, Relevanz, Fundiertheit\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">RAG\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Retrieval-Recall-Proxys, leeres Retrieval, veraltete Quellen, Zitatabdeckung\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Agenten\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Tool-Auswahl, Wiederholungen, Schleifen, Übergaben, Genehmigungshäufigkeit\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Sicherheit\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verweigerte Aktionen, Prompt-Injection-Indikatoren, Mandantengrenzenverletzungen\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Benutzerfeedback\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Korrekturen, Abbruch, Eskalation, explizite Bewertungen\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Änderungsdrift\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Anbieter-\u002FModell-\u002FKonfigurationsänderungen relativ zum genehmigten Release\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-95\">Produktions-Traces können zu Evaluierungsdaten werden\u003C\u002Fh2>\n\u003Cp>Eines der nützlichsten modernen LLMOps-Muster besteht darin, gesampelte Produktions-Traces in Evaluierungsdatensätze umzuwandeln.\u003C\u002Fp>\n\u003Cp>MLflow unterstützt derzeit das Abrufen von Produktions-Traces und die Bewertung nicht nur von Ausgaben, sondern auch von Zwischen-Spans wie Retrieval- oder Tool-Aufruftrajektorien.\u003C\u002Fp>\n\u003Cp>Dies schließt den Kreislauf zwischen Observability und Entwicklung: echte Fehler können zu Regressionsfällen im nächsten Release werden, anstatt in Logs zu verschwinden.\u003C\u002Fp>\n\u003Ch2 id=\"section-99\">Reproduzierbarkeit wird bedingt statt exakt\u003C\u002Fh2>\n\u003Cp>Klassische ML-Reproduzierbarkeit zielt oft darauf ab, ein Modell aus versioniertem Code, Daten, Umgebung und Trainingsparametern nachzubilden.\u003C\u002Fp>\n\u003Cp>Gehostete LLM-Anwendungen können nicht immer identische Ausgaben Token für Token reproduzieren, da die Generierung probabilistisch ist und Anbieter die Infrastruktur kontrollieren können.\u003C\u002Fp>\n\u003Cp>LLMOps zielt daher auf verhaltensbasierte Reproduzierbarkeit ab: genügend Modell-\u002FAnbieter-\u002FVersions-, Prompt-, Kontext-Eingaben, Retrieval-Zustand und Laufzeitkonfiguration aufzeichnen, um die Bedingungen zu reproduzieren und das Verhalten innerhalb erwarteter Toleranzen zu validieren.\u003C\u002Fp>\n\u003Ch2 id=\"section-103\">Lineage erweitert sich von Modell-Lineage zu Anwendungs-Lineage\u003C\u002Fh2>\n\u003Cp>Die MLOps-Richtlinien von AWS betrachten Modell-Lineage als die Historie von Code-, Daten-, Modell- und Infrastruktur-Artefakten, die für Diagnose und Reproduzierbarkeit benötigt werden.\u003C\u002Fp>\n\u003Cp>Für LLM-Anwendungen sollte Lineage zusätzlich Prompts, Eval-Datensätze, Retrieval-\u002FIndex-Versionen, Tool-Schemas, Agent-\u002FLaufzeitkonfiguration und Anbieter-\u002FModell-Snapshots verbinden.\u003C\u002Fp>\n\u003Cp>Die Ziel Frage wird: Welche exakte Anwendungskonfiguration hat diesen Trace erzeugt?\u003C\u002Fp>\n\u003Ch2 id=\"section-107\">Multi-Provider- und Modell-Routing erzeugen Betriebsrichtlinien\u003C\u002Fh2>\n\u003Cp>Sobald eine Anwendung mehrere Anbieter oder lokale Modelle nutzen kann, wird Routing zu einer Betriebsrichtlinie statt zu einem einfachen Modell-String.\u003C\u002Fp>\n\u003Cp>Das Routing kann von Fähigkeit, Latenz, Kosten, Datenschutz, Kontextlänge, Verfügbarkeit, Tool-Unterstützung oder Lokalität abhängen. Ein Fallback kann die Verfügbarkeit aufrechterhalten, während sich die Antwortqualität oder die Annahmen zur Datenverarbeitung ändern.\u003C\u002Fp>\n\u003Cp>LLMOps sollte daher protokollieren, welche Route tatsächlich ausgewählt wurde, und Routen unabhängig voneinander bewerten, anstatt jeden kompatiblen Endpunkt als verhaltensmäßig austauschbar zu behandeln.\u003C\u002Fp>\n\u003Ch2 id=\"section-111\">Belege aus der ursprünglichen Implementierung\u003C\u002Fh2>\n\u003Ch3 id=\"section-112\">Aaasaasa AI Client: Anbieter, Modell und Laufzeitumgebung sind separate operative Objekte\u003C\u002Fh3>\n\u003Cp>Der Aaasaasa AI Client trennt Agent\u002FClient, Anbieter, Modell, Laufzeitort und Berechtigungen. Sein AI Hub unterstützt Ollama, LM Studio\u002FOpenAI-kompatible Endpunkte und andere Anbieterprotokolle, anstatt „das Modell“ als eine globale Einstellung zu behandeln.\u003C\u002Fp>\n\u003Cp>Die Implementierung umfasst dynamische lokale Modellerkennung, Streaming, Thinking-Ausgabe und explizite Ollama-Steuerungen zum Warmladen\u002FLaden und Entladen. Das ist ein operativer Beleg dafür, dass lokales LLM-Serving Ressourcen-Lebenszyklus-Probleme mit sich bringt, die über einen API-Modellnamen hinausgehen.\u003C\u002Fp>\n\u003Cp>Der Anbieterstatus wird über Anbieteradapter abgefragt, und Verbindungstypen unterscheiden lokale, Cloud-API-, kontogestützte, Remote-Agent- und Web-Client-Pfade. Dies sind konkrete operative Dimensionen, die eine LLM-bewusste Plattform sichtbar machen muss.\u003C\u002Fp>\n\u003Cp>Das Repository wahrt außerdem eine wichtige Grenze: Eine lokale Laufzeitumgebung ist nicht automatisch lokale Inferenz. Anbieter\u002FModell\u002FLaufzeitort sind versionierte oder konfigurierbare Belange, die Datenschutz, Latenz, Kosten und Verfügbarkeit beeinflussen.\u003C\u002Fp>\n\u003Ch3 id=\"section-117\">Source of Truth Research Engine: Der Zustand einer LLM-Anwendung reicht über das Modell hinaus\u003C\u002Fh3>\n\u003Cp>Die Source of Truth Research Engine kombiniert lexikalische Suche, optionale Embeddings, Quellen-Snapshots, SHA-256-Identität, Claims, Provenienz und Widerspruchsverfolgung rund um lokale modellgestützte Recherche.\u003C\u002Fp>\n\u003Cp>Dies ist ein nützlicher LLMOps-Beleg, weil allein die Änderung des Modells das Recherchesystem nicht definiert. Retrieval, Quellenerfassung, Evidenzklassifizierung und persistente Provenienz sind unabhängige operative Artefakte.\u003C\u002Fp>\n\u003Cp>Die Implementierung behandelt semantische Ähnlichkeit bewusst als Entdeckung und nicht als Evidenz, was zeigt, warum LLMOps-Observability Retrieval-Verhalten von der Gültigkeit von Claims unterscheiden sollte.\u003C\u002Fp>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Beobachtete Implementierung\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">LLMOps-Lektion\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Mehrere Anbieterprotokolle\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Anbieteridentität ist eine operative Abhängigkeit\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Dynamische Modellerkennung\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verfügbare Modelle können sich unabhängig vom Anwendungscode ändern\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ollama-Steuerungen zum Laden\u002FEntladen\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Lokale Modelle haben einen Speicher-\u002FRessourcen-Lebenszyklus\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Adapter für Anbieterzustand\u002F-status\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modellverfügbarkeit erfordert Laufzeit-Observability\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Getrennter Laufzeit- und Inferenzort\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Deployment-Topologie ist nicht ein boolesches „lokal\u002FCloud“\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Zentrale Berechtigungen\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modellfähigkeit und Tool-Autorität müssen getrennt bleiben\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Lexikalische + semantische Retrieval-Pipeline\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Retrieval-Konfiguration ist Teil des Anwendungsverhaltens\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Persistenz von Quelle\u002FProvenienz\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Operativer Zustand und Evidenz liegen außerhalb der Modellgewichte\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Caside class=\"editorjs-callout editorjs-callout--note my-6 rounded-xl border p-5 border-gray-300 bg-gray-50 dark:border-gray-700 dark:bg-gray-900\u002F40\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Evidenzgrenze\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">Diese Projekte demonstrieren Multi-Provider-\u002FLokale-Modell-Operationen, Berechtigungstrennung, Retrieval-Infrastruktur und Evidenzpersistenz. Sie werden nicht als vollständige kommerzielle LLMOps-Plattform oder als Nachweis für Produktionsverkehr in großem Maßstab präsentiert.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Ch2 id=\"section-123\">Häufige LLMOps-Fehlermodi\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Fehlermodus\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Was tatsächlich schiefging\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modell-Alias wurde stillschweigend aktualisiert\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verhalten änderte sich ohne kontrollierte Freigabe\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Prompt ohne Evals geändert\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verhaltensregression bestand normale Unit-Tests\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">RAG-Index veraltet\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Das Generierungsmodell wurde für Retrieval-\u002FDatenfehler verantwortlich gemacht\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Nur die endgültige Antwort wird protokolliert\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Grundursache in Retrieval-\u002FTool-\u002FKontext-Trajektorie ist unsichtbar\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Anbieter-Fallback erfolgt stillschweigend\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Anderer Modell-\u002FDatenpfad ändert Verhalten ohne Zuordnung\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Token-Kosten werden global erfasst\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Teure Workflows können nicht lokalisiert werden\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Judge-Modell geändert\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Bewertungsergebnisse driften ohne Anwendungsänderung\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Produktions-Traces werden nie zu Tests\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Bekannte Fehler kehren wiederholt zurück\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Lokales Modell bleibt unbegrenzt geladen\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">VRAM-\u002FRessourcendruck wird zu operativer Instabilität\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Berechtigungen nur im Prompt kodiert\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modellverhalten wird mit Autorisierung verwechselt\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ein Eval-Score steuert alles\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verschiedene Qualitätsdimensionen werden zu einer irreführenden Zahl zusammengefasst\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modell-Registry existiert, aber Prompt-\u002FIndex-Versionen nicht\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Anwendungs-Lineage bleibt unvollständig\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-125\">Häufige Missverständnisse\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Missverständnis\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Korrektur\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„LLMOps ersetzt MLOps.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">LLMOps erweitert MLOps-Prinzipien auf LLM-spezifisches Anwendungsverhalten.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„LLMOps ist Prompt Engineering.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Prompts sind ein Artefakt unter Modellen, Anbietern, Kontext, Retrieval, Tools, Evals und Laufzeit.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Gehostete APIs beseitigen operative Arbeit.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Sie beseitigen einige Arbeiten zum Modell-Serving\u002FTraining, fügen aber Anbieter-Lebenszyklus-, Versions- und Abhängigkeitsmanagement hinzu.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Wenn die API stabil ist, ist die App stabil.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modellverhalten und Anbieter-\u002FModell-Snapshots können sich unabhängig vom API-Schema ändern.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„RAG ist nur Datenvorverarbeitung.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">In der Produktion hat es seinen eigenen Lebenszyklus für Ingestion, Index, Retrieval und Aktualität.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„LLM-Ausgaben können nicht getestet werden.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Sie können mit deterministischen, Referenz-, Judge- und menschlichen Kriterien bewertet werden.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„LLM-Judges sind objektive Ground Truth.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Sie sind modellbasierte Bewerter, die ebenfalls Kalibrierung und Versionskontrolle erfordern.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Ein lokales Modell eliminiert LLMOps.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Lokales Serving bringt Modell-Dateien, VRAM, Laden\u002FEntladen, Laufzeit-Zustand und Upgrade-Belange mit sich.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Observability bedeutet Token-Zahlen.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Nützliche Observability folgt Prompts, Retrievals, Tools, Modell-Spans und Ergebnissen.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Kontinuierliches Training ist verpflichtend.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Viele LLM-Apps verwenden kontinuierliche Evaluierung, ohne das Foundation-Modell zu trainieren.\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-127\">Eine praktische LLMOps-Designsequenz\u003C\u002Fh2>\n\u003Csection class=\"editorjs-process my-6\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">Das vollständige verhaltenserzeugende System betreiben\u003C\u002Fh3>\u003Cdiv class=\"grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4\">\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">1\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">1. Die Verhaltenseinheit definieren\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Listen Sie jede Komponente auf, die die Ausgabe wesentlich verändern kann: Modell, Prompt, Retrieval, Tools, Kontext und Policy.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">2\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">2. Anwendungs-Lineage herstellen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Versionieren Sie Code, Modell\u002FAnbieter, Prompts, Eval-Datensätze, Retrieval-Konfiguration und Tool-Verträge.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">3\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">3. Repräsentative Eval-Datensätze erstellen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Verwenden Sie erwartete Erfolgs-\u002FFehlerfälle aus Design und Produktion.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">4\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">4. Deterministische und verhaltensbezogene Tests trennen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Halten Sie Schema-\u002FSicherheitsassertions getrennt von der semantischen Ausgabebewertung.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">5\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">5. End-to-End-Ausführung nachverfolgen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Instrumentieren Sie Modell-, Retrieval-, Reranking-, Tool- und Agent-\u002FRuntime-Spans.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">6\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">6. Release-Gates definieren\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Legen Sie Schwellenwerte für Qualität, Sicherheit, Latenz und Kosten fest.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">7\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">7. Modellversionen pinnen oder explizit aufzeichnen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Behandeln Sie Modell-\u002FAnbieteränderungen als Release-Ereignisse.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">8\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">8. Progressiv deployen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Verwenden Sie Flags, Canaries oder gestaffelte Rollouts, wo die Konsequenz es rechtfertigt.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">9\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">9. Produktions-Traces bewerten\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Messen Sie reales Aufgabenverhalten und identifizieren Sie wiederkehrende Fehler.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">10\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">10. Fehler zurück in Eval-Datensätze einspeisen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Verwandeln Sie Vorfälle und Korrekturen in dauerhafte Regressionsabdeckung.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">11\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">11. Anbieter- und Datenlebenszyklen überwachen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Verfolgen Sie Deprecations, Index-Aktualität, Quelländerungen und Runtime-Verfügbarkeit.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">12\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">12. Veraltete Versionen sauber außer Betrieb nehmen\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Entfernen Sie alte Prompts\u002FModelle\u002FIndizes\u002FAnmeldedaten nach Migration und Entscheidungen zur Aufbewahrung von Nachweisen.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fsection>\n\u003Ch2 id=\"section-129\">LLMOps-Architektur-Checkliste\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Frage\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Erwarteter Nachweis\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Welches Modell\u002Fwelcher Anbieter\u002Fwelche Version hat die Anfrage bedient?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Nachverfolgbare Modellidentität\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Welcher Prompt\u002Fwelche Anweisungen waren aktiv?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Versionierter Anwendungscode\u002F-konfiguration\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Welcher Kontext hat das Modell erreicht?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kontext-\u002FRetrieval-Trace\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Welche Korpus-\u002FIndexversion wurde verwendet?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Retrieval-Lineage\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Welche Tools waren verfügbar und wurden aufgerufen?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Tool-Schema + Trajektorien-Trace\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Welche Berechtigungen galten?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Runtime-Autorisierungsdatensatz\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Wie wird Qualität gemessen?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Versionierter Eval-Datensatz + Scorer\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Wie werden Modell-Upgrades getestet?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verhaltensbezogene Regressionssuite\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Wie wird Produktionsqualität gesampelt?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Trace-Bewertungs-\u002FFeedback-Prozess\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kann ein Fehler näherungsweise reproduziert werden?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modell-\u002FKontext-\u002FAnbieter-\u002FAnwendungs-Lineage\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Wo fallen Kosten an?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modell-\u002FTool-\u002FRetrieval-Zuordnung pro Trace\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Was löst ein Rollback aus?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Definierter Schwellenwert für Qualität\u002FSicherheit\u002FKosten\u002FVerfügbarkeit\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Wie wird mit Anbieter-Deprecations umgegangen?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Migrations-\u002FFallback-Prozess\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Wie werden lokale Modelle betrieben?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Health-, Ressourcen-, Lade-\u002FEntlade- und Versionskontrollen\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-131\">Randfälle und Einschränkungen\u003C\u002Fh2>\n\u003Cp>Eine einfache Anwendung, die ein festes gehostetes Modell ohne Retrieval oder Tools aufruft, benötigt möglicherweise nur leichtgewichtige LLMOps: versionierten Prompt-Code, Evals, Modell-Pinning, grundlegendes Tracing und Anbieterüberwachung.\u003C\u002Fp>\n\u003Cp>Ein selbst gehostetes feinabgestimmtes Modell kann nahezu den vollständigen klassischen MLOps-Stack plus LLM-spezifische Anwendungsbewertung erfordern, wodurch die Grenze zwischen MLOps und LLMOps absichtlich unscharf wird.\u003C\u002Fp>\n\u003Cp>Eine Agentenplattform kann minimale Modelltrainingsoperationen, aber umfangreiche Runtime-Operationen haben, weil Fehler bei Tool-Auswahl, Zustand und Orchestrierung auftreten.\u003C\u002Fp>\n\u003Cp>Ein RAG-lastiges System kann operativ von Dokumentenaufnahme und Retrieval-Qualität statt von Modell-Serving dominiert werden.\u003C\u002Fp>\n\u003Cp>Die Terminologie wird sich weiterentwickeln. Die dauerhafte Architekturfrage ist nicht, welches „Ops“-Label gewinnt, sondern welche Artefakte Verhalten erzeugen und daher versioniert, bewertet, beobachtet und gesteuert werden müssen.\u003C\u002Fp>\n\u003Ch2 id=\"section-137\">Was würde diese Antwort ändern?\u003C\u002Fh2>\n\u003Cp>Wenn Foundation-Model-Anbieter perfekt stabiles Modellverhalten und langfristige Versionsunterstützung standardisieren, könnte Anbieter-\u002FSnapshot-Management operativ weniger bedeutsam werden.\u003C\u002Fp>\n\u003Cp>Wenn Anwendungen zunehmend Fine-Tuning oder Training übernehmen, rücken klassische MLOps-Anliegen wieder stärker in den Mittelpunkt.\u003C\u002Fp>\n\u003Cp>Das operative Prinzip bliebe: Jede Komponente, die das Produktionsverhalten wesentlich verändern kann, gehört in Lineage, Testing, Observability und Change Control.\u003C\u002Fp>\n\u003Ch2 id=\"section-141\">Verwandtes kanonisches Wissen\u003C\u002Fh2>\n\u003Cp>LLMOps steht unter AI Governance und Enterprise AI Architecture: Governance definiert, welche Änderungen Nachweise und Genehmigung erfordern, während LLMOps die operative Maschinerie bereitstellt, um diese Änderungen zu versionieren, zu bewerten, bereitzustellen und zu beobachten.\u003C\u002Fp>\n\u003Cp>Context Engineering und RAG sind operative Subdomänen innerhalb vieler LLM-Anwendungen, weil Kontext und Retrieval das Verhalten unabhängig vom Modell ändern können.\u003C\u002Fp>\n\u003Cp>Agentic AI erweitert LLMOps weiter in Trajektorien-, Berechtigungs- und Tool-Runtime-Operationen.\u003C\u002Fp>\n\u003Caside class=\"editorjs-referral my-6\">\u003Ca href=\"https:\u002F\u002Fstajic.de\u002Fde\u002Fblog\u002Fai-agent-memory-is-not-rag-how-to-separate-memory-retrieval-state-and-context\" class=\"flex flex-col sm:flex-row gap-4 rounded-xl border border-gray-200 dark:border-gray-700 p-4 transition hover:border-primary-500\">\u003Cdiv class=\"min-w-0 flex-1\">\u003Cstrong class=\"block text-lg text-gray-900 dark:text-gray-100\">KI-Agenten-Gedächtnis ist nicht RAG: Wie man Gedächtnis, Retrieval, Zustand und Kontext trennt\u003C\u002Fstrong>\u003Cp class=\"mt-2 text-sm text-gray-600 dark:text-gray-300\">Die operative Zuverlässigkeit verbessert sich, wenn Gedächtnis, Retrieval, Anwendungszustand und Modellkontext getrennte Lebenszyklusobjekte bleiben.\u003C\u002Fp>\u003Cspan class=\"mt-3 inline-flex text-sm font-medium text-primary-600 dark:text-primary-400\">Architekturartikel lesen →\u003C\u002Fspan>\u003C\u002Fdiv>\u003C\u002Fa>\u003C\u002Faside>\n\u003Caside class=\"editorjs-referral my-6\">\u003Ca href=\"https:\u002F\u002Fstajic.de\u002Fde\u002Fblog\u002Fthe-answer-validity-boundary-the-missing-layer-between-relevance-and-reliable-ai-answers\" class=\"flex flex-col sm:flex-row gap-4 rounded-xl border border-gray-200 dark:border-gray-700 p-4 transition hover:border-primary-500\">\u003Cdiv class=\"min-w-0 flex-1\">\u003Cstrong class=\"block text-lg text-gray-900 dark:text-gray-100\">Die Antwortgültigkeitsgrenze: Die fehlende Schicht zwischen Relevanz und zuverlässigen KI-Antworten\u003C\u002Fstrong>\u003Cp class=\"mt-2 text-sm text-gray-600 dark:text-gray-300\">LLMOps-Evaluierung sollte die Version, den Umfang und die Evidenzbedingungen bewahren, unter denen eine Antwort weiterhin gestützt wird.\u003C\u002Fp>\u003Cspan class=\"mt-3 inline-flex text-sm font-medium text-primary-600 dark:text-primary-400\">Antwortgültigkeitsgrenze lesen →\u003C\u002Fspan>\u003C\u002Fdiv>\u003C\u002Fa>\u003C\u002Faside>\n\u003Ch2 id=\"section-147\">Häufig gestellte Fragen\u003C\u002Fh2>\n\u003Csection class=\"editorjs-faq my-6 rounded-xl border border-gray-200 p-5 dark:border-gray-700\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">MLOps vs. LLMOps FAQ\u003C\u002Fh3>\u003Cdiv id=\"faq1\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Was ist der Unterschied zwischen MLOps und LLMOps?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">MLOps betreibt Machine-Learning-Systeme über Daten, Training, Deployment und Monitoring. LLMOps erweitert diese Praktiken auf LLM-Anwendungen, bei denen Prompts, Kontext, Retrieval, Anbieter, Tools und Evaluierungen das Verhalten ebenfalls wesentlich beeinflussen.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq2\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Ersetzt LLMOps MLOps?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Nein. LLMOps nutzt MLOps-Disziplinen wie CI\u002FCD, Lineage, Evaluierung, Deployment und Monitoring wieder und fügt LLM-spezifische operative Belange hinzu.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq3\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Benötigen LLM-Anwendungen kontinuierliches Training?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Nicht unbedingt. Viele verwenden externe Foundation-Modelle und setzen stattdessen auf kontinuierliche Evaluierung von Prompts, Modellen, Retrieval und Anwendungsverhalten. Feinabgestimmte oder selbst trainierte Systeme können dennoch Trainingspipelines erfordern.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq4\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Warum sind Evals in LLMOps so wichtig?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Generative Ausgaben sind offen und das Modellverhalten kann sich über Prompts, Snapshots und Kontext hinweg ändern. Evals liefern wiederholbare Belege dafür, dass ein Release weiterhin definierte Qualitäts- und Sicherheitskriterien erfüllt.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq5\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Was sollte in LLMOps versioniert werden?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Mindestens: Anwendungscode, Modell\u002FAnbieter\u002FVersion, Prompts, Eval-Datensätze\u002FScorer, Retrieval-Konfiguration\u002FIndizes, Tool-Schemas, Kontextregeln und relevante Sicherheits-\u002FBerechtigungskonfiguration.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq6\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Reicht Prompt-Versionierung aus?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Nein. Derselbe Prompt kann sich mit einem anderen Modell, Retrieval-Set, einer anderen Kontextreihenfolge, Tool-Oberfläche oder einem anderen Anbieter anders verhalten.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq7\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Was ist GenAIOps?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">GenAIOps ist ein weiterer Branchenbegriff für den Betrieb generativer KI-Anwendungen. Einige Anbieter verwenden ihn synonym oder als breiteres Label als LLMOps.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq8\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Wie überwacht man eine LLM-Anwendung?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Überwachen Sie End-to-End-Traces einschließlich Modellaufrufen, Prompts\u002FKontext, Retrieval, Tools, Latenz, Token\u002FKosten, Qualitätsstichproben, Sicherheit und endgültigen Aufgabenergebnissen.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq9\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Können lokale LLMs LLMOps-Praktiken nutzen?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Ja. Lokale Modelle bringen eigene operative Belange mit sich, wie Modelldateien, Hardware\u002FVRAM, Laden\u002FEntladen, Laufzeitintegrität, Quantisierung und Upgrade-Management.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fsection>\n\u003Ch2 id=\"section-149\">Glossar\u003C\u002Fh2>\n\u003Csection class=\"editorjs-glossary my-6 rounded-xl border border-gray-200 dark:border-gray-700 p-5\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">Wichtige MLOps- und LLMOps-Begriffe\u003C\u002Fh3>\u003Cdl>\u003Cdiv id=\"mlops\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">MLOps\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Engineering-Praktiken zum Erstellen, Bereitstellen, Überwachen und Warten von Machine-Learning-Systemen und deren Daten-\u002FModelllebenszyklus.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"llmops\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">LLMOps\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Operative Praktiken für Produktionsanwendungen, deren Verhalten wesentlich von großen Sprachmodellen und umgebenden Prompts, Kontext, Retrieval, Tools und Laufzeit abhängt.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"genaiops\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">GenAIOps\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Operative Disziplin für generative KI-Anwendungen; wird oft als breiteres oder alternatives Label für LLMOps verwendet.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"continuous-training\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Kontinuierliches Training\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Automatisiertes oder wiederholtes Neutraining und Serving von ML-Modellen, wenn sich Daten oder Implementierungen ändern.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"continuous-evaluation\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Kontinuierliche Evaluierung\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Wiederholte Evaluierung des Verhaltens von Kandidaten- und Produktions-KI anhand versionierter Datensätze und Kriterien.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"model-snapshot\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Modell-Snapshot\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Eine konkrete Version eines gehosteten oder paketierten Modells, deren Verhalten getestet und referenziert werden kann.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"application-lineage\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Anwendungs-Lineage\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Nachvollziehbare Beziehung zwischen Code, Modell\u002FAnbieter, Prompts, Daten\u002FRetrieval, Tools, Laufzeit und Release-Konfiguration.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"trace\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Trace\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Strukturierte Aufzeichnung einer Anwendungsausführung mit Spans wie Modellaufrufen, Retrievals und Tool-Operationen.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"eval-dataset\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Eval-Datensatz\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Versionierter Satz repräsentativer Eingaben, Erwartungen und optional Traces\u002FAusgaben zur Verhaltensmessung.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"llm-judge\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">LLM-Richter\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Ein Sprachmodell, das als Evaluator für qualitative oder semantische Kriterien verwendet wird; es ist selbst eine versionierte Evaluierungsabhängigkeit.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"behavioral-regression\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Verhaltensregression\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Eine Verschlechterung der Anwendungsausgabe oder -trajektorie, obwohl Schnittstellen und Code weiterhin erfolgreich ausgeführt werden.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"provider-routing\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Anbieter-Routing\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Richtlinie zur Auswahl unter verfügbaren Modellanbietern\u002FEndpunkten nach Fähigkeit, Kosten, Latenz, Datenschutz oder Verfügbarkeit.\u003C\u002Fdd>\u003C\u002Fdiv>\u003C\u002Fdl>\u003C\u002Fsection>\n\u003Ch2 id=\"section-151\">Fazit\u003C\u002Fh2>\n\u003Cp>MLOps und LLMOps teilen dasselbe Engineering-Ziel: KI-Systeme ausreichend reproduzierbar, testbar und beobachtbar zu machen, um sie zuverlässig in der Produktion zu betreiben.\u003C\u002Fp>\n\u003Cp>Der Unterschied liegt in der Form des Systems. Klassisches MLOps konzentriert sich oft auf Training und Serving von Modellartefakten; LLMOps muss einen Verhaltens-Stack betreiben, in dem Modell-Snapshots, Prompts, Kontext, Retrieval, Tools, Berechtigungen und Anbieter sich unabhängig ändern können.\u003C\u002Fp>\n\u003Cp>Die kürzeste nützliche Regel lautet: Versionieren, evaluieren und beobachten Sie alles, was das Verhalten der LLM-Anwendung wesentlich ändern kann – nicht nur das Modell.\u003C\u002Fp>\n\u003Ch2 id=\"section-155\">Primärquellen und aktuelle Dokumentation\u003C\u002Fh2>\n\u003Cp>Die folgenden Quellen untermauern die MLOps-Basis und die aktuellen operativen Muster für LLM- und Agenten-Anwendungen. Projektabschnitte sind originale Implementierungsbelege und bewusst enger gefasst als Aussagen über eine vollständige LLMOps-Plattform.\u003C\u002Fp>\n\u003Ca href=\"https:\u002F\u002Fdocs.cloud.google.com\u002Farchitecture\u002Fmlops-continuous-delivery-and-automation-pipelines-in-machine-learning\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">Google Cloud — MLOps: Continuous delivery and automation pipelines\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Referenzarchitektur, die CI, CD, kontinuierliches Training, Modellregistry, Metadaten, Serving und Monitoring für ML-Systeme beschreibt.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops02-bp04.html\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">AWS Machine Learning Lens — Model lineage\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktuelle Anleitung zur Nachverfolgung von Code, Daten, Modellen, Umgebungen und Infrastruktur über ML-Releases hinweg.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops06-bp02.html\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">AWS Machine Learning Lens — Model observability and tracking\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktuelle Anleitung für Produktionsmodell-Monitoring, Drift, Endpoint-Integrität und Lineage.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fmachine-learning\u002Fprompt-flow\u002Fhow-to-end-to-end-llmops-with-prompt-flow\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">Microsoft Azure — GenAIOps \u002F LLMOps lifecycle\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Offizielle Anleitung, die GenAIOps, manchmal LLMOps genannt, über Initialisierung, Experimentierung, Evaluierung\u002FVerfeinerung und Deployment beschreibt.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002F\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">MLflow — Agents and LLM applications\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktuelle GenAI-Betriebsdokumentation zu Tracing, Evaluierung, Prompts und Produktionsbeobachtbarkeit für LLM-Anwendungen und Agenten.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fwww.mlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Feval-monitor\u002Frunning-evaluation\u002Ftraces\u002F\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">MLflow — Evaluating production traces\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktuelle Anleitung zur Evaluierung vollständiger LLM-\u002FAgenten-Traces, einschließlich Retrieval- und Tool-Aufruf-Trajektorien.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Fprompt-registry\u002Fevaluate-prompts\u002F\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">MLflow — Bewertung von Prompts\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktueller Workflow zur Bewertung von Prompts\u002FModellen unter Verwendung versionierter Prompts, Datensätze, Scorer und Traces.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Freference\u002Foverview\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">OpenAI API — Versionierung und Modell-Snapshots\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktuelle API-Empfehlung, gepinnte Modellversionen und Evals zu verwenden, da sich das Prompting-Verhalten zwischen Snapshots ändern kann.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fguides\u002Fprompting\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">OpenAI — Prompting\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktuelle Empfehlung, Produktions-Prompts wie Anwendungscode zu behandeln, sie über Quellcodeverwaltung zu versionieren und Änderungen mit Tests und Bewertungsprüfungen abzudecken.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fdeprecations\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">OpenAI — Deprecations\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktuelle Anbieter-Lebenszyklus-Nachweise, die die Stilllegung von Modellen und Plattformoberflächen als betriebliche Abhängigkeit zeigen.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdevelopers.openai.com\u002Fcookbook\u002Fexamples\u002Fevaluation\u002Fmoving-from-openai-evals-to-promptfoo\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">OpenAI — Evaluierungsworkflows zu Promptfoo verlagern\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Aktuelle Migrationsempfehlung aus 2026, die veranschaulicht, warum Bewertungsressourcen portabel bleiben sollten, wenn sich die Anbieter-Tooling ändert.\u003C\u002Fp>\u003C\u002Fa>",{"time":211,"blocks":212,"version":1557},1791487379909,[213,219,227,234,241,247,255,260,265,270,275,280,285,290,295,300,305,310,315,347,352,357,362,367,372,420,425,430,435,440,445,495,500,505,510,515,520,525,530,535,540,545,550,555,560,565,570,575,580,585,590,595,604,609,614,619,624,631,636,641,646,651,656,661,666,671,676,681,686,691,699,704,709,714,719,724,729,734,739,744,749,754,759,764,799,804,809,814,819,824,829,834,839,844,879,884,889,894,899,904,909,914,919,924,929,934,939,944,949,954,959,964,969,974,979,984,989,994,999,1004,1009,1041,1047,1052,1096,1101,1139,1144,1186,1191,1241,1246,1251,1256,1261,1266,1271,1276,1281,1286,1291,1296,1301,1306,1311,1319,1327,1332,1374,1379,1427,1432,1437,1442,1447,1452,1457,1467,1476,1485,1494,1503,1512,1521,1530,1539,1548],{"id":214,"data":215,"type":217,"tunes":218},"intro",{"text":216},"MLOps ist die Ingenieursdisziplin für die zuverlässige Entwicklung, Bereitstellung, Versionierung und den Betrieb von Machine-Learning-Systemen; LLMOps erweitert diese Disziplin auf Anwendungen, die auf großen Sprachmodellen basieren, bei denen das Produktionsverhalten nicht nur von einem Modellartefakt abhängt, sondern auch von Prompts, Kontext, Retrieval, Anbieter-\u002FModellversionen, Tool-Aufrufen, Sicherheitskontrollen und Evaluierungspipelines. LLMOps ersetzt MLOps nicht. Es verändert die operative Einheit von „einem Modell plus Serving-Pipeline“ hin zu „einer sich entwickelnden LLM-Anwendung, deren Verhalten aus mehreren unabhängig voneinander veränderlichen Komponenten entsteht“.","paragraph",{},{"id":220,"data":221,"type":225,"tunes":226},"direct",{"body":222,"title":223,"variant":224},"\u003Cstrong>MLOps betreibt ML-Systeme. LLMOps betreibt LLM-Anwendungen.\u003C\u002Fstrong>\u003Cbr>\u003Cbr>Klassisches MLOps konzentriert sich üblicherweise auf Datenpipelines, Training, Validierung, Modellregistrierung, Bereitstellung, Drift und Neutraining. LLMOps behält diese Disziplinen bei, wo sie relevant sind, fügt aber oft Prompt-\u002FKontextversionierung, Modell-\u002FAnbieterabstraktion, RAG-Indizes, Agent-\u002FTool-Traces, semantische Evaluierungen, Sicherheitstests, Token-\u002FKostenüberwachung und Regressionstests über sich schnell ändernde Modell-Snapshots hinzu.","Direkte Antwort","info","callout",{},{"id":228,"data":229,"type":225,"tunes":233},"boundary",{"body":230,"title":231,"variant":232},"Eine Produktions-LLM-Anwendung kann selbst dann fehlschlagen, wenn der Prompt unverändert bleibt: Der Anbieter kann einen Modell-Snapshot ändern, ein RAG-Korpus kann veralten, ein Reranker kann sich verschlechtern, Tool-Berechtigungen können sich ändern, die Kontextzusammenstellung kann Belege auslassen oder ein Agent kann eine falsche Trajektorie einschlagen. LLMOps muss daher das System um das Modell herum beobachten und versionieren, nicht nur den Prompt-Text.","LLMOps ist nicht nur Prompt-Management","warning",{},{"id":235,"data":236,"type":225,"tunes":240},"term-note",{"body":237,"title":238,"variant":239},"\u003Cstrong>LLMOps\u003C\u002Fstrong>, \u003Cstrong>GenAIOps\u003C\u002Fstrong> und verwandte Begriffe sind weit verbreitete Engineering-Bezeichnungen, aber sie sind kein einheitlicher formaler Standard mit einem einzigen kanonischen Lebenszyklus. Microsoft beschreibt GenAIOps derzeit als „manchmal LLMOps genannt“, während MLflow operative Tooling um Agenten und LLM-Anwendungen gruppiert. Dieser Artikel verwendet LLMOps als praktischen Architekturbegriff für den Betrieb von Produktionssystemen, deren Verhalten maßgeblich von LLMs abhängt.","Terminologische Abgrenzung","note",{},{"id":242,"data":243,"type":225,"tunes":246},"current",{"body":244,"title":245,"variant":239},"Die operative Oberfläche verändert sich schnell. OpenAI empfiehlt derzeit, Modell-Snapshots zu pinnen und Evals auszuführen, weil sich das Prompting-Verhalten zwischen Snapshots ändern kann, und mehrere ältere plattformspezifische Prompt-\u002FEval-Oberflächen werden 2026 eingestellt. Die stabile architektonische Lehre ist, Prompts, Tests und Evals portabel zu halten und mit der Anwendung zu versionieren, anstatt von einem Dashboard-Objektmodell eines Anbieters abhängig zu sein.","Hinweis zu aktuellen Quellen — 8. Oktober 2026",{},{"id":248,"data":249,"type":253,"tunes":254},"toc",{"title":250,"maxLevel":251,"minLevel":252},"Inhalt",3,2,"tableOfContents",{},{"id":256,"data":257,"type":41,"tunes":259},"h-meaning",{"text":258,"level":252},"Was MLOps wirklich bedeutet",{},{"id":261,"data":262,"type":217,"tunes":264},"p-mlops-1",{"text":263},"MLOps wendet Software-Engineering- und Betriebsdisziplin auf Machine-Learning-Systeme an. Die Produktionsherausforderung ist breiter als das Trainieren eines Modells: Datenerfassung, Datenvalidierung, Experimentieren, Reproduzierbarkeit, Modellbewertung, Bereitstellung, Infrastruktur und Monitoring müssen alle zusammenwirken.",{},{"id":266,"data":267,"type":217,"tunes":269},"p-mlops-2",{"text":268},"Googles MLOps-Architekturleitfaden rahmt die Disziplin um kontinuierliche Integration, kontinuierliche Bereitstellung und kontinuierliches Training. CI validiert nicht nur Code, sondern auch Daten, Schemata und Modelle; CD stellt ML-Pipelines und Vorhersagedienste bereit; CT kann Modelle neu trainieren und erneut bereitstellen, wenn sich Daten oder Implementierungen ändern.",{},{"id":271,"data":272,"type":217,"tunes":274},"p-mlops-3",{"text":273},"AWS-Leitfaden ergänzt dieselben operativen Belange aus einem anderen Blickwinkel: Modellherkunft, Modell-\u002FVersionsnachverfolgbarkeit, Drift-Monitoring und Produktionsqualitätsüberwachung sind Kernbestandteile, um ML-Systeme nach der Bereitstellung zuverlässig zu halten.",{},{"id":276,"data":277,"type":41,"tunes":279},"h-llmops",{"text":278,"level":252},"Was sich ändert, wenn das Modell ein LLM ist",{},{"id":281,"data":282,"type":217,"tunes":284},"p-llmops-1",{"text":283},"Große Sprachmodelle verändern das Produktionsproblem, weil die Anwendung oft nicht den vollständigen Modelltrainingslebenszyklus besitzt. Ein Team kann eine gehostete Modell-API aufrufen, ein offenes Modell lokal ausführen, zwischen Anbietern wechseln oder mehrere Modelle für verschiedene Aufgaben verwenden.",{},{"id":286,"data":287,"type":217,"tunes":289},"p-llmops-2",{"text":288},"Das Modell ist daher nur eine versionierte Abhängigkeit innerhalb eines größeren Verhaltenssystems. Prompts, Retrieval-Ergebnisse, Kontextreihenfolge, Tools, Modell-Snapshot, Temperatur-\u002FReasoning-Einstellungen, Sicherheitsfilter und Laufzeitorchestrierung können alle die Ausgabe verändern.",{},{"id":291,"data":292,"type":217,"tunes":294},"p-llmops-3",{"text":293},"Dies schafft eine breitere operative Frage: Welche Kombination aus Modell, Kontext, Daten, Prompt, Tools und Laufzeit hat dieses Verhalten erzeugt? LLMOps existiert, um diese Frage beantwortbar und die Antwort reproduzierbar genug für Ingenieursarbeit zu machen.",{},{"id":296,"data":297,"type":41,"tunes":299},"h-simple",{"text":298,"level":252},"Das einfachste Beispiel",{},{"id":301,"data":302,"type":217,"tunes":304},"p-simple-1",{"text":303},"Angenommen, eine Anwendung beantwortet interne Richtlinienfragen.",{},{"id":306,"data":307,"type":217,"tunes":309},"p-simple-2",{"text":308},"In einer klassischen ML-Rahmung könnten Sie einen trainierten Klassifikator versionieren, ihn bereitstellen und die Vorhersagequalität überwachen. In einer LLM-Anwendung könnte die Antwort von einem gehosteten Modell-Snapshot, einem System-Prompt, einem Embedding-Modell, einem Vektorindex, Retrieval-Filtern, einem Reranker und dem schließlich ausgewählten Kontext abhängen.",{},{"id":311,"data":312,"type":217,"tunes":314},"p-simple-3",{"text":313},"Das Ändern einer dieser Komponenten kann die endgültige Antwort verändern, selbst wenn der Anwendungsendpunkt und die Benutzerfrage identisch bleiben.",{},{"id":316,"data":317,"type":345,"tunes":346},"simple-flow",{"steps":318,"title":343,"orientation":344},[319,322,325,328,331,334,337,340],{"label":320,"description":321},"1. Eine Komponente ändern","Prompt, Modell, Anbieter, Retrieval-Einstellung, Tool-Schema oder Anwendungscode ändern sich.",{"label":323,"description":324},"2. Deterministische Tests ausführen","Schemata, Berechtigungen, Tool-Verträge, Retrieval-Filter und Anwendungsverhalten validieren.",{"label":326,"description":327},"3. Verhaltens-Evals ausführen","Repräsentative Ausgaben, Retrieval-Qualität und Agent-\u002FTool-Trajektorien mit Akzeptanzkriterien vergleichen.",{"label":329,"description":330},"4. Kosten und Latenz vergleichen","Token-Nutzung, Modellaufrufe, Retrieval-\u002FTool-Overhead und Antwortlatenz messen.",{"label":332,"description":333},"5. Kontrollierte Version bereitstellen","Die konkrete Anwendungskonfiguration mit aufgezeichneten Modell-\u002FAnbieter-Versionen ausliefern.",{"label":335,"description":336},"6. Produktionsverhalten nachverfolgen","Relevante Modell-, Retrieval-, Tool- und Laufzeit-Spans erfassen.",{"label":338,"description":339},"7. Produktions-Traces auswerten","Echte Ausführungen auf Qualität, Fundierung, Sicherheit und Aufgabenerfolg stichprobenartig prüfen.",{"label":341,"description":342},"8. Zurückrollen oder iterieren","Regressionsnachweise und Betriebssignale nutzen, um das nächste Release zu entscheiden.","Ein typischer LLMOps-Release-Pfad","auto","processFlow",{},{"id":348,"data":349,"type":41,"tunes":351},"h-stops",{"text":350,"level":252},"Wo das einfache Beispiel endet",{},{"id":353,"data":354,"type":217,"tunes":356},"p-stops-1",{"text":355},"Einige LLM-Systeme trainieren oder fine-tunen noch ihre eigenen Modelle, sodass traditionelle MLOps-Praktiken wie Trainingspipelines, Modell-Registry und Datenherkunft weiterhin direkt relevant bleiben.",{},{"id":358,"data":359,"type":217,"tunes":361},"p-stops-2",{"text":360},"Andere Systeme verwenden nur externe Foundation-Model-APIs und führen nie kontinuierliches Training durch. Ihre Hauptbetriebsaufgabe ist Anwendungsbewertung, Modell-\u002FAnbieter-Änderungsmanagement, Prompt-\u002FKontext-Versionierung, Retrieval-Qualität und Observability.",{},{"id":363,"data":364,"type":217,"tunes":366},"p-stops-3",{"text":365},"Es gibt daher keine einzige universelle „LLMOps-Pipeline“. Der genaue Lebenszyklus hängt davon ab, ob Sie trainieren, fine-tunen, selbst hosten, externes Wissen abrufen, Agents ausführen oder von verwalteten Modell-APIs abhängen.",{},{"id":368,"data":369,"type":41,"tunes":371},"h-compare",{"text":370,"level":252},"MLOps vs. LLMOps",{},{"id":373,"data":374,"type":418,"tunes":419},"main-comparison",{"rows":375,"title":409,"layout":410,"columns":411},[376,381,385,389,393,397,401,405],{"id":377,"label":378,"values":379},"unit","Primäre operative Einheit",[380,380],"",{"id":382,"label":383,"values":384},"model","Modell-Eigentümerschaft",[380,380],{"id":386,"label":387,"values":388},"change","Typische Änderung",[380,380],{"id":390,"label":391,"values":392},"eval","Bewertung",[380,380],{"id":394,"label":395,"values":396},"monitor","Produktionsüberwachung",[380,380],{"id":398,"label":399,"values":400},"training","Kontinuierliches Training",[380,380],{"id":402,"label":403,"values":404},"registry","Versionierte Artefakte",[380,380],{"id":406,"label":407,"values":408},"rollback","Rollback-Ziel",[380,380],"Was gleich bleibt und was sich erweitert","table",[412,415],{"id":413,"label":414},"mlops","MLOps",{"id":416,"label":417},"llmops","LLMOps","comparison",{},{"id":421,"data":422,"type":41,"tunes":424},"h-extension",{"text":423,"level":252},"LLMOps erweitert MLOps, statt es zu ersetzen",{},{"id":426,"data":427,"type":217,"tunes":429},"p-extension-1",{"text":428},"Die Kernbetriebsprinzipien verschwinden nicht: Quellcodeverwaltung, CI\u002FCD, Reproduzierbarkeit, Herkunft, Deployment-Kontrollen, Monitoring, Rollback und messbare Akzeptanzkriterien bleiben unerlässlich.",{},{"id":431,"data":432,"type":217,"tunes":434},"p-extension-2",{"text":433},"Die Erweiterung besteht darin, dass sich mehr verhaltensbestimmende Artefakte nun außerhalb der Modellgewichte befinden. Ein verwaltetes Foundation-Modell kann sein Verhalten durch Snapshot-Upgrades ändern, während sich die Anwendungsausgabe durch Prompt- oder Retrieval-Änderungen ohne jegliches Modell-Retraining ändern kann.",{},{"id":436,"data":437,"type":217,"tunes":439},"p-extension-3",{"text":438},"Deshalb ist die nützliche Hierarchie normalerweise DevOps → MLOps → LLMOps\u002FGenAIOps als zunehmend spezialisierte operative Anliegen, nicht drei sich gegenseitig ausschließende Praktiken.",{},{"id":441,"data":442,"type":41,"tunes":444},"h-artifacts",{"text":443,"level":252},"Was muss in LLMOps versioniert werden?",{},{"id":446,"data":447,"type":410,"tunes":494},"artifact-table",{"content":448,"stretched":42,"withHeadings":13},[449,452,455,458,461,464,467,470,473,476,479,482,485,488,491],[450,451],"Artefakt","Warum es wichtig ist",[453,454],"Anwendungscode","Definiert Orchestrierung, Validierung, Wiederholungen und Geschäftsverhalten",[456,457],"Modellfamilie + Snapshot\u002FVersion","Unterschiedliche Snapshots können unterschiedliches Verhalten erzeugen",[459,460],"Anbieter \u002F Endpunkt","Ändert Datenfluss, Latenz, Limits, Preisgestaltung und Verfügbarkeit",[462,463],"Prompt-\u002FInstruktionscode","Ändert das Modellverhalten auch bei gleichem Modell",[465,466],"Generierungs-\u002FReasoning-Parameter","Können Determinismus, Latenz, Tiefe und Kosten verändern",[468,469],"Eval-Datensatz","Definiert, wogegen „gut genug“ getestet wird",[471,472],"Scorer \u002F Bewerter","Definieren, wie Qualität gemessen wird",[474,475],"Embedding-Modell","Ändert Vektorrepräsentation und Retrieval-Verhalten",[477,478],"Chunking-\u002FIndex-Konfiguration","Ändert, was abgerufen werden kann",[480,481],"Reranker \u002F Retrieval-Fusion","Ändert die Ergebnisreihenfolge",[483,484],"Tool-Schemata","Ändern, was das Modell anfordern kann und wie",[486,487],"Berechtigungsprofil","Ändert, welche Tool-Aktionen tatsächlich ausgeführt werden dürfen",[489,490],"Kontext-Assemblierungsregeln","Ändern, welche Evidenz und welcher Zustand das Modell erreichen",[492,493],"Sicherheits-\u002FGuardrail-Konfiguration","Ändert erlaubtes oder blockiertes Verhalten",{},{"id":496,"data":497,"type":41,"tunes":499},"h-model-version",{"text":498,"level":252},"Modell-Snapshots werden zu Release-Abhängigkeiten",{},{"id":501,"data":502,"type":217,"tunes":504},"p-model-version-1",{"text":503},"Bei gehosteten LLMs kontrolliert das Team möglicherweise nicht das Modelltraining, aber es kontrolliert weiterhin, welches Modell oder welchen Snapshot die Anwendung aufruft.",{},{"id":506,"data":507,"type":217,"tunes":509},"p-model-version-2",{"text":508},"Die aktuelle API-Richtlinie von OpenAI warnt ausdrücklich, dass sich das Prompting-Verhalten zwischen Modell-Snapshots ändern kann, und empfiehlt, Produktionsanwendungen auf bestimmte Snapshots festzulegen, wo Konsistenz wichtig ist, und dann beim Upgrade Evals auszuführen.",{},{"id":511,"data":512,"type":217,"tunes":514},"p-model-version-3",{"text":513},"Die operative Konsequenz ist eindeutig: Modell-Upgrades sollten als Anwendungs-Releases behandelt werden, nicht als unsichtbare Infrastrukturwartung.",{},{"id":516,"data":517,"type":41,"tunes":519},"h-provider",{"text":518,"level":252},"Der Anbieter-Lebenszyklus wird Teil des Betriebs",{},{"id":521,"data":522,"type":217,"tunes":524},"p-provider-1",{"text":523},"LLM-Anwendungen hängen oft von Anbieter-Rate-Limits, Deprecation-Zeitplänen, API-Semantik, Kontextlimits, Datenverarbeitungsregeln und Preisen ab.",{},{"id":526,"data":527,"type":217,"tunes":529},"p-provider-2",{"text":528},"Ein Anbieter kann ein Modell als veraltet einstufen, während Ihr Anwendungscode unverändert bleibt. Der aktuelle Deprecation-Zeitplan von OpenAI enthält beispielsweise Auslaufdaten für 2026 für ältere Modell-Snapshots und Plattformoberflächen.",{},{"id":531,"data":532,"type":217,"tunes":534},"p-provider-3",{"text":533},"LLMOps benötigt daher zusätzlich zur Modellqualitätsüberwachung auch die Verfolgung des Anbieter-Lebenszyklus, Migrationstests und Fallback-Entscheidungen.",{},{"id":536,"data":537,"type":41,"tunes":539},"h-prompt",{"text":538,"level":252},"Prompts verhalten sich wie Produktionscode",{},{"id":541,"data":542,"type":217,"tunes":544},"p-prompt-1",{"text":543},"Prompts sind ausführbare Verhaltenskonfiguration. Kleine Änderungen können die Ausgabequalität, die Werkzeugauswahl und die Richtlinieninterpretation verändern.",{},{"id":546,"data":547,"type":217,"tunes":549},"p-prompt-2",{"text":548},"Die aktuelle Empfehlung von OpenAI rät dazu, Produktions-Prompts im Anwendungscode zu speichern, Prompt-Änderungen über Pull Requests zu überprüfen, typisierte Eingaben zu verwenden und Änderungen mit Tests und Evaluierungsprüfungen abzudecken.",{},{"id":551,"data":552,"type":217,"tunes":554},"p-prompt-3",{"text":553},"Dadurch ähnelt die Prompt-Versionierung weniger dem Bearbeiten von Marketingtexten und mehr dem Ändern einer Funktion, deren Ausgabe probabilistisch und modellabhängig ist.",{},{"id":556,"data":557,"type":41,"tunes":559},"h-context",{"text":558,"level":252},"Context Engineering wird zu einem operativen Anliegen",{},{"id":561,"data":562,"type":217,"tunes":564},"p-context-1",{"text":563},"Das Produktionsmodell erhält selten nur einen statischen Prompt. Es kann Gesprächsverlauf, abgerufene Dokumente, Werkzeugausgaben, Speicher, aktuellen Anwendungszustand und Richtlinienanweisungen erhalten.",{},{"id":566,"data":567,"type":217,"tunes":569},"p-context-2",{"text":568},"LLMOps muss daher die Kontextzusammenstellung beobachten: welche Belege ausgewählt wurden, welche Zustandsversion aktuell war, ob eine Kürzung stattfand und ob wichtige Anweisungen die Komprimierung überlebt haben.",{},{"id":571,"data":572,"type":217,"tunes":574},"p-context-3",{"text":573},"Eine Modellregression und eine Kontextregression können bei der endgültigen Antwort identisch aussehen. Das Nachverfolgen des tatsächlichen Kontextpfads ermöglicht es dem Team, sie zu unterscheiden.",{},{"id":576,"data":577,"type":41,"tunes":579},"h-rag",{"text":578,"level":252},"RAG schafft einen eigenen operativen Lebenszyklus",{},{"id":581,"data":582,"type":217,"tunes":584},"p-rag-1",{"text":583},"Ein RAG-System führt eine zweite Produktionspipeline neben der Modellinferenz ein: Ingestion, Extraktion, Chunking, Metadaten, Embeddings, Indizes, Retrieval, Reranking und Kontextauswahl.",{},{"id":586,"data":587,"type":217,"tunes":589},"p-rag-2",{"text":588},"Der Wissenskorpus kann sich täglich ändern, selbst wenn Modell und Prompt unverändert bleiben. Ein veralteter Index oder ein defekter Metadatenfilter kann daher die Antwortqualität beeinträchtigen, ohne dass ein Modell-Drift vorliegt.",{},{"id":591,"data":592,"type":217,"tunes":594},"p-rag-3",{"text":593},"LLMOps für RAG sollte Korpus-\u002FIndexversion, Embedding-Modell, Chunking-Richtlinie, Retrieval-Konfiguration, Quellenaktualität und Retrieval-Metriken getrennt von der Generierungsqualität verfolgen.",{},{"id":596,"data":597,"type":602,"tunes":603},"ref-rag-diagnostic",{"url":598,"title":599,"excerpt":600,"ctaLabel":601},"https:\u002F\u002Fstajic.de\u002Fde\u002Fblog\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method","RAG fehlgeschlagen – aber welche Ebene ist tatsächlich fehlgeschlagen? Eine diagnostische Methode","Eine Produktions-LLM-Pipeline benötigt separate Observability für Quellenabdeckung, Retrieval, Ranking, Kontextzusammenstellung und Generierung.","Die RAG-Diagnosemethode lesen","referralArticle",{},{"id":605,"data":606,"type":41,"tunes":608},"h-evals",{"text":607,"level":252},"Evals ersetzen „sieht gut aus für mich“ durch Release-Beweise",{},{"id":610,"data":611,"type":217,"tunes":613},"p-eval-1",{"text":612},"Generative Ausgaben sind oft offen, sodass Exact-Match-Tests für viele Aufgaben unzureichend sind. LLMOps fügt Evaluierungsdatensätze und Scorer hinzu, die Aufgabenerfolg, Korrektheit, Sicherheit, Fundiertheit, Stil oder domänenspezifische Akzeptanzkriterien messen können.",{},{"id":615,"data":616,"type":217,"tunes":618},"p-eval-2",{"text":617},"Der aktuelle GenAI-Evaluierungsstack von MLflow unterstützt versionierte Evaluierungsdatensätze, Prompt-\u002FModellvergleiche, benutzerdefinierte Scorer und die Evaluierung über vollständige Traces.",{},{"id":620,"data":621,"type":217,"tunes":623},"p-eval-3",{"text":622},"Die stärkste Praxis ist evaluierungsgetriebene Entwicklung: Definieren Sie repräsentative Fälle und Akzeptanzkriterien vor oder parallel zu Änderungen und vergleichen Sie dann Releases anhand derselben Evidenz.",{},{"id":625,"data":626,"type":225,"tunes":630},"eval-rule",{"body":627,"title":628,"variant":629},"Ein Deployment sollte nicht allein deshalb als gleichwertig betrachtet werden, weil der API-Vertrag noch funktioniert. Wenn sich Prompt, Modell, Retrieval oder Tools geändert haben, sollte die behavioral Regression Suite erneut ausgeführt werden.","Verhaltensänderungen erfordern Verhaltenstests","success",{},{"id":632,"data":633,"type":41,"tunes":635},"h-judges",{"text":634,"level":252},"LLM-as-a-judge ist nützlich, aber keine Ground Truth",{},{"id":637,"data":638,"type":217,"tunes":640},"p-judge-1",{"text":639},"LLM-Richter können die Evaluierung für Qualitäten skalieren, die teuer als deterministische Assertions zu kodieren sind, wie Relevanz, Tonalität oder Groundedness.",{},{"id":642,"data":643,"type":217,"tunes":645},"p-judge-2",{"text":644},"Der Richter ist jedoch ein weiteres Modell mit eigenem Bias, eigener Version und eigenem Prompt. Die Richter-Konfiguration sollte daher versioniert und gegen menschliche oder deterministische Referenzfälle kalibriert werden, wo Konsequenzen wichtig sind.",{},{"id":647,"data":648,"type":217,"tunes":650},"p-judge-3",{"text":649},"Eine Produktions-Evaluierung kann deterministische Checks, referenzbasierte Metriken, Modell-Richter und menschliche Überprüfung kombinieren, anstatt eine einzige Metrik alle Qualitätsdimensionen repräsentieren zu lassen.",{},{"id":652,"data":653,"type":41,"tunes":655},"h-tracing",{"text":654,"level":252},"Tracing wird wichtiger als Endpoint-Logs",{},{"id":657,"data":658,"type":217,"tunes":660},"p-trace-1",{"text":659},"Traditionelle API-Logs können Ihnen sagen, dass eine Anfrage zwei Sekunden gedauert hat und HTTP 200 zurückgegeben wurde. Sie können Ihnen nicht sagen, welche abgerufenen Chunks ausgewählt wurden, welches Tool der Agent aufgerufen hat oder welcher Modell-Span die meisten Tokens verbraucht hat.",{},{"id":662,"data":663,"type":217,"tunes":665},"p-trace-2",{"text":664},"Das aktuelle GenAI-Tracing von MLflow erfasst Prompts, Retrievals, Tool-Aufrufe und Anwendungs-Spans, und sein Produktions-Evaluierungsablauf kann Zwischeninformationen der Trajektorie bewerten, nicht nur den finalen Text.",{},{"id":667,"data":668,"type":217,"tunes":670},"p-trace-3",{"text":669},"Dies ist ein bedeutender LLMOps-Wandel: Observability folgt dem Verhaltensgraphen der Anwendung, nicht nur dem Serving-Endpoint.",{},{"id":672,"data":673,"type":41,"tunes":675},"h-agent",{"text":674,"level":252},"Agenten erweitern LLMOps zu Runtime-Operationen",{},{"id":677,"data":678,"type":217,"tunes":680},"p-agent-1",{"text":679},"Eine agentische Anwendung kann mehrere Modellaufrufe, Tool-Aufrufe und Zustandsübergänge ausführen, bevor sie ein Ergebnis liefert.",{},{"id":682,"data":683,"type":217,"tunes":685},"p-agent-2",{"text":684},"Der Betrieb von Agenten erfordert daher Schrittanzahlen, Tool-Call-Traces, Berechtigungsverweigerungen, Wiederholungen, Schleifenerkennung, menschliche Genehmigungen und verifizierten Endzustand zusätzlich zu gewöhnlichen Modell-Latenz- und Token-Metriken.",{},{"id":687,"data":688,"type":217,"tunes":690},"p-agent-3",{"text":689},"Eine korrekte finale Antwort kann eine schlechte Trajektorie verbergen, daher muss die Agenten-Evaluierung sowohl den Pfad als auch das Ergebnis untersuchen.",{},{"id":692,"data":693,"type":602,"tunes":698},"ref-agent-reliability",{"url":694,"title":695,"excerpt":696,"ctaLabel":697},"https:\u002F\u002Fstajic.de\u002Fde\u002Fblog\u002Fai-agent-reliability-why-the-final-answer-is-not-enough","Zuverlässigkeit von KI-Agenten: Warum die finale Antwort nicht ausreicht","Warum die Produktions-Evaluierung von Agenten Tool-Aufrufe, Zustandsübergänge, Genehmigungen und Wiederherstellbarkeit umfassen muss.","Lesen Sie den Artikel zur Agenten-Zuverlässigkeit",{},{"id":700,"data":701,"type":41,"tunes":703},"h-cost",{"text":702,"level":252},"Tokens, Modellaufrufe und Kontext werden zu Kostenvariablen",{},{"id":705,"data":706,"type":217,"tunes":708},"p-cost-1",{"text":707},"Die klassischen ML-Inferenzkosten werden oft von der Serving-Infrastruktur oder der Compute pro Vorhersage dominiert. LLM-Anwendungen können Anbieter-Token-Preise, wiederholte Agentenaufrufe, Embedding-Aufrufe, Reranking und Tool-\u002FRuntime-Overhead hinzufügen.",{},{"id":710,"data":711,"type":217,"tunes":713},"p-cost-2",{"text":712},"Kosten müssen daher einer Aufgabe oder einem Trace zugeordnet werden, nicht nur einem Endpunkt. Ein Workflow, der acht versteckte Modellaufrufe macht, kann funktional korrekt, aber operativ inakzeptabel sein.",{},{"id":715,"data":716,"type":217,"tunes":718},"p-cost-3",{"text":717},"Latenz verhält sich genauso: Modelllatenz, Retrieval, Reranking und externe Tools setzen sich zu einer End-to-End-Benutzerlatenz zusammen.",{},{"id":720,"data":721,"type":41,"tunes":723},"h-cache",{"text":722,"level":252},"Caching wird semantisch, nicht nur technisch",{},{"id":725,"data":726,"type":217,"tunes":728},"p-cache-1",{"text":727},"LLM-Systeme können Prompts, Embeddings, Retrieval-Ergebnisse oder vollständige Antworten cachen, aber der Cache-Schlüssel muss die Semantik widerspiegeln, die das Ergebnis verändern kann.",{},{"id":730,"data":731,"type":217,"tunes":733},"p-cache-2",{"text":732},"Ein Antwort-Cache, der Modellversion, Mandant, Berechtigungen oder Aktualität der Quelle ignoriert, kann eine technisch gültige, aber semantisch ungültige Antwort zurückgeben.",{},{"id":735,"data":736,"type":217,"tunes":738},"p-cache-3",{"text":737},"LLMOps behandelt Cache-Invalidierung daher als Teil der Modell-\u002FKontext-\u002FDaten-Versionierung und nicht nur als Infrastruktur-Optimierung.",{},{"id":740,"data":741,"type":41,"tunes":743},"h-safety",{"text":742,"level":252},"Sicherheit und Berechtigungen werden zu Release-Kriterien",{},{"id":745,"data":746,"type":217,"tunes":748},"p-safety-1",{"text":747},"Generative Systeme können unbegrenzten Text erzeugen und Agenten können externe Aktionen auslösen. Sicherheitstests stehen daher näher an gewöhnlicher CI\u002FCD als in vielen klassischen prädiktiven ML-Systemen.",{},{"id":750,"data":751,"type":217,"tunes":753},"p-safety-2",{"text":752},"Berechtigungsprüfungen, Prompt-Injection-Tests, Mandantenisolationstests und Genehmigungen für Nebenwirkungen sollten reproduzierbare Regressionstests sein, wo diese Risiken bestehen.",{},{"id":755,"data":756,"type":217,"tunes":758},"p-safety-3",{"text":757},"Das Modell kann eine Operation vorschlagen, aber die Laufzeit muss weiterhin die Autorisierung durchsetzen. LLMOps verantwortet den Nachweis, dass diese Kontrollen nach Änderungen an Modell, Prompt oder Tools weiterhin funktionieren.",{},{"id":760,"data":761,"type":41,"tunes":763},"h-ci",{"text":762,"level":252},"Wie CI in LLMOps aussieht",{},{"id":765,"data":766,"type":410,"tunes":798},"ci-table",{"content":767,"stretched":42,"withHeadings":13},[768,771,774,777,780,783,786,789,792,795],[769,770],"CI-Ebene","Beispielprüfungen",[772,773],"Code","Unit-Tests, Typprüfungen, Schema-Validierung",[775,776],"Prompts","Template-Rendering, erforderliche Variablen, Richtlinientext, Snapshot-Review",[778,779],"Modelle\u002FAnbieter","Kompatibilität, Ausgabeschema, Fähigkeits- und Regressionstests",[781,782],"RAG","Chunking-Fixtures, Filtertests, Recall@k, Reranker-Regression",[784,785],"Tools","Ein-\u002FAusgabeschema-Tests, Berechtigungstests, Idempotenz-Tests",[787,788],"Agenten","Trajektorien-Fixtures, Schleifenlimits, Handoff-\u002FTool-Auswahltests",[790,791],"Sicherheit","Prompt-Injection, nicht autorisierte Tools, mandantenübergreifende Negativtests",[793,794],"Verhaltensevaluierungen","Aufgabenerfolg, Korrektheit, Fundierung, Sicherheit, Domänenkriterien",[796,797],"Operativ","Latenz, Token-\u002FKostenbudgets, Timeout-\u002FFallback-Verhalten",{},{"id":800,"data":801,"type":41,"tunes":803},"h-cd",{"text":802,"level":252},"Wie CD in LLMOps aussieht",{},{"id":805,"data":806,"type":217,"tunes":808},"p-cd-1",{"text":807},"Ein Produktions-Release kann überhaupt kein neues Modellartefakt bereitstellen. Es kann einfach einen neuen Prompt, eine Retrieval-Konfiguration, ein Toolset oder eine Anbieterzuordnung ausliefern.",{},{"id":810,"data":811,"type":217,"tunes":813},"p-cd-2",{"text":812},"Das Release-Bundle sollte daher die vollständige verhaltensbestimmende Konfiguration identifizieren und nicht nur das Anwendungs-Container-Image.",{},{"id":815,"data":816,"type":217,"tunes":818},"p-cd-3",{"text":817},"Feature-Flags, gestaffelte Rollouts, Shadow-Evaluierung, Canary-Traffic und Rollback sind nützlich, weil LLM-Verhalten auf Arten regressieren kann, die statische Vertragstests nicht erkennen.",{},{"id":820,"data":821,"type":41,"tunes":823},"h-ct",{"text":822,"level":252},"Kontinuierliches Training wird optional; kontinuierliche Evaluierung wird zentral",{},{"id":825,"data":826,"type":217,"tunes":828},"p-ct-1",{"text":827},"Traditionelles MLOps betont oft kontinuierliches Training, wenn neue Daten oder Drift ein Neutraining rechtfertigen.",{},{"id":830,"data":831,"type":217,"tunes":833},"p-ct-2",{"text":832},"Viele LLM-Anwendungen trainieren niemals das Foundation-Modell. Ihr äquivalenter kontinuierlicher Kreislauf ist die kontinuierliche Evaluierung: Fehler und repräsentative Produktionsfälle sammeln, sie den Evaluierungsdatensätzen hinzufügen, Kandidatenänderungen an Prompt\u002FModell\u002FRetrieval testen und nur dann neu bereitstellen, wenn sich die Evidenz verbessert.",{},{"id":835,"data":836,"type":217,"tunes":838},"p-ct-3",{"text":837},"Fine-Tuning kann einen Trainingslebenszyklus wieder einführen, sollte aber in denselben übergeordneten Evaluierungs- und Release-Prozess eingebettet sein.",{},{"id":840,"data":841,"type":41,"tunes":843},"h-monitor",{"text":842,"level":252},"Was sollte in der Produktion überwacht werden?",{},{"id":845,"data":846,"type":410,"tunes":878},"monitor-table",{"content":847,"stretched":42,"withHeadings":13},[848,851,854,857,860,863,866,868,870,872,875],[849,850],"Signalklasse","Beispiele",[852,853],"Systemzustand","Fehler, Timeouts, Endpunktverfügbarkeit",[855,856],"Modell\u002FAnbieter","Modell-ID, Snapshot, Ratenlimits, Anbieterfehler",[858,859],"Latenz","End-to-End-, Modell-, Retrieval-, Tool- und Reranker-Spans",[861,862],"Kosten","Eingabe-\u002FAusgabe-Token, Embeddings, Tool-\u002FAPI-Ausgaben",[864,865],"Qualität","Stichprobenartiger Aufgabenerfolg, Korrektheit, Relevanz, Fundiertheit",[781,867],"Retrieval-Recall-Proxys, leeres Retrieval, veraltete Quellen, Zitatabdeckung",[787,869],"Tool-Auswahl, Wiederholungen, Schleifen, Übergaben, Genehmigungshäufigkeit",[790,871],"Verweigerte Aktionen, Prompt-Injection-Indikatoren, Mandantengrenzenverletzungen",[873,874],"Benutzerfeedback","Korrekturen, Abbruch, Eskalation, explizite Bewertungen",[876,877],"Änderungsdrift","Anbieter-\u002FModell-\u002FKonfigurationsänderungen relativ zum genehmigten Release",{},{"id":880,"data":881,"type":41,"tunes":883},"h-prod-eval",{"text":882,"level":252},"Produktions-Traces können zu Evaluierungsdaten werden",{},{"id":885,"data":886,"type":217,"tunes":888},"p-prod-1",{"text":887},"Eines der nützlichsten modernen LLMOps-Muster besteht darin, gesampelte Produktions-Traces in Evaluierungsdatensätze umzuwandeln.",{},{"id":890,"data":891,"type":217,"tunes":893},"p-prod-2",{"text":892},"MLflow unterstützt derzeit das Abrufen von Produktions-Traces und die Bewertung nicht nur von Ausgaben, sondern auch von Zwischen-Spans wie Retrieval- oder Tool-Aufruftrajektorien.",{},{"id":895,"data":896,"type":217,"tunes":898},"p-prod-3",{"text":897},"Dies schließt den Kreislauf zwischen Observability und Entwicklung: echte Fehler können zu Regressionsfällen im nächsten Release werden, anstatt in Logs zu verschwinden.",{},{"id":900,"data":901,"type":41,"tunes":903},"h-repro",{"text":902,"level":252},"Reproduzierbarkeit wird bedingt statt exakt",{},{"id":905,"data":906,"type":217,"tunes":908},"p-repro-1",{"text":907},"Klassische ML-Reproduzierbarkeit zielt oft darauf ab, ein Modell aus versioniertem Code, Daten, Umgebung und Trainingsparametern nachzubilden.",{},{"id":910,"data":911,"type":217,"tunes":913},"p-repro-2",{"text":912},"Gehostete LLM-Anwendungen können nicht immer identische Ausgaben Token für Token reproduzieren, da die Generierung probabilistisch ist und Anbieter die Infrastruktur kontrollieren können.",{},{"id":915,"data":916,"type":217,"tunes":918},"p-repro-3",{"text":917},"LLMOps zielt daher auf verhaltensbasierte Reproduzierbarkeit ab: genügend Modell-\u002FAnbieter-\u002FVersions-, Prompt-, Kontext-Eingaben, Retrieval-Zustand und Laufzeitkonfiguration aufzeichnen, um die Bedingungen zu reproduzieren und das Verhalten innerhalb erwarteter Toleranzen zu validieren.",{},{"id":920,"data":921,"type":41,"tunes":923},"h-lineage",{"text":922,"level":252},"Lineage erweitert sich von Modell-Lineage zu Anwendungs-Lineage",{},{"id":925,"data":926,"type":217,"tunes":928},"p-lineage-1",{"text":927},"Die MLOps-Richtlinien von AWS betrachten Modell-Lineage als die Historie von Code-, Daten-, Modell- und Infrastruktur-Artefakten, die für Diagnose und Reproduzierbarkeit benötigt werden.",{},{"id":930,"data":931,"type":217,"tunes":933},"p-lineage-2",{"text":932},"Für LLM-Anwendungen sollte Lineage zusätzlich Prompts, Eval-Datensätze, Retrieval-\u002FIndex-Versionen, Tool-Schemas, Agent-\u002FLaufzeitkonfiguration und Anbieter-\u002FModell-Snapshots verbinden.",{},{"id":935,"data":936,"type":217,"tunes":938},"p-lineage-3",{"text":937},"Die Ziel Frage wird: Welche exakte Anwendungskonfiguration hat diesen Trace erzeugt?",{},{"id":940,"data":941,"type":41,"tunes":943},"h-routing",{"text":942,"level":252},"Multi-Provider- und Modell-Routing erzeugen Betriebsrichtlinien",{},{"id":945,"data":946,"type":217,"tunes":948},"p-route-1",{"text":947},"Sobald eine Anwendung mehrere Anbieter oder lokale Modelle nutzen kann, wird Routing zu einer Betriebsrichtlinie statt zu einem einfachen Modell-String.",{},{"id":950,"data":951,"type":217,"tunes":953},"p-route-2",{"text":952},"Das Routing kann von Fähigkeit, Latenz, Kosten, Datenschutz, Kontextlänge, Verfügbarkeit, Tool-Unterstützung oder Lokalität abhängen. Ein Fallback kann die Verfügbarkeit aufrechterhalten, während sich die Antwortqualität oder die Annahmen zur Datenverarbeitung ändern.",{},{"id":955,"data":956,"type":217,"tunes":958},"p-route-3",{"text":957},"LLMOps sollte daher protokollieren, welche Route tatsächlich ausgewählt wurde, und Routen unabhängig voneinander bewerten, anstatt jeden kompatiblen Endpunkt als verhaltensmäßig austauschbar zu behandeln.",{},{"id":960,"data":961,"type":41,"tunes":963},"h-implementation",{"text":962,"level":252},"Belege aus der ursprünglichen Implementierung",{},{"id":965,"data":966,"type":41,"tunes":968},"h-client",{"text":967,"level":251},"Aaasaasa AI Client: Anbieter, Modell und Laufzeitumgebung sind separate operative Objekte",{},{"id":970,"data":971,"type":217,"tunes":973},"p-client-1",{"text":972},"Der Aaasaasa AI Client trennt Agent\u002FClient, Anbieter, Modell, Laufzeitort und Berechtigungen. Sein AI Hub unterstützt Ollama, LM Studio\u002FOpenAI-kompatible Endpunkte und andere Anbieterprotokolle, anstatt „das Modell“ als eine globale Einstellung zu behandeln.",{},{"id":975,"data":976,"type":217,"tunes":978},"p-client-2",{"text":977},"Die Implementierung umfasst dynamische lokale Modellerkennung, Streaming, Thinking-Ausgabe und explizite Ollama-Steuerungen zum Warmladen\u002FLaden und Entladen. Das ist ein operativer Beleg dafür, dass lokales LLM-Serving Ressourcen-Lebenszyklus-Probleme mit sich bringt, die über einen API-Modellnamen hinausgehen.",{},{"id":980,"data":981,"type":217,"tunes":983},"p-client-3",{"text":982},"Der Anbieterstatus wird über Anbieteradapter abgefragt, und Verbindungstypen unterscheiden lokale, Cloud-API-, kontogestützte, Remote-Agent- und Web-Client-Pfade. Dies sind konkrete operative Dimensionen, die eine LLM-bewusste Plattform sichtbar machen muss.",{},{"id":985,"data":986,"type":217,"tunes":988},"p-client-4",{"text":987},"Das Repository wahrt außerdem eine wichtige Grenze: Eine lokale Laufzeitumgebung ist nicht automatisch lokale Inferenz. Anbieter\u002FModell\u002FLaufzeitort sind versionierte oder konfigurierbare Belange, die Datenschutz, Latenz, Kosten und Verfügbarkeit beeinflussen.",{},{"id":990,"data":991,"type":41,"tunes":993},"h-sot",{"text":992,"level":251},"Source of Truth Research Engine: Der Zustand einer LLM-Anwendung reicht über das Modell hinaus",{},{"id":995,"data":996,"type":217,"tunes":998},"p-sot-1",{"text":997},"Die Source of Truth Research Engine kombiniert lexikalische Suche, optionale Embeddings, Quellen-Snapshots, SHA-256-Identität, Claims, Provenienz und Widerspruchsverfolgung rund um lokale modellgestützte Recherche.",{},{"id":1000,"data":1001,"type":217,"tunes":1003},"p-sot-2",{"text":1002},"Dies ist ein nützlicher LLMOps-Beleg, weil allein die Änderung des Modells das Recherchesystem nicht definiert. Retrieval, Quellenerfassung, Evidenzklassifizierung und persistente Provenienz sind unabhängige operative Artefakte.",{},{"id":1005,"data":1006,"type":217,"tunes":1008},"p-sot-3",{"text":1007},"Die Implementierung behandelt semantische Ähnlichkeit bewusst als Entdeckung und nicht als Evidenz, was zeigt, warum LLMOps-Observability Retrieval-Verhalten von der Gültigkeit von Claims unterscheiden sollte.",{},{"id":1010,"data":1011,"type":410,"tunes":1040},"impl-table",{"content":1012,"stretched":42,"withHeadings":13},[1013,1016,1019,1022,1025,1028,1031,1034,1037],[1014,1015],"Beobachtete Implementierung","LLMOps-Lektion",[1017,1018],"Mehrere Anbieterprotokolle","Anbieteridentität ist eine operative Abhängigkeit",[1020,1021],"Dynamische Modellerkennung","Verfügbare Modelle können sich unabhängig vom Anwendungscode ändern",[1023,1024],"Ollama-Steuerungen zum Laden\u002FEntladen","Lokale Modelle haben einen Speicher-\u002FRessourcen-Lebenszyklus",[1026,1027],"Adapter für Anbieterzustand\u002F-status","Modellverfügbarkeit erfordert Laufzeit-Observability",[1029,1030],"Getrennter Laufzeit- und Inferenzort","Deployment-Topologie ist nicht ein boolesches „lokal\u002FCloud“",[1032,1033],"Zentrale Berechtigungen","Modellfähigkeit und Tool-Autorität müssen getrennt bleiben",[1035,1036],"Lexikalische + semantische Retrieval-Pipeline","Retrieval-Konfiguration ist Teil des Anwendungsverhaltens",[1038,1039],"Persistenz von Quelle\u002FProvenienz","Operativer Zustand und Evidenz liegen außerhalb der Modellgewichte",{},{"id":1042,"data":1043,"type":225,"tunes":1046},"impl-boundary",{"body":1044,"title":1045,"variant":239},"Diese Projekte demonstrieren Multi-Provider-\u002FLokale-Modell-Operationen, Berechtigungstrennung, Retrieval-Infrastruktur und Evidenzpersistenz. Sie werden nicht als vollständige kommerzielle LLMOps-Plattform oder als Nachweis für Produktionsverkehr in großem Maßstab präsentiert.","Evidenzgrenze",{},{"id":1048,"data":1049,"type":41,"tunes":1051},"h-failures",{"text":1050,"level":252},"Häufige LLMOps-Fehlermodi",{},{"id":1053,"data":1054,"type":410,"tunes":1095},"failure-table",{"content":1055,"stretched":42,"withHeadings":13},[1056,1059,1062,1065,1068,1071,1074,1077,1080,1083,1086,1089,1092],[1057,1058],"Fehlermodus","Was tatsächlich schiefging",[1060,1061],"Modell-Alias wurde stillschweigend aktualisiert","Verhalten änderte sich ohne kontrollierte Freigabe",[1063,1064],"Prompt ohne Evals geändert","Verhaltensregression bestand normale Unit-Tests",[1066,1067],"RAG-Index veraltet","Das Generierungsmodell wurde für Retrieval-\u002FDatenfehler verantwortlich gemacht",[1069,1070],"Nur die endgültige Antwort wird protokolliert","Grundursache in Retrieval-\u002FTool-\u002FKontext-Trajektorie ist unsichtbar",[1072,1073],"Anbieter-Fallback erfolgt stillschweigend","Anderer Modell-\u002FDatenpfad ändert Verhalten ohne Zuordnung",[1075,1076],"Token-Kosten werden global erfasst","Teure Workflows können nicht lokalisiert werden",[1078,1079],"Judge-Modell geändert","Bewertungsergebnisse driften ohne Anwendungsänderung",[1081,1082],"Produktions-Traces werden nie zu Tests","Bekannte Fehler kehren wiederholt zurück",[1084,1085],"Lokales Modell bleibt unbegrenzt geladen","VRAM-\u002FRessourcendruck wird zu operativer Instabilität",[1087,1088],"Berechtigungen nur im Prompt kodiert","Modellverhalten wird mit Autorisierung verwechselt",[1090,1091],"Ein Eval-Score steuert alles","Verschiedene Qualitätsdimensionen werden zu einer irreführenden Zahl zusammengefasst",[1093,1094],"Modell-Registry existiert, aber Prompt-\u002FIndex-Versionen nicht","Anwendungs-Lineage bleibt unvollständig",{},{"id":1097,"data":1098,"type":41,"tunes":1100},"h-misconceptions",{"text":1099,"level":252},"Häufige Missverständnisse",{},{"id":1102,"data":1103,"type":410,"tunes":1138},"misconceptions-table",{"content":1104,"stretched":42,"withHeadings":13},[1105,1108,1111,1114,1117,1120,1123,1126,1129,1132,1135],[1106,1107],"Missverständnis","Korrektur",[1109,1110],"„LLMOps ersetzt MLOps.“","LLMOps erweitert MLOps-Prinzipien auf LLM-spezifisches Anwendungsverhalten.",[1112,1113],"„LLMOps ist Prompt Engineering.“","Prompts sind ein Artefakt unter Modellen, Anbietern, Kontext, Retrieval, Tools, Evals und Laufzeit.",[1115,1116],"„Gehostete APIs beseitigen operative Arbeit.“","Sie beseitigen einige Arbeiten zum Modell-Serving\u002FTraining, fügen aber Anbieter-Lebenszyklus-, Versions- und Abhängigkeitsmanagement hinzu.",[1118,1119],"„Wenn die API stabil ist, ist die App stabil.“","Modellverhalten und Anbieter-\u002FModell-Snapshots können sich unabhängig vom API-Schema ändern.",[1121,1122],"„RAG ist nur Datenvorverarbeitung.“","In der Produktion hat es seinen eigenen Lebenszyklus für Ingestion, Index, Retrieval und Aktualität.",[1124,1125],"„LLM-Ausgaben können nicht getestet werden.“","Sie können mit deterministischen, Referenz-, Judge- und menschlichen Kriterien bewertet werden.",[1127,1128],"„LLM-Judges sind objektive Ground Truth.“","Sie sind modellbasierte Bewerter, die ebenfalls Kalibrierung und Versionskontrolle erfordern.",[1130,1131],"„Ein lokales Modell eliminiert LLMOps.“","Lokales Serving bringt Modell-Dateien, VRAM, Laden\u002FEntladen, Laufzeit-Zustand und Upgrade-Belange mit sich.",[1133,1134],"„Observability bedeutet Token-Zahlen.“","Nützliche Observability folgt Prompts, Retrievals, Tools, Modell-Spans und Ergebnissen.",[1136,1137],"„Kontinuierliches Training ist verpflichtend.“","Viele LLM-Apps verwenden kontinuierliche Evaluierung, ohne das Foundation-Modell zu trainieren.",{},{"id":1140,"data":1141,"type":41,"tunes":1143},"h-design",{"text":1142,"level":252},"Eine praktische LLMOps-Designsequenz",{},{"id":1145,"data":1146,"type":345,"tunes":1185},"design-flow",{"steps":1147,"title":1184,"orientation":344},[1148,1151,1154,1157,1160,1163,1166,1169,1172,1175,1178,1181],{"label":1149,"description":1150},"1. Die Verhaltenseinheit definieren","Listen Sie jede Komponente auf, die die Ausgabe wesentlich verändern kann: Modell, Prompt, Retrieval, Tools, Kontext und Policy.",{"label":1152,"description":1153},"2. Anwendungs-Lineage herstellen","Versionieren Sie Code, Modell\u002FAnbieter, Prompts, Eval-Datensätze, Retrieval-Konfiguration und Tool-Verträge.",{"label":1155,"description":1156},"3. Repräsentative Eval-Datensätze erstellen","Verwenden Sie erwartete Erfolgs-\u002FFehlerfälle aus Design und Produktion.",{"label":1158,"description":1159},"4. Deterministische und verhaltensbezogene Tests trennen","Halten Sie Schema-\u002FSicherheitsassertions getrennt von der semantischen Ausgabebewertung.",{"label":1161,"description":1162},"5. End-to-End-Ausführung nachverfolgen","Instrumentieren Sie Modell-, Retrieval-, Reranking-, Tool- und Agent-\u002FRuntime-Spans.",{"label":1164,"description":1165},"6. Release-Gates definieren","Legen Sie Schwellenwerte für Qualität, Sicherheit, Latenz und Kosten fest.",{"label":1167,"description":1168},"7. Modellversionen pinnen oder explizit aufzeichnen","Behandeln Sie Modell-\u002FAnbieteränderungen als Release-Ereignisse.",{"label":1170,"description":1171},"8. Progressiv deployen","Verwenden Sie Flags, Canaries oder gestaffelte Rollouts, wo die Konsequenz es rechtfertigt.",{"label":1173,"description":1174},"9. Produktions-Traces bewerten","Messen Sie reales Aufgabenverhalten und identifizieren Sie wiederkehrende Fehler.",{"label":1176,"description":1177},"10. Fehler zurück in Eval-Datensätze einspeisen","Verwandeln Sie Vorfälle und Korrekturen in dauerhafte Regressionsabdeckung.",{"label":1179,"description":1180},"11. Anbieter- und Datenlebenszyklen überwachen","Verfolgen Sie Deprecations, Index-Aktualität, Quelländerungen und Runtime-Verfügbarkeit.",{"label":1182,"description":1183},"12. Veraltete Versionen sauber außer Betrieb nehmen","Entfernen Sie alte Prompts\u002FModelle\u002FIndizes\u002FAnmeldedaten nach Migration und Entscheidungen zur Aufbewahrung von Nachweisen.","Das vollständige verhaltenserzeugende System betreiben",{},{"id":1187,"data":1188,"type":41,"tunes":1190},"h-checklist",{"text":1189,"level":252},"LLMOps-Architektur-Checkliste",{},{"id":1192,"data":1193,"type":410,"tunes":1240},"checklist-table",{"content":1194,"stretched":42,"withHeadings":13},[1195,1198,1201,1204,1207,1210,1213,1216,1219,1222,1225,1228,1231,1234,1237],[1196,1197],"Frage","Erwarteter Nachweis",[1199,1200],"Welches Modell\u002Fwelcher Anbieter\u002Fwelche Version hat die Anfrage bedient?","Nachverfolgbare Modellidentität",[1202,1203],"Welcher Prompt\u002Fwelche Anweisungen waren aktiv?","Versionierter Anwendungscode\u002F-konfiguration",[1205,1206],"Welcher Kontext hat das Modell erreicht?","Kontext-\u002FRetrieval-Trace",[1208,1209],"Welche Korpus-\u002FIndexversion wurde verwendet?","Retrieval-Lineage",[1211,1212],"Welche Tools waren verfügbar und wurden aufgerufen?","Tool-Schema + Trajektorien-Trace",[1214,1215],"Welche Berechtigungen galten?","Runtime-Autorisierungsdatensatz",[1217,1218],"Wie wird Qualität gemessen?","Versionierter Eval-Datensatz + Scorer",[1220,1221],"Wie werden Modell-Upgrades getestet?","Verhaltensbezogene Regressionssuite",[1223,1224],"Wie wird Produktionsqualität gesampelt?","Trace-Bewertungs-\u002FFeedback-Prozess",[1226,1227],"Kann ein Fehler näherungsweise reproduziert werden?","Modell-\u002FKontext-\u002FAnbieter-\u002FAnwendungs-Lineage",[1229,1230],"Wo fallen Kosten an?","Modell-\u002FTool-\u002FRetrieval-Zuordnung pro Trace",[1232,1233],"Was löst ein Rollback aus?","Definierter Schwellenwert für Qualität\u002FSicherheit\u002FKosten\u002FVerfügbarkeit",[1235,1236],"Wie wird mit Anbieter-Deprecations umgegangen?","Migrations-\u002FFallback-Prozess",[1238,1239],"Wie werden lokale Modelle betrieben?","Health-, Ressourcen-, Lade-\u002FEntlade- und Versionskontrollen",{},{"id":1242,"data":1243,"type":41,"tunes":1245},"h-edge",{"text":1244,"level":252},"Randfälle und Einschränkungen",{},{"id":1247,"data":1248,"type":217,"tunes":1250},"p-edge-1",{"text":1249},"Eine einfache Anwendung, die ein festes gehostetes Modell ohne Retrieval oder Tools aufruft, benötigt möglicherweise nur leichtgewichtige LLMOps: versionierten Prompt-Code, Evals, Modell-Pinning, grundlegendes Tracing und Anbieterüberwachung.",{},{"id":1252,"data":1253,"type":217,"tunes":1255},"p-edge-2",{"text":1254},"Ein selbst gehostetes feinabgestimmtes Modell kann nahezu den vollständigen klassischen MLOps-Stack plus LLM-spezifische Anwendungsbewertung erfordern, wodurch die Grenze zwischen MLOps und LLMOps absichtlich unscharf wird.",{},{"id":1257,"data":1258,"type":217,"tunes":1260},"p-edge-3",{"text":1259},"Eine Agentenplattform kann minimale Modelltrainingsoperationen, aber umfangreiche Runtime-Operationen haben, weil Fehler bei Tool-Auswahl, Zustand und Orchestrierung auftreten.",{},{"id":1262,"data":1263,"type":217,"tunes":1265},"p-edge-4",{"text":1264},"Ein RAG-lastiges System kann operativ von Dokumentenaufnahme und Retrieval-Qualität statt von Modell-Serving dominiert werden.",{},{"id":1267,"data":1268,"type":217,"tunes":1270},"p-edge-5",{"text":1269},"Die Terminologie wird sich weiterentwickeln. Die dauerhafte Architekturfrage ist nicht, welches „Ops“-Label gewinnt, sondern welche Artefakte Verhalten erzeugen und daher versioniert, bewertet, beobachtet und gesteuert werden müssen.",{},{"id":1272,"data":1273,"type":41,"tunes":1275},"h-change",{"text":1274,"level":252},"Was würde diese Antwort ändern?",{},{"id":1277,"data":1278,"type":217,"tunes":1280},"p-change-1",{"text":1279},"Wenn Foundation-Model-Anbieter perfekt stabiles Modellverhalten und langfristige Versionsunterstützung standardisieren, könnte Anbieter-\u002FSnapshot-Management operativ weniger bedeutsam werden.",{},{"id":1282,"data":1283,"type":217,"tunes":1285},"p-change-2",{"text":1284},"Wenn Anwendungen zunehmend Fine-Tuning oder Training übernehmen, rücken klassische MLOps-Anliegen wieder stärker in den Mittelpunkt.",{},{"id":1287,"data":1288,"type":217,"tunes":1290},"p-change-3",{"text":1289},"Das operative Prinzip bliebe: Jede Komponente, die das Produktionsverhalten wesentlich verändern kann, gehört in Lineage, Testing, Observability und Change Control.",{},{"id":1292,"data":1293,"type":41,"tunes":1295},"h-related",{"text":1294,"level":252},"Verwandtes kanonisches Wissen",{},{"id":1297,"data":1298,"type":217,"tunes":1300},"p-related-1",{"text":1299},"LLMOps steht unter AI Governance und Enterprise AI Architecture: Governance definiert, welche Änderungen Nachweise und Genehmigung erfordern, während LLMOps die operative Maschinerie bereitstellt, um diese Änderungen zu versionieren, zu bewerten, bereitzustellen und zu beobachten.",{},{"id":1302,"data":1303,"type":217,"tunes":1305},"p-related-2",{"text":1304},"Context Engineering und RAG sind operative Subdomänen innerhalb vieler LLM-Anwendungen, weil Kontext und Retrieval das Verhalten unabhängig vom Modell ändern können.",{},{"id":1307,"data":1308,"type":217,"tunes":1310},"p-related-3",{"text":1309},"Agentic AI erweitert LLMOps weiter in Trajektorien-, Berechtigungs- und Tool-Runtime-Operationen.",{},{"id":1312,"data":1313,"type":602,"tunes":1318},"ref-memory",{"url":1314,"title":1315,"excerpt":1316,"ctaLabel":1317},"https:\u002F\u002Fstajic.de\u002Fde\u002Fblog\u002Fai-agent-memory-is-not-rag-how-to-separate-memory-retrieval-state-and-context","KI-Agenten-Gedächtnis ist nicht RAG: Wie man Gedächtnis, Retrieval, Zustand und Kontext trennt","Die operative Zuverlässigkeit verbessert sich, wenn Gedächtnis, Retrieval, Anwendungszustand und Modellkontext getrennte Lebenszyklusobjekte bleiben.","Architekturartikel lesen",{},{"id":1320,"data":1321,"type":602,"tunes":1326},"ref-avb",{"url":1322,"title":1323,"excerpt":1324,"ctaLabel":1325},"https:\u002F\u002Fstajic.de\u002Fde\u002Fblog\u002Fthe-answer-validity-boundary-the-missing-layer-between-relevance-and-reliable-ai-answers","Die Antwortgültigkeitsgrenze: Die fehlende Schicht zwischen Relevanz und zuverlässigen KI-Antworten","LLMOps-Evaluierung sollte die Version, den Umfang und die Evidenzbedingungen bewahren, unter denen eine Antwort weiterhin gestützt wird.","Antwortgültigkeitsgrenze lesen",{},{"id":1328,"data":1329,"type":41,"tunes":1331},"h-faq",{"text":1330,"level":252},"Häufig gestellte Fragen",{},{"id":1333,"data":1334,"type":1333,"tunes":1373},"faq",{"items":1335,"title":1372},[1336,1340,1344,1348,1352,1356,1360,1364,1368],{"id":1337,"answer":1338,"question":1339},"faq1","MLOps betreibt Machine-Learning-Systeme über Daten, Training, Deployment und Monitoring. LLMOps erweitert diese Praktiken auf LLM-Anwendungen, bei denen Prompts, Kontext, Retrieval, Anbieter, Tools und Evaluierungen das Verhalten ebenfalls wesentlich beeinflussen.","Was ist der Unterschied zwischen MLOps und LLMOps?",{"id":1341,"answer":1342,"question":1343},"faq2","Nein. LLMOps nutzt MLOps-Disziplinen wie CI\u002FCD, Lineage, Evaluierung, Deployment und Monitoring wieder und fügt LLM-spezifische operative Belange hinzu.","Ersetzt LLMOps MLOps?",{"id":1345,"answer":1346,"question":1347},"faq3","Nicht unbedingt. Viele verwenden externe Foundation-Modelle und setzen stattdessen auf kontinuierliche Evaluierung von Prompts, Modellen, Retrieval und Anwendungsverhalten. Feinabgestimmte oder selbst trainierte Systeme können dennoch Trainingspipelines erfordern.","Benötigen LLM-Anwendungen kontinuierliches Training?",{"id":1349,"answer":1350,"question":1351},"faq4","Generative Ausgaben sind offen und das Modellverhalten kann sich über Prompts, Snapshots und Kontext hinweg ändern. Evals liefern wiederholbare Belege dafür, dass ein Release weiterhin definierte Qualitäts- und Sicherheitskriterien erfüllt.","Warum sind Evals in LLMOps so wichtig?",{"id":1353,"answer":1354,"question":1355},"faq5","Mindestens: Anwendungscode, Modell\u002FAnbieter\u002FVersion, Prompts, Eval-Datensätze\u002FScorer, Retrieval-Konfiguration\u002FIndizes, Tool-Schemas, Kontextregeln und relevante Sicherheits-\u002FBerechtigungskonfiguration.","Was sollte in LLMOps versioniert werden?",{"id":1357,"answer":1358,"question":1359},"faq6","Nein. Derselbe Prompt kann sich mit einem anderen Modell, Retrieval-Set, einer anderen Kontextreihenfolge, Tool-Oberfläche oder einem anderen Anbieter anders verhalten.","Reicht Prompt-Versionierung aus?",{"id":1361,"answer":1362,"question":1363},"faq7","GenAIOps ist ein weiterer Branchenbegriff für den Betrieb generativer KI-Anwendungen. Einige Anbieter verwenden ihn synonym oder als breiteres Label als LLMOps.","Was ist GenAIOps?",{"id":1365,"answer":1366,"question":1367},"faq8","Überwachen Sie End-to-End-Traces einschließlich Modellaufrufen, Prompts\u002FKontext, Retrieval, Tools, Latenz, Token\u002FKosten, Qualitätsstichproben, Sicherheit und endgültigen Aufgabenergebnissen.","Wie überwacht man eine LLM-Anwendung?",{"id":1369,"answer":1370,"question":1371},"faq9","Ja. Lokale Modelle bringen eigene operative Belange mit sich, wie Modelldateien, Hardware\u002FVRAM, Laden\u002FEntladen, Laufzeitintegrität, Quantisierung und Upgrade-Management.","Können lokale LLMs LLMOps-Praktiken nutzen?","MLOps vs. LLMOps FAQ",{},{"id":1375,"data":1376,"type":41,"tunes":1378},"h-glossary",{"text":1377,"level":252},"Glossar",{},{"id":1380,"data":1381,"type":1380,"tunes":1426},"glossary",{"title":1382,"entries":1383},"Wichtige MLOps- und LLMOps-Begriffe",[1384,1386,1388,1392,1395,1399,1403,1407,1411,1414,1418,1422],{"term":414,"anchor":413,"definition":1385},"Engineering-Praktiken zum Erstellen, Bereitstellen, Überwachen und Warten von Machine-Learning-Systemen und deren Daten-\u002FModelllebenszyklus.",{"term":417,"anchor":416,"definition":1387},"Operative Praktiken für Produktionsanwendungen, deren Verhalten wesentlich von großen Sprachmodellen und umgebenden Prompts, Kontext, Retrieval, Tools und Laufzeit abhängt.",{"term":1389,"anchor":1390,"definition":1391},"GenAIOps","genaiops","Operative Disziplin für generative KI-Anwendungen; wird oft als breiteres oder alternatives Label für LLMOps verwendet.",{"term":399,"anchor":1393,"definition":1394},"continuous-training","Automatisiertes oder wiederholtes Neutraining und Serving von ML-Modellen, wenn sich Daten oder Implementierungen ändern.",{"term":1396,"anchor":1397,"definition":1398},"Kontinuierliche Evaluierung","continuous-evaluation","Wiederholte Evaluierung des Verhaltens von Kandidaten- und Produktions-KI anhand versionierter Datensätze und Kriterien.",{"term":1400,"anchor":1401,"definition":1402},"Modell-Snapshot","model-snapshot","Eine konkrete Version eines gehosteten oder paketierten Modells, deren Verhalten getestet und referenziert werden kann.",{"term":1404,"anchor":1405,"definition":1406},"Anwendungs-Lineage","application-lineage","Nachvollziehbare Beziehung zwischen Code, Modell\u002FAnbieter, Prompts, Daten\u002FRetrieval, Tools, Laufzeit und Release-Konfiguration.",{"term":1408,"anchor":1409,"definition":1410},"Trace","trace","Strukturierte Aufzeichnung einer Anwendungsausführung mit Spans wie Modellaufrufen, Retrievals und Tool-Operationen.",{"term":468,"anchor":1412,"definition":1413},"eval-dataset","Versionierter Satz repräsentativer Eingaben, Erwartungen und optional Traces\u002FAusgaben zur Verhaltensmessung.",{"term":1415,"anchor":1416,"definition":1417},"LLM-Richter","llm-judge","Ein Sprachmodell, das als Evaluator für qualitative oder semantische Kriterien verwendet wird; es ist selbst eine versionierte Evaluierungsabhängigkeit.",{"term":1419,"anchor":1420,"definition":1421},"Verhaltensregression","behavioral-regression","Eine Verschlechterung der Anwendungsausgabe oder -trajektorie, obwohl Schnittstellen und Code weiterhin erfolgreich ausgeführt werden.",{"term":1423,"anchor":1424,"definition":1425},"Anbieter-Routing","provider-routing","Richtlinie zur Auswahl unter verfügbaren Modellanbietern\u002FEndpunkten nach Fähigkeit, Kosten, Latenz, Datenschutz oder Verfügbarkeit.",{},{"id":1428,"data":1429,"type":41,"tunes":1431},"h-conclusion",{"text":1430,"level":252},"Fazit",{},{"id":1433,"data":1434,"type":217,"tunes":1436},"p-conclusion-1",{"text":1435},"MLOps und LLMOps teilen dasselbe Engineering-Ziel: KI-Systeme ausreichend reproduzierbar, testbar und beobachtbar zu machen, um sie zuverlässig in der Produktion zu betreiben.",{},{"id":1438,"data":1439,"type":217,"tunes":1441},"p-conclusion-2",{"text":1440},"Der Unterschied liegt in der Form des Systems. Klassisches MLOps konzentriert sich oft auf Training und Serving von Modellartefakten; LLMOps muss einen Verhaltens-Stack betreiben, in dem Modell-Snapshots, Prompts, Kontext, Retrieval, Tools, Berechtigungen und Anbieter sich unabhängig ändern können.",{},{"id":1443,"data":1444,"type":217,"tunes":1446},"p-conclusion-3",{"text":1445},"Die kürzeste nützliche Regel lautet: Versionieren, evaluieren und beobachten Sie alles, was das Verhalten der LLM-Anwendung wesentlich ändern kann – nicht nur das Modell.",{},{"id":1448,"data":1449,"type":41,"tunes":1451},"h-sources",{"text":1450,"level":252},"Primärquellen und aktuelle Dokumentation",{},{"id":1453,"data":1454,"type":217,"tunes":1456},"p-sources-note",{"text":1455},"Die folgenden Quellen untermauern die MLOps-Basis und die aktuellen operativen Muster für LLM- und Agenten-Anwendungen. Projektabschnitte sind originale Implementierungsbelege und bewusst enger gefasst als Aussagen über eine vollständige LLMOps-Plattform.",{},{"id":1458,"data":1459,"type":1465,"tunes":1466},"src-google-mlops",{"link":1460,"meta":1461},"https:\u002F\u002Fdocs.cloud.google.com\u002Farchitecture\u002Fmlops-continuous-delivery-and-automation-pipelines-in-machine-learning",{"image":1462,"title":1463,"description":1464},{"url":380},"Google Cloud — MLOps: Continuous delivery and automation pipelines","Referenzarchitektur, die CI, CD, kontinuierliches Training, Modellregistry, Metadaten, Serving und Monitoring für ML-Systeme beschreibt.","linkTool",{},{"id":1468,"data":1469,"type":1465,"tunes":1475},"src-aws-lineage",{"link":1470,"meta":1471},"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops02-bp04.html",{"image":1472,"title":1473,"description":1474},{"url":380},"AWS Machine Learning Lens — Model lineage","Aktuelle Anleitung zur Nachverfolgung von Code, Daten, Modellen, Umgebungen und Infrastruktur über ML-Releases hinweg.",{},{"id":1477,"data":1478,"type":1465,"tunes":1484},"src-aws-monitor",{"link":1479,"meta":1480},"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops06-bp02.html",{"image":1481,"title":1482,"description":1483},{"url":380},"AWS Machine Learning Lens — Model observability and tracking","Aktuelle Anleitung für Produktionsmodell-Monitoring, Drift, Endpoint-Integrität und Lineage.",{},{"id":1486,"data":1487,"type":1465,"tunes":1493},"src-azure-llmops",{"link":1488,"meta":1489},"https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fmachine-learning\u002Fprompt-flow\u002Fhow-to-end-to-end-llmops-with-prompt-flow",{"image":1490,"title":1491,"description":1492},{"url":380},"Microsoft Azure — GenAIOps \u002F LLMOps lifecycle","Offizielle Anleitung, die GenAIOps, manchmal LLMOps genannt, über Initialisierung, Experimentierung, Evaluierung\u002FVerfeinerung und Deployment beschreibt.",{},{"id":1495,"data":1496,"type":1465,"tunes":1502},"src-mlflow-genai",{"link":1497,"meta":1498},"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002F",{"image":1499,"title":1500,"description":1501},{"url":380},"MLflow — Agents and LLM applications","Aktuelle GenAI-Betriebsdokumentation zu Tracing, Evaluierung, Prompts und Produktionsbeobachtbarkeit für LLM-Anwendungen und Agenten.",{},{"id":1504,"data":1505,"type":1465,"tunes":1511},"src-mlflow-traces",{"link":1506,"meta":1507},"https:\u002F\u002Fwww.mlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Feval-monitor\u002Frunning-evaluation\u002Ftraces\u002F",{"image":1508,"title":1509,"description":1510},{"url":380},"MLflow — Evaluating production traces","Aktuelle Anleitung zur Evaluierung vollständiger LLM-\u002FAgenten-Traces, einschließlich Retrieval- und Tool-Aufruf-Trajektorien.",{},{"id":1513,"data":1514,"type":1465,"tunes":1520},"src-mlflow-prompt-eval",{"link":1515,"meta":1516},"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Fprompt-registry\u002Fevaluate-prompts\u002F",{"image":1517,"title":1518,"description":1519},{"url":380},"MLflow — Bewertung von Prompts","Aktueller Workflow zur Bewertung von Prompts\u002FModellen unter Verwendung versionierter Prompts, Datensätze, Scorer und Traces.",{},{"id":1522,"data":1523,"type":1465,"tunes":1529},"src-openai-api",{"link":1524,"meta":1525},"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Freference\u002Foverview",{"image":1526,"title":1527,"description":1528},{"url":380},"OpenAI API — Versionierung und Modell-Snapshots","Aktuelle API-Empfehlung, gepinnte Modellversionen und Evals zu verwenden, da sich das Prompting-Verhalten zwischen Snapshots ändern kann.",{},{"id":1531,"data":1532,"type":1465,"tunes":1538},"src-openai-prompting",{"link":1533,"meta":1534},"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fguides\u002Fprompting",{"image":1535,"title":1536,"description":1537},{"url":380},"OpenAI — Prompting","Aktuelle Empfehlung, Produktions-Prompts wie Anwendungscode zu behandeln, sie über Quellcodeverwaltung zu versionieren und Änderungen mit Tests und Bewertungsprüfungen abzudecken.",{},{"id":1540,"data":1541,"type":1465,"tunes":1547},"src-openai-deprecations",{"link":1542,"meta":1543},"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fdeprecations",{"image":1544,"title":1545,"description":1546},{"url":380},"OpenAI — Deprecations","Aktuelle Anbieter-Lebenszyklus-Nachweise, die die Stilllegung von Modellen und Plattformoberflächen als betriebliche Abhängigkeit zeigen.",{},{"id":1549,"data":1550,"type":1465,"tunes":1556},"src-openai-promptfoo",{"link":1551,"meta":1552},"https:\u002F\u002Fdevelopers.openai.com\u002Fcookbook\u002Fexamples\u002Fevaluation\u002Fmoving-from-openai-evals-to-promptfoo",{"image":1553,"title":1554,"description":1555},{"url":380},"OpenAI — Evaluierungsworkflows zu Promptfoo verlagern","Aktuelle Migrationsempfehlung aus 2026, die veranschaulicht, warum Bewertungsressourcen portabel bleiben sollten, wenn sich die Anbieter-Tooling ändert.",{},"2.31","MLOps betreibt Systeme für maschinelles Lernen; LLMOps erweitert diese Praktiken auf Prompts, Kontext, Retrieval, Anbieter, Tools, Evaluierungen und Laufzeitverhalten rund um große Sprachmodelle.","\u002Fuploads\u002F2026\u002F10\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm-1791487319869-2v7hxo.webp","mlops-vs-llmops-what-changes-when-the-model-is-an-llm-1791487319869-2v7hxo","PUBLISHED","2026-10-08T15:20:00.000Z","2026-10-08T19:20:01.249Z","2026-10-08T19:31:17.955Z",{"en":1566,"de":1567,"sr":1568,"es":1569,"fr":1570,"it":1571,"ru":1572,"zh":1573},"\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fde\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fsr\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fes\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Ffr\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fit\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fru\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fzh\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm",[1575,1579,1583,1587],{"id":1576,"name":1577,"slug":1578},88,"Versionierung (Prompts, Modelle)","versioning",{"id":1580,"name":1581,"slug":1582},91,"Monitoring (Qualität, Drift)","monitoring",{"id":1584,"name":1585,"slug":1586},89,"Evaluation-Harness","evaluation-harness",{"id":1588,"name":1589,"slug":1590},58,"Evaluation & Qualitäts-Gates","evaluation",{"id":1592,"login":1593,"email":1594,"displayName":1595},"20","rooth8233","aleksandar@stajic.de","Aleksandar Stajić",[1597,2279],{"lang":7,"title":207,"content":209,"contentJson":1598,"excerpt":1558},{"time":211,"blocks":1599,"version":1557},[1600,1603,1606,1609,1612,1615,1618,1621,1624,1627,1630,1633,1636,1639,1642,1645,1648,1651,1654,1666,1669,1672,1675,1678,1681,1704,1707,1710,1713,1716,1719,1738,1741,1744,1747,1750,1753,1756,1759,1762,1765,1768,1771,1774,1777,1780,1783,1786,1789,1792,1795,1798,1801,1804,1807,1810,1813,1816,1819,1822,1825,1828,1831,1834,1837,1840,1843,1846,1849,1852,1855,1858,1861,1864,1867,1870,1873,1876,1879,1882,1885,1888,1891,1894,1908,1911,1914,1917,1920,1923,1926,1929,1932,1935,1950,1953,1956,1959,1962,1965,1968,1971,1974,1977,1980,1983,1986,1989,1992,1995,1998,2001,2004,2007,2010,2013,2016,2019,2022,2025,2028,2041,2044,2047,2064,2067,2082,2085,2101,2104,2123,2126,2129,2132,2135,2138,2141,2144,2147,2150,2153,2156,2159,2162,2165,2168,2171,2174,2187,2190,2206,2209,2212,2215,2218,2221,2224,2229,2234,2239,2244,2249,2254,2259,2264,2269,2274],{"id":214,"data":1601,"type":217,"tunes":1602},{"text":216},{},{"id":220,"data":1604,"type":225,"tunes":1605},{"body":222,"title":223,"variant":224},{},{"id":228,"data":1607,"type":225,"tunes":1608},{"body":230,"title":231,"variant":232},{},{"id":235,"data":1610,"type":225,"tunes":1611},{"body":237,"title":238,"variant":239},{},{"id":242,"data":1613,"type":225,"tunes":1614},{"body":244,"title":245,"variant":239},{},{"id":248,"data":1616,"type":253,"tunes":1617},{"title":250,"maxLevel":251,"minLevel":252},{},{"id":256,"data":1619,"type":41,"tunes":1620},{"text":258,"level":252},{},{"id":261,"data":1622,"type":217,"tunes":1623},{"text":263},{},{"id":266,"data":1625,"type":217,"tunes":1626},{"text":268},{},{"id":271,"data":1628,"type":217,"tunes":1629},{"text":273},{},{"id":276,"data":1631,"type":41,"tunes":1632},{"text":278,"level":252},{},{"id":281,"data":1634,"type":217,"tunes":1635},{"text":283},{},{"id":286,"data":1637,"type":217,"tunes":1638},{"text":288},{},{"id":291,"data":1640,"type":217,"tunes":1641},{"text":293},{},{"id":296,"data":1643,"type":41,"tunes":1644},{"text":298,"level":252},{},{"id":301,"data":1646,"type":217,"tunes":1647},{"text":303},{},{"id":306,"data":1649,"type":217,"tunes":1650},{"text":308},{},{"id":311,"data":1652,"type":217,"tunes":1653},{"text":313},{},{"id":316,"data":1655,"type":345,"tunes":1665},{"steps":1656,"title":343,"orientation":344},[1657,1658,1659,1660,1661,1662,1663,1664],{"label":320,"description":321},{"label":323,"description":324},{"label":326,"description":327},{"label":329,"description":330},{"label":332,"description":333},{"label":335,"description":336},{"label":338,"description":339},{"label":341,"description":342},{},{"id":348,"data":1667,"type":41,"tunes":1668},{"text":350,"level":252},{},{"id":353,"data":1670,"type":217,"tunes":1671},{"text":355},{},{"id":358,"data":1673,"type":217,"tunes":1674},{"text":360},{},{"id":363,"data":1676,"type":217,"tunes":1677},{"text":365},{},{"id":368,"data":1679,"type":41,"tunes":1680},{"text":370,"level":252},{},{"id":373,"data":1682,"type":418,"tunes":1703},{"rows":1683,"title":409,"layout":410,"columns":1700},[1684,1686,1688,1690,1692,1694,1696,1698],{"id":377,"label":378,"values":1685},[380,380],{"id":382,"label":383,"values":1687},[380,380],{"id":386,"label":387,"values":1689},[380,380],{"id":390,"label":391,"values":1691},[380,380],{"id":394,"label":395,"values":1693},[380,380],{"id":398,"label":399,"values":1695},[380,380],{"id":402,"label":403,"values":1697},[380,380],{"id":406,"label":407,"values":1699},[380,380],[1701,1702],{"id":413,"label":414},{"id":416,"label":417},{},{"id":421,"data":1705,"type":41,"tunes":1706},{"text":423,"level":252},{},{"id":426,"data":1708,"type":217,"tunes":1709},{"text":428},{},{"id":431,"data":1711,"type":217,"tunes":1712},{"text":433},{},{"id":436,"data":1714,"type":217,"tunes":1715},{"text":438},{},{"id":441,"data":1717,"type":41,"tunes":1718},{"text":443,"level":252},{},{"id":446,"data":1720,"type":410,"tunes":1737},{"content":1721,"stretched":42,"withHeadings":13},[1722,1723,1724,1725,1726,1727,1728,1729,1730,1731,1732,1733,1734,1735,1736],[450,451],[453,454],[456,457],[459,460],[462,463],[465,466],[468,469],[471,472],[474,475],[477,478],[480,481],[483,484],[486,487],[489,490],[492,493],{},{"id":496,"data":1739,"type":41,"tunes":1740},{"text":498,"level":252},{},{"id":501,"data":1742,"type":217,"tunes":1743},{"text":503},{},{"id":506,"data":1745,"type":217,"tunes":1746},{"text":508},{},{"id":511,"data":1748,"type":217,"tunes":1749},{"text":513},{},{"id":516,"data":1751,"type":41,"tunes":1752},{"text":518,"level":252},{},{"id":521,"data":1754,"type":217,"tunes":1755},{"text":523},{},{"id":526,"data":1757,"type":217,"tunes":1758},{"text":528},{},{"id":531,"data":1760,"type":217,"tunes":1761},{"text":533},{},{"id":536,"data":1763,"type":41,"tunes":1764},{"text":538,"level":252},{},{"id":541,"data":1766,"type":217,"tunes":1767},{"text":543},{},{"id":546,"data":1769,"type":217,"tunes":1770},{"text":548},{},{"id":551,"data":1772,"type":217,"tunes":1773},{"text":553},{},{"id":556,"data":1775,"type":41,"tunes":1776},{"text":558,"level":252},{},{"id":561,"data":1778,"type":217,"tunes":1779},{"text":563},{},{"id":566,"data":1781,"type":217,"tunes":1782},{"text":568},{},{"id":571,"data":1784,"type":217,"tunes":1785},{"text":573},{},{"id":576,"data":1787,"type":41,"tunes":1788},{"text":578,"level":252},{},{"id":581,"data":1790,"type":217,"tunes":1791},{"text":583},{},{"id":586,"data":1793,"type":217,"tunes":1794},{"text":588},{},{"id":591,"data":1796,"type":217,"tunes":1797},{"text":593},{},{"id":596,"data":1799,"type":602,"tunes":1800},{"url":598,"title":599,"excerpt":600,"ctaLabel":601},{},{"id":605,"data":1802,"type":41,"tunes":1803},{"text":607,"level":252},{},{"id":610,"data":1805,"type":217,"tunes":1806},{"text":612},{},{"id":615,"data":1808,"type":217,"tunes":1809},{"text":617},{},{"id":620,"data":1811,"type":217,"tunes":1812},{"text":622},{},{"id":625,"data":1814,"type":225,"tunes":1815},{"body":627,"title":628,"variant":629},{},{"id":632,"data":1817,"type":41,"tunes":1818},{"text":634,"level":252},{},{"id":637,"data":1820,"type":217,"tunes":1821},{"text":639},{},{"id":642,"data":1823,"type":217,"tunes":1824},{"text":644},{},{"id":647,"data":1826,"type":217,"tunes":1827},{"text":649},{},{"id":652,"data":1829,"type":41,"tunes":1830},{"text":654,"level":252},{},{"id":657,"data":1832,"type":217,"tunes":1833},{"text":659},{},{"id":662,"data":1835,"type":217,"tunes":1836},{"text":664},{},{"id":667,"data":1838,"type":217,"tunes":1839},{"text":669},{},{"id":672,"data":1841,"type":41,"tunes":1842},{"text":674,"level":252},{},{"id":677,"data":1844,"type":217,"tunes":1845},{"text":679},{},{"id":682,"data":1847,"type":217,"tunes":1848},{"text":684},{},{"id":687,"data":1850,"type":217,"tunes":1851},{"text":689},{},{"id":692,"data":1853,"type":602,"tunes":1854},{"url":694,"title":695,"excerpt":696,"ctaLabel":697},{},{"id":700,"data":1856,"type":41,"tunes":1857},{"text":702,"level":252},{},{"id":705,"data":1859,"type":217,"tunes":1860},{"text":707},{},{"id":710,"data":1862,"type":217,"tunes":1863},{"text":712},{},{"id":715,"data":1865,"type":217,"tunes":1866},{"text":717},{},{"id":720,"data":1868,"type":41,"tunes":1869},{"text":722,"level":252},{},{"id":725,"data":1871,"type":217,"tunes":1872},{"text":727},{},{"id":730,"data":1874,"type":217,"tunes":1875},{"text":732},{},{"id":735,"data":1877,"type":217,"tunes":1878},{"text":737},{},{"id":740,"data":1880,"type":41,"tunes":1881},{"text":742,"level":252},{},{"id":745,"data":1883,"type":217,"tunes":1884},{"text":747},{},{"id":750,"data":1886,"type":217,"tunes":1887},{"text":752},{},{"id":755,"data":1889,"type":217,"tunes":1890},{"text":757},{},{"id":760,"data":1892,"type":41,"tunes":1893},{"text":762,"level":252},{},{"id":765,"data":1895,"type":410,"tunes":1907},{"content":1896,"stretched":42,"withHeadings":13},[1897,1898,1899,1900,1901,1902,1903,1904,1905,1906],[769,770],[772,773],[775,776],[778,779],[781,782],[784,785],[787,788],[790,791],[793,794],[796,797],{},{"id":800,"data":1909,"type":41,"tunes":1910},{"text":802,"level":252},{},{"id":805,"data":1912,"type":217,"tunes":1913},{"text":807},{},{"id":810,"data":1915,"type":217,"tunes":1916},{"text":812},{},{"id":815,"data":1918,"type":217,"tunes":1919},{"text":817},{},{"id":820,"data":1921,"type":41,"tunes":1922},{"text":822,"level":252},{},{"id":825,"data":1924,"type":217,"tunes":1925},{"text":827},{},{"id":830,"data":1927,"type":217,"tunes":1928},{"text":832},{},{"id":835,"data":1930,"type":217,"tunes":1931},{"text":837},{},{"id":840,"data":1933,"type":41,"tunes":1934},{"text":842,"level":252},{},{"id":845,"data":1936,"type":410,"tunes":1949},{"content":1937,"stretched":42,"withHeadings":13},[1938,1939,1940,1941,1942,1943,1944,1945,1946,1947,1948],[849,850],[852,853],[855,856],[858,859],[861,862],[864,865],[781,867],[787,869],[790,871],[873,874],[876,877],{},{"id":880,"data":1951,"type":41,"tunes":1952},{"text":882,"level":252},{},{"id":885,"data":1954,"type":217,"tunes":1955},{"text":887},{},{"id":890,"data":1957,"type":217,"tunes":1958},{"text":892},{},{"id":895,"data":1960,"type":217,"tunes":1961},{"text":897},{},{"id":900,"data":1963,"type":41,"tunes":1964},{"text":902,"level":252},{},{"id":905,"data":1966,"type":217,"tunes":1967},{"text":907},{},{"id":910,"data":1969,"type":217,"tunes":1970},{"text":912},{},{"id":915,"data":1972,"type":217,"tunes":1973},{"text":917},{},{"id":920,"data":1975,"type":41,"tunes":1976},{"text":922,"level":252},{},{"id":925,"data":1978,"type":217,"tunes":1979},{"text":927},{},{"id":930,"data":1981,"type":217,"tunes":1982},{"text":932},{},{"id":935,"data":1984,"type":217,"tunes":1985},{"text":937},{},{"id":940,"data":1987,"type":41,"tunes":1988},{"text":942,"level":252},{},{"id":945,"data":1990,"type":217,"tunes":1991},{"text":947},{},{"id":950,"data":1993,"type":217,"tunes":1994},{"text":952},{},{"id":955,"data":1996,"type":217,"tunes":1997},{"text":957},{},{"id":960,"data":1999,"type":41,"tunes":2000},{"text":962,"level":252},{},{"id":965,"data":2002,"type":41,"tunes":2003},{"text":967,"level":251},{},{"id":970,"data":2005,"type":217,"tunes":2006},{"text":972},{},{"id":975,"data":2008,"type":217,"tunes":2009},{"text":977},{},{"id":980,"data":2011,"type":217,"tunes":2012},{"text":982},{},{"id":985,"data":2014,"type":217,"tunes":2015},{"text":987},{},{"id":990,"data":2017,"type":41,"tunes":2018},{"text":992,"level":251},{},{"id":995,"data":2020,"type":217,"tunes":2021},{"text":997},{},{"id":1000,"data":2023,"type":217,"tunes":2024},{"text":1002},{},{"id":1005,"data":2026,"type":217,"tunes":2027},{"text":1007},{},{"id":1010,"data":2029,"type":410,"tunes":2040},{"content":2030,"stretched":42,"withHeadings":13},[2031,2032,2033,2034,2035,2036,2037,2038,2039],[1014,1015],[1017,1018],[1020,1021],[1023,1024],[1026,1027],[1029,1030],[1032,1033],[1035,1036],[1038,1039],{},{"id":1042,"data":2042,"type":225,"tunes":2043},{"body":1044,"title":1045,"variant":239},{},{"id":1048,"data":2045,"type":41,"tunes":2046},{"text":1050,"level":252},{},{"id":1053,"data":2048,"type":410,"tunes":2063},{"content":2049,"stretched":42,"withHeadings":13},[2050,2051,2052,2053,2054,2055,2056,2057,2058,2059,2060,2061,2062],[1057,1058],[1060,1061],[1063,1064],[1066,1067],[1069,1070],[1072,1073],[1075,1076],[1078,1079],[1081,1082],[1084,1085],[1087,1088],[1090,1091],[1093,1094],{},{"id":1097,"data":2065,"type":41,"tunes":2066},{"text":1099,"level":252},{},{"id":1102,"data":2068,"type":410,"tunes":2081},{"content":2069,"stretched":42,"withHeadings":13},[2070,2071,2072,2073,2074,2075,2076,2077,2078,2079,2080],[1106,1107],[1109,1110],[1112,1113],[1115,1116],[1118,1119],[1121,1122],[1124,1125],[1127,1128],[1130,1131],[1133,1134],[1136,1137],{},{"id":1140,"data":2083,"type":41,"tunes":2084},{"text":1142,"level":252},{},{"id":1145,"data":2086,"type":345,"tunes":2100},{"steps":2087,"title":1184,"orientation":344},[2088,2089,2090,2091,2092,2093,2094,2095,2096,2097,2098,2099],{"label":1149,"description":1150},{"label":1152,"description":1153},{"label":1155,"description":1156},{"label":1158,"description":1159},{"label":1161,"description":1162},{"label":1164,"description":1165},{"label":1167,"description":1168},{"label":1170,"description":1171},{"label":1173,"description":1174},{"label":1176,"description":1177},{"label":1179,"description":1180},{"label":1182,"description":1183},{},{"id":1187,"data":2102,"type":41,"tunes":2103},{"text":1189,"level":252},{},{"id":1192,"data":2105,"type":410,"tunes":2122},{"content":2106,"stretched":42,"withHeadings":13},[2107,2108,2109,2110,2111,2112,2113,2114,2115,2116,2117,2118,2119,2120,2121],[1196,1197],[1199,1200],[1202,1203],[1205,1206],[1208,1209],[1211,1212],[1214,1215],[1217,1218],[1220,1221],[1223,1224],[1226,1227],[1229,1230],[1232,1233],[1235,1236],[1238,1239],{},{"id":1242,"data":2124,"type":41,"tunes":2125},{"text":1244,"level":252},{},{"id":1247,"data":2127,"type":217,"tunes":2128},{"text":1249},{},{"id":1252,"data":2130,"type":217,"tunes":2131},{"text":1254},{},{"id":1257,"data":2133,"type":217,"tunes":2134},{"text":1259},{},{"id":1262,"data":2136,"type":217,"tunes":2137},{"text":1264},{},{"id":1267,"data":2139,"type":217,"tunes":2140},{"text":1269},{},{"id":1272,"data":2142,"type":41,"tunes":2143},{"text":1274,"level":252},{},{"id":1277,"data":2145,"type":217,"tunes":2146},{"text":1279},{},{"id":1282,"data":2148,"type":217,"tunes":2149},{"text":1284},{},{"id":1287,"data":2151,"type":217,"tunes":2152},{"text":1289},{},{"id":1292,"data":2154,"type":41,"tunes":2155},{"text":1294,"level":252},{},{"id":1297,"data":2157,"type":217,"tunes":2158},{"text":1299},{},{"id":1302,"data":2160,"type":217,"tunes":2161},{"text":1304},{},{"id":1307,"data":2163,"type":217,"tunes":2164},{"text":1309},{},{"id":1312,"data":2166,"type":602,"tunes":2167},{"url":1314,"title":1315,"excerpt":1316,"ctaLabel":1317},{},{"id":1320,"data":2169,"type":602,"tunes":2170},{"url":1322,"title":1323,"excerpt":1324,"ctaLabel":1325},{},{"id":1328,"data":2172,"type":41,"tunes":2173},{"text":1330,"level":252},{},{"id":1333,"data":2175,"type":1333,"tunes":2186},{"items":2176,"title":1372},[2177,2178,2179,2180,2181,2182,2183,2184,2185],{"id":1337,"answer":1338,"question":1339},{"id":1341,"answer":1342,"question":1343},{"id":1345,"answer":1346,"question":1347},{"id":1349,"answer":1350,"question":1351},{"id":1353,"answer":1354,"question":1355},{"id":1357,"answer":1358,"question":1359},{"id":1361,"answer":1362,"question":1363},{"id":1365,"answer":1366,"question":1367},{"id":1369,"answer":1370,"question":1371},{},{"id":1375,"data":2188,"type":41,"tunes":2189},{"text":1377,"level":252},{},{"id":1380,"data":2191,"type":1380,"tunes":2205},{"title":1382,"entries":2192},[2193,2194,2195,2196,2197,2198,2199,2200,2201,2202,2203,2204],{"term":414,"anchor":413,"definition":1385},{"term":417,"anchor":416,"definition":1387},{"term":1389,"anchor":1390,"definition":1391},{"term":399,"anchor":1393,"definition":1394},{"term":1396,"anchor":1397,"definition":1398},{"term":1400,"anchor":1401,"definition":1402},{"term":1404,"anchor":1405,"definition":1406},{"term":1408,"anchor":1409,"definition":1410},{"term":468,"anchor":1412,"definition":1413},{"term":1415,"anchor":1416,"definition":1417},{"term":1419,"anchor":1420,"definition":1421},{"term":1423,"anchor":1424,"definition":1425},{},{"id":1428,"data":2207,"type":41,"tunes":2208},{"text":1430,"level":252},{},{"id":1433,"data":2210,"type":217,"tunes":2211},{"text":1435},{},{"id":1438,"data":2213,"type":217,"tunes":2214},{"text":1440},{},{"id":1443,"data":2216,"type":217,"tunes":2217},{"text":1445},{},{"id":1448,"data":2219,"type":41,"tunes":2220},{"text":1450,"level":252},{},{"id":1453,"data":2222,"type":217,"tunes":2223},{"text":1455},{},{"id":1458,"data":2225,"type":1465,"tunes":2228},{"link":1460,"meta":2226},{"image":2227,"title":1463,"description":1464},{"url":380},{},{"id":1468,"data":2230,"type":1465,"tunes":2233},{"link":1470,"meta":2231},{"image":2232,"title":1473,"description":1474},{"url":380},{},{"id":1477,"data":2235,"type":1465,"tunes":2238},{"link":1479,"meta":2236},{"image":2237,"title":1482,"description":1483},{"url":380},{},{"id":1486,"data":2240,"type":1465,"tunes":2243},{"link":1488,"meta":2241},{"image":2242,"title":1491,"description":1492},{"url":380},{},{"id":1495,"data":2245,"type":1465,"tunes":2248},{"link":1497,"meta":2246},{"image":2247,"title":1500,"description":1501},{"url":380},{},{"id":1504,"data":2250,"type":1465,"tunes":2253},{"link":1506,"meta":2251},{"image":2252,"title":1509,"description":1510},{"url":380},{},{"id":1513,"data":2255,"type":1465,"tunes":2258},{"link":1515,"meta":2256},{"image":2257,"title":1518,"description":1519},{"url":380},{},{"id":1522,"data":2260,"type":1465,"tunes":2263},{"link":1524,"meta":2261},{"image":2262,"title":1527,"description":1528},{"url":380},{},{"id":1531,"data":2265,"type":1465,"tunes":2268},{"link":1533,"meta":2266},{"image":2267,"title":1536,"description":1537},{"url":380},{},{"id":1540,"data":2270,"type":1465,"tunes":2273},{"link":1542,"meta":2271},{"image":2272,"title":1545,"description":1546},{"url":380},{},{"id":1549,"data":2275,"type":1465,"tunes":2278},{"link":1551,"meta":2276},{"image":2277,"title":1554,"description":1555},{"url":380},{},{"lang":2280,"title":2281,"content":2282,"contentJson":2283,"excerpt":3392},"en","MLOps vs LLMOps: What Changes When the Model Is an LLM","{\"time\":1791487321430,\"blocks\":[{\"id\":\"intro\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLOps is the engineering discipline for reliably developing, deploying, versioning and operating machine-learning systems; LLMOps extends that discipline to applications built around large language models, where production behavior depends not only on a model artifact but also on prompts, context, retrieval, provider\u002Fmodel versions, tool calls, safety controls and evaluation pipelines. LLMOps does not replace MLOps. It changes the operational unit from “a model plus serving pipeline” toward “an evolving LLM application whose behavior emerges from several independently changing components.”\"},\"tunes\":{}},{\"id\":\"direct\",\"type\":\"callout\",\"data\":{\"variant\":\"info\",\"title\":\"Direct answer\",\"body\":\"\u003Cstrong>MLOps operates ML systems. LLMOps operates LLM applications.\u003C\u002Fstrong>\u003Cbr>\u003Cbr>Classical MLOps commonly centers on data pipelines, training, validation, model registry, deployment, drift and retraining. LLMOps keeps those disciplines where relevant, but often adds prompt\u002Fcontext versioning, model\u002Fprovider abstraction, RAG indexes, agent\u002Ftool traces, semantic evaluations, safety tests, token\u002Fcost monitoring and regression testing across rapidly changing model snapshots.\"},\"tunes\":{}},{\"id\":\"boundary\",\"type\":\"callout\",\"data\":{\"variant\":\"warning\",\"title\":\"LLMOps is not just prompt management\",\"body\":\"A production LLM application can fail even when the prompt is unchanged: the provider can change a model snapshot, a RAG corpus can become stale, a reranker can regress, tool permissions can change, context assembly can drop evidence, or an agent can take a wrong trajectory. LLMOps therefore has to observe and version the system around the model, not only prompt text.\"},\"tunes\":{}},{\"id\":\"term-note\",\"type\":\"callout\",\"data\":{\"variant\":\"note\",\"title\":\"Terminology boundary\",\"body\":\"\u003Cstrong>LLMOps\u003C\u002Fstrong>, \u003Cstrong>GenAIOps\u003C\u002Fstrong> and related terms are widely used engineering labels, but they are not one universal formal standard with a single canonical lifecycle. Microsoft currently describes GenAIOps as “sometimes called LLMOps,” while MLflow groups operational tooling around agents and LLM applications. This article uses LLMOps as a practical architecture term for operating production systems whose behavior materially depends on LLMs.\"},\"tunes\":{}},{\"id\":\"current\",\"type\":\"callout\",\"data\":{\"variant\":\"note\",\"title\":\"Current-source note — 8 October 2026\",\"body\":\"The operational surface is changing quickly. OpenAI currently recommends pinning model snapshots and running evals because prompting behavior can change between snapshots, and several older platform-specific prompt\u002Feval surfaces are being retired in 2026. The stable architectural lesson is to keep prompts, tests and evals portable and versioned with the application rather than depend on one provider's dashboard object model.\"},\"tunes\":{}},{\"id\":\"toc\",\"type\":\"tableOfContents\",\"data\":{\"title\":\"Contents\",\"minLevel\":2,\"maxLevel\":3},\"tunes\":{}},{\"id\":\"h-meaning\",\"type\":\"header\",\"data\":{\"text\":\"What MLOps really means\",\"level\":2},\"tunes\":{}},{\"id\":\"p-mlops-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLOps applies software-engineering and operational discipline to machine-learning systems. The production challenge is broader than training a model: data collection, data validation, experimentation, reproducibility, model evaluation, deployment, infrastructure and monitoring all have to work together.\"},\"tunes\":{}},{\"id\":\"p-mlops-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Google's MLOps architecture guidance frames the discipline around continuous integration, continuous delivery and continuous training. CI validates not only code but also data, schemas and models; CD deploys ML pipelines and prediction services; CT can retrain and redeploy models as data or implementations change.\"},\"tunes\":{}},{\"id\":\"p-mlops-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"AWS guidance adds the same operational concerns from another angle: model lineage, model\u002Fversion traceability, drift monitoring and production-quality monitoring are core parts of keeping ML systems reliable after deployment.\"},\"tunes\":{}},{\"id\":\"h-llmops\",\"type\":\"header\",\"data\":{\"text\":\"What changes when the model is an LLM\",\"level\":2},\"tunes\":{}},{\"id\":\"p-llmops-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Large language models change the production problem because the application often does not own the complete model-training lifecycle. A team may call a hosted model API, run an open model locally, switch between providers or use several models for different tasks.\"},\"tunes\":{}},{\"id\":\"p-llmops-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The model is therefore only one versioned dependency inside a larger behavioral system. Prompts, retrieval results, context order, tools, model snapshot, temperature\u002Freasoning settings, safety filters and runtime orchestration can all change the output.\"},\"tunes\":{}},{\"id\":\"p-llmops-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"This creates a broader operational question: which combination of model, context, data, prompt, tools and runtime produced this behavior? LLMOps exists to make that question answerable and the answer reproducible enough for engineering work.\"},\"tunes\":{}},{\"id\":\"h-simple\",\"type\":\"header\",\"data\":{\"text\":\"The simplest example\",\"level\":2},\"tunes\":{}},{\"id\":\"p-simple-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Suppose an application answers internal policy questions.\"},\"tunes\":{}},{\"id\":\"p-simple-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"In a classical ML framing, you might version a trained classifier, deploy it and monitor prediction quality. In an LLM application, the answer might depend on a hosted model snapshot, a system prompt, an embedding model, a vector index, retrieval filters, a reranker and the final selected context.\"},\"tunes\":{}},{\"id\":\"p-simple-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Changing any one of those components can change the final answer even though the application endpoint and user question stay identical.\"},\"tunes\":{}},{\"id\":\"simple-flow\",\"type\":\"processFlow\",\"data\":{\"title\":\"A typical LLMOps release path\",\"orientation\":\"auto\",\"steps\":[{\"label\":\"1. Change one component\",\"description\":\"Prompt, model, provider, retrieval setting, tool schema or application code changes.\"},{\"label\":\"2. Run deterministic tests\",\"description\":\"Validate schemas, permissions, tool contracts, retrieval filters and application behavior.\"},{\"label\":\"3. Run behavioral evals\",\"description\":\"Compare representative outputs, retrieval quality and agent\u002Ftool trajectories against acceptance criteria.\"},{\"label\":\"4. Compare cost and latency\",\"description\":\"Measure token use, model calls, retrieval\u002Ftool overhead and response latency.\"},{\"label\":\"5. Deploy controlled version\",\"description\":\"Ship the concrete application configuration with model\u002Fprovider versions recorded.\"},{\"label\":\"6. Trace production behavior\",\"description\":\"Capture relevant model, retrieval, tool and runtime spans.\"},{\"label\":\"7. Evaluate production traces\",\"description\":\"Sample real executions for quality, grounding, safety and task success.\"},{\"label\":\"8. Roll back or iterate\",\"description\":\"Use regression evidence and operational signals to decide the next release.\"}]},\"tunes\":{}},{\"id\":\"h-stops\",\"type\":\"header\",\"data\":{\"text\":\"Where the simple example stops\",\"level\":2},\"tunes\":{}},{\"id\":\"p-stops-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Some LLM systems still train or fine-tune their own models, so traditional MLOps practices such as training pipelines, model registry and data lineage remain directly relevant.\"},\"tunes\":{}},{\"id\":\"p-stops-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Other systems use only external foundation-model APIs and never run continuous training. Their main operational workload is application evaluation, model\u002Fprovider change management, prompt\u002Fcontext versioning, retrieval quality and observability.\"},\"tunes\":{}},{\"id\":\"p-stops-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"There is therefore no single universal “LLMOps pipeline.” The exact lifecycle depends on whether you train, fine-tune, self-host, retrieve external knowledge, run agents or depend on managed model APIs.\"},\"tunes\":{}},{\"id\":\"h-compare\",\"type\":\"header\",\"data\":{\"text\":\"MLOps vs LLMOps\",\"level\":2},\"tunes\":{}},{\"id\":\"main-comparison\",\"type\":\"comparison\",\"data\":{\"title\":\"What stays the same and what expands\",\"layout\":\"table\",\"columns\":[{\"id\":\"mlops\",\"label\":\"MLOps\"},{\"id\":\"llmops\",\"label\":\"LLMOps\"}],\"rows\":[{\"id\":\"unit\",\"label\":\"Primary operational unit\",\"values\":[\"\",\"\"]},{\"id\":\"model\",\"label\":\"Model ownership\",\"values\":[\"\",\"\"]},{\"id\":\"change\",\"label\":\"Typical change\",\"values\":[\"\",\"\"]},{\"id\":\"eval\",\"label\":\"Evaluation\",\"values\":[\"\",\"\"]},{\"id\":\"monitor\",\"label\":\"Production monitoring\",\"values\":[\"\",\"\"]},{\"id\":\"training\",\"label\":\"Continuous training\",\"values\":[\"\",\"\"]},{\"id\":\"registry\",\"label\":\"Versioned artifacts\",\"values\":[\"\",\"\"]},{\"id\":\"rollback\",\"label\":\"Rollback target\",\"values\":[\"\",\"\"]}]},\"tunes\":{}},{\"id\":\"h-extension\",\"type\":\"header\",\"data\":{\"text\":\"LLMOps extends MLOps rather than replacing it\",\"level\":2},\"tunes\":{}},{\"id\":\"p-extension-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"The core operational principles do not disappear: source control, CI\u002FCD, reproducibility, lineage, deployment controls, monitoring, rollback and measurable acceptance criteria remain essential.\"},\"tunes\":{}},{\"id\":\"p-extension-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The extension is that more behavior-defining artifacts now sit outside the model weights. A managed foundation model can change behavior through snapshot upgrades, while application output can change through prompt or retrieval changes without any model retraining.\"},\"tunes\":{}},{\"id\":\"p-extension-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"This is why the useful hierarchy is usually DevOps → MLOps → LLMOps\u002FGenAIOps as increasingly specialized operational concerns, not three mutually exclusive practices.\"},\"tunes\":{}},{\"id\":\"h-artifacts\",\"type\":\"header\",\"data\":{\"text\":\"What has to be versioned in LLMOps?\",\"level\":2},\"tunes\":{}},{\"id\":\"artifact-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Artifact\",\"Why it matters\"],[\"Application code\",\"Defines orchestration, validation, retries and business behavior\"],[\"Model family + snapshot\u002Fversion\",\"Different snapshots can produce different behavior\"],[\"Provider \u002F endpoint\",\"Changes data flow, latency, limits, pricing and availability\"],[\"Prompt\u002Finstruction code\",\"Changes model behavior even with same model\"],[\"Generation\u002Freasoning parameters\",\"Can alter determinism, latency, depth and cost\"],[\"Eval dataset\",\"Defines what “good enough” is tested against\"],[\"Scorers \u002F graders\",\"Define how quality is measured\"],[\"Embedding model\",\"Changes vector representation and retrieval behavior\"],[\"Chunking\u002Findex configuration\",\"Changes what can be retrieved\"],[\"Reranker \u002F retrieval fusion\",\"Changes result ordering\"],[\"Tool schemas\",\"Change what the model can request and how\"],[\"Permission profile\",\"Changes what tool actions may actually execute\"],[\"Context assembly rules\",\"Change what evidence and state reach the model\"],[\"Safety\u002Fguardrail configuration\",\"Changes allowed or blocked behavior\"]]},\"tunes\":{}},{\"id\":\"h-model-version\",\"type\":\"header\",\"data\":{\"text\":\"Model snapshots become release dependencies\",\"level\":2},\"tunes\":{}},{\"id\":\"p-model-version-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"With hosted LLMs, the team may not control model training, but it still controls which model or snapshot the application calls.\"},\"tunes\":{}},{\"id\":\"p-model-version-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"OpenAI's current API guidance explicitly warns that prompting behavior can change between model snapshots and recommends pinning production applications to specific snapshots where consistency matters, then running evals when upgrading.\"},\"tunes\":{}},{\"id\":\"p-model-version-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The operational consequence is straightforward: model upgrades should be treated as application releases, not invisible infrastructure maintenance.\"},\"tunes\":{}},{\"id\":\"h-provider\",\"type\":\"header\",\"data\":{\"text\":\"Provider lifecycle becomes part of operations\",\"level\":2},\"tunes\":{}},{\"id\":\"p-provider-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLM applications often depend on provider rate limits, deprecation schedules, API semantics, context limits, data-handling rules and pricing.\"},\"tunes\":{}},{\"id\":\"p-provider-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"A provider can deprecate a model while your application code remains unchanged. OpenAI's current deprecation schedule, for example, includes 2026 retirement dates for older model snapshots and platform surfaces.\"},\"tunes\":{}},{\"id\":\"p-provider-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps therefore needs provider lifecycle tracking, migration testing and fallback decisions in addition to model-quality monitoring.\"},\"tunes\":{}},{\"id\":\"h-prompt\",\"type\":\"header\",\"data\":{\"text\":\"Prompts behave like production code\",\"level\":2},\"tunes\":{}},{\"id\":\"p-prompt-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Prompts are executable behavioral configuration. Small changes can alter output quality, tool selection and policy interpretation.\"},\"tunes\":{}},{\"id\":\"p-prompt-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"OpenAI's current guidance recommends storing production prompts in application code, reviewing prompt changes through pull requests, using typed inputs and covering changes with tests and evaluation checks.\"},\"tunes\":{}},{\"id\":\"p-prompt-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"That makes prompt versioning less like editing marketing copy and more like changing a function whose output is probabilistic and model-dependent.\"},\"tunes\":{}},{\"id\":\"h-context\",\"type\":\"header\",\"data\":{\"text\":\"Context engineering becomes an operational concern\",\"level\":2},\"tunes\":{}},{\"id\":\"p-context-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"The production model rarely receives only a static prompt. It may receive conversation history, retrieved documents, tool outputs, memory, current application state and policy instructions.\"},\"tunes\":{}},{\"id\":\"p-context-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps must therefore observe context assembly: which evidence was selected, which state version was current, whether truncation occurred and whether important instructions survived compaction.\"},\"tunes\":{}},{\"id\":\"p-context-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"A model regression and a context regression can look identical at the final answer. Tracing the actual context path is what lets the team separate them.\"},\"tunes\":{}},{\"id\":\"h-rag\",\"type\":\"header\",\"data\":{\"text\":\"RAG creates its own operational lifecycle\",\"level\":2},\"tunes\":{}},{\"id\":\"p-rag-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"A RAG system introduces a second production pipeline beside model inference: ingestion, extraction, chunking, metadata, embeddings, indexes, retrieval, reranking and context selection.\"},\"tunes\":{}},{\"id\":\"p-rag-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The knowledge corpus can change every day even when the model and prompt do not. A stale index or broken metadata filter can therefore degrade answer quality without any model drift.\"},\"tunes\":{}},{\"id\":\"p-rag-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps for RAG should track corpus\u002Findex version, embedding model, chunking policy, retrieval configuration, source freshness and retrieval metrics separately from generation quality.\"},\"tunes\":{}},{\"id\":\"ref-rag-diagnostic\",\"type\":\"referralArticle\",\"data\":{\"url\":\"https:\u002F\u002Fstajic.de\u002Fblog\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method\",\"title\":\"RAG Failed — But Which Layer Actually Failed? A Diagnostic Method\",\"excerpt\":\"A production LLM pipeline needs separate observability for source coverage, retrieval, ranking, context assembly and generation.\",\"ctaLabel\":\"Read the RAG diagnostic method\"},\"tunes\":{}},{\"id\":\"h-evals\",\"type\":\"header\",\"data\":{\"text\":\"Evals replace “looks good to me” with release evidence\",\"level\":2},\"tunes\":{}},{\"id\":\"p-eval-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Generative outputs are often open-ended, so exact-match tests are insufficient for many tasks. LLMOps adds evaluation datasets and scorers that can measure task success, correctness, safety, groundedness, style or domain-specific acceptance criteria.\"},\"tunes\":{}},{\"id\":\"p-eval-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLflow's current GenAI evaluation stack supports versioned evaluation datasets, prompt\u002Fmodel comparisons, custom scorers and evaluation over complete traces.\"},\"tunes\":{}},{\"id\":\"p-eval-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The strongest practice is evaluation-driven development: define representative cases and acceptance criteria before or alongside changes, then compare releases against the same evidence.\"},\"tunes\":{}},{\"id\":\"eval-rule\",\"type\":\"callout\",\"data\":{\"variant\":\"success\",\"title\":\"Behavioral changes need behavioral tests\",\"body\":\"A deployment should not be considered equivalent merely because the API contract still works. If the prompt, model, retrieval or tools changed, the behavioral regression suite should run again.\"},\"tunes\":{}},{\"id\":\"h-judges\",\"type\":\"header\",\"data\":{\"text\":\"LLM-as-a-judge is useful but not ground truth\",\"level\":2},\"tunes\":{}},{\"id\":\"p-judge-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLM judges can scale evaluation for qualities that are expensive to encode as deterministic assertions, such as relevance, tone or groundedness.\"},\"tunes\":{}},{\"id\":\"p-judge-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"However, the judge is another model with its own bias, version and prompt. Judge configuration should therefore be versioned and calibrated against human or deterministic reference cases where consequence matters.\"},\"tunes\":{}},{\"id\":\"p-judge-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"A production eval can mix deterministic checks, reference-based metrics, model judges and human review rather than asking one metric to represent every quality dimension.\"},\"tunes\":{}},{\"id\":\"h-tracing\",\"type\":\"header\",\"data\":{\"text\":\"Tracing becomes more important than endpoint logs\",\"level\":2},\"tunes\":{}},{\"id\":\"p-trace-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Traditional API logs can tell you that a request took two seconds and returned HTTP 200. They cannot tell you which retrieved chunks were selected, which tool the agent called or which model span consumed most tokens.\"},\"tunes\":{}},{\"id\":\"p-trace-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLflow's current GenAI tracing captures prompts, retrievals, tool calls and application spans, and its production evaluation flow can score intermediate trajectory information rather than only final text.\"},\"tunes\":{}},{\"id\":\"p-trace-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"This is a major LLMOps shift: observability follows the behavioral graph of the application, not only the serving endpoint.\"},\"tunes\":{}},{\"id\":\"h-agent\",\"type\":\"header\",\"data\":{\"text\":\"Agents expand LLMOps into runtime operations\",\"level\":2},\"tunes\":{}},{\"id\":\"p-agent-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"An agentic application can perform several model calls, tool invocations and state transitions before producing a result.\"},\"tunes\":{}},{\"id\":\"p-agent-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Operating agents therefore requires step counts, tool-call traces, permission denials, retries, loop detection, human approvals and verified final state in addition to ordinary model latency and token metrics.\"},\"tunes\":{}},{\"id\":\"p-agent-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"A correct final answer can hide a bad trajectory, so agent evaluation must inspect the path as well as the result.\"},\"tunes\":{}},{\"id\":\"ref-agent-reliability\",\"type\":\"referralArticle\",\"data\":{\"url\":\"https:\u002F\u002Fstajic.de\u002Fblog\u002Fai-agent-reliability-why-the-final-answer-is-not-enough\",\"title\":\"AI Agent Reliability: Why the Final Answer Is Not Enough\",\"excerpt\":\"Why agent production evaluation must include tool calls, state transitions, approvals and recoverability.\",\"ctaLabel\":\"Read the agent reliability article\"},\"tunes\":{}},{\"id\":\"h-cost\",\"type\":\"header\",\"data\":{\"text\":\"Tokens, model calls and context become cost variables\",\"level\":2},\"tunes\":{}},{\"id\":\"p-cost-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Classical ML inference cost is often dominated by serving infrastructure or per-prediction compute. LLM applications can add provider token pricing, repeated agent calls, embedding calls, reranking and tool\u002Fruntime overhead.\"},\"tunes\":{}},{\"id\":\"p-cost-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Cost therefore has to be attributed to task or trace, not only to one endpoint. A workflow that makes eight hidden model calls can be functionally correct but operationally unacceptable.\"},\"tunes\":{}},{\"id\":\"p-cost-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Latency behaves the same way: model latency, retrieval, reranking and external tools compose into end-to-end user latency.\"},\"tunes\":{}},{\"id\":\"h-cache\",\"type\":\"header\",\"data\":{\"text\":\"Caching becomes semantic, not only technical\",\"level\":2},\"tunes\":{}},{\"id\":\"p-cache-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLM systems can cache prompts, embeddings, retrieval results or full responses, but the cache key must reflect the semantics that can change the result.\"},\"tunes\":{}},{\"id\":\"p-cache-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"A response cache that ignores model version, tenant, permissions or source freshness can return a technically valid but semantically invalid answer.\"},\"tunes\":{}},{\"id\":\"p-cache-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps therefore treats cache invalidation as part of model\u002Fcontext\u002Fdata versioning rather than only infrastructure optimization.\"},\"tunes\":{}},{\"id\":\"h-safety\",\"type\":\"header\",\"data\":{\"text\":\"Safety and permissions become release criteria\",\"level\":2},\"tunes\":{}},{\"id\":\"p-safety-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Generative systems can produce unbounded text and agents can trigger external actions. Safety testing therefore sits closer to ordinary CI\u002FCD than in many classical predictive ML systems.\"},\"tunes\":{}},{\"id\":\"p-safety-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Permission checks, prompt-injection tests, tenant-isolation tests and side-effect approvals should be reproducible regression tests where those risks exist.\"},\"tunes\":{}},{\"id\":\"p-safety-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The model may suggest an operation, but the runtime still has to enforce authorization. LLMOps owns the evidence that those controls continue to work after model, prompt or tool changes.\"},\"tunes\":{}},{\"id\":\"h-ci\",\"type\":\"header\",\"data\":{\"text\":\"What CI looks like in LLMOps\",\"level\":2},\"tunes\":{}},{\"id\":\"ci-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"CI layer\",\"Example checks\"],[\"Code\",\"Unit tests, type checks, schema validation\"],[\"Prompts\",\"Template rendering, required variables, policy text, snapshot review\"],[\"Models\u002Fproviders\",\"Compatibility, output schema, capability and regression tests\"],[\"RAG\",\"Chunking fixtures, filter tests, Recall@k, reranker regression\"],[\"Tools\",\"Input\u002Foutput schema tests, permission tests, idempotency tests\"],[\"Agents\",\"Trajectory fixtures, loop limits, handoff\u002Ftool-selection tests\"],[\"Security\",\"Prompt injection, unauthorized tools, cross-tenant negative tests\"],[\"Behavioral evals\",\"Task success, correctness, grounding, safety, domain criteria\"],[\"Operational\",\"Latency, token\u002Fcost budgets, timeout\u002Ffallback behavior\"]]},\"tunes\":{}},{\"id\":\"h-cd\",\"type\":\"header\",\"data\":{\"text\":\"What CD looks like in LLMOps\",\"level\":2},\"tunes\":{}},{\"id\":\"p-cd-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"A production release may deploy no new model artifact at all. It may simply ship a new prompt, retrieval configuration, tool set or provider mapping.\"},\"tunes\":{}},{\"id\":\"p-cd-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The release bundle should therefore identify the complete behavior-defining configuration rather than only the application container image.\"},\"tunes\":{}},{\"id\":\"p-cd-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Feature flags, staged rollout, shadow evaluation, canary traffic and rollback are useful because LLM behavior can regress in ways that static contract tests do not detect.\"},\"tunes\":{}},{\"id\":\"h-ct\",\"type\":\"header\",\"data\":{\"text\":\"Continuous training becomes optional; continuous evaluation becomes central\",\"level\":2},\"tunes\":{}},{\"id\":\"p-ct-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Traditional MLOps often emphasizes continuous training when new data or drift justifies retraining.\"},\"tunes\":{}},{\"id\":\"p-ct-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Many LLM applications never train the foundation model. Their equivalent continuous loop is continuous evaluation: collect failures and representative production cases, add them to evaluation datasets, test candidate prompt\u002Fmodel\u002Fretrieval changes and redeploy only when evidence improves.\"},\"tunes\":{}},{\"id\":\"p-ct-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Fine-tuning can reintroduce a training lifecycle, but it should sit inside the same broader evaluation and release process.\"},\"tunes\":{}},{\"id\":\"h-monitor\",\"type\":\"header\",\"data\":{\"text\":\"What should be monitored in production?\",\"level\":2},\"tunes\":{}},{\"id\":\"monitor-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Signal class\",\"Examples\"],[\"System health\",\"Errors, timeouts, endpoint availability\"],[\"Model\u002Fprovider\",\"Model ID, snapshot, rate limits, provider errors\"],[\"Latency\",\"End-to-end, model, retrieval, tool and reranker spans\"],[\"Cost\",\"Input\u002Foutput tokens, embeddings, tool\u002FAPI spend\"],[\"Quality\",\"Sampled task success, correctness, relevance, groundedness\"],[\"RAG\",\"Retrieval recall proxies, empty retrieval, stale sources, citation coverage\"],[\"Agents\",\"Tool selection, retries, loops, handoffs, approval frequency\"],[\"Security\",\"Denied actions, prompt-injection indicators, tenant-boundary failures\"],[\"User feedback\",\"Corrections, abandonment, escalation, explicit ratings\"],[\"Change drift\",\"Provider\u002Fmodel\u002Fconfig changes relative to approved release\"]]},\"tunes\":{}},{\"id\":\"h-prod-eval\",\"type\":\"header\",\"data\":{\"text\":\"Production traces can become evaluation data\",\"level\":2},\"tunes\":{}},{\"id\":\"p-prod-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"One of the most useful modern LLMOps patterns is to turn sampled production traces into evaluation records.\"},\"tunes\":{}},{\"id\":\"p-prod-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLflow currently supports retrieving production traces and scoring not only outputs but intermediate spans such as retrieval or tool-call trajectories.\"},\"tunes\":{}},{\"id\":\"p-prod-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"This closes the loop between observability and development: real failures can become regression cases in the next release rather than disappear inside logs.\"},\"tunes\":{}},{\"id\":\"h-repro\",\"type\":\"header\",\"data\":{\"text\":\"Reproducibility becomes conditional rather than exact\",\"level\":2},\"tunes\":{}},{\"id\":\"p-repro-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Classical ML reproducibility often aims to recreate a model from versioned code, data, environment and training parameters.\"},\"tunes\":{}},{\"id\":\"p-repro-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Hosted LLM applications cannot always reproduce identical output token-for-token because generation is probabilistic and providers may control infrastructure.\"},\"tunes\":{}},{\"id\":\"p-repro-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps therefore aims for behavioral reproducibility: record enough model\u002Fprovider\u002Fversion, prompt, context inputs, retrieval state and runtime configuration to reproduce the conditions and validate behavior within expected tolerances.\"},\"tunes\":{}},{\"id\":\"h-lineage\",\"type\":\"header\",\"data\":{\"text\":\"Lineage expands from model lineage to application lineage\",\"level\":2},\"tunes\":{}},{\"id\":\"p-lineage-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"AWS's MLOps guidance treats model lineage as the history of code, data, model and infrastructure artifacts needed for diagnosis and reproducibility.\"},\"tunes\":{}},{\"id\":\"p-lineage-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"For LLM applications, lineage should additionally connect prompts, eval datasets, retrieval\u002Findex versions, tool schemas, agent\u002Fruntime configuration and provider\u002Fmodel snapshots.\"},\"tunes\":{}},{\"id\":\"p-lineage-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The target question becomes: Which exact application configuration produced this trace?\"},\"tunes\":{}},{\"id\":\"h-routing\",\"type\":\"header\",\"data\":{\"text\":\"Multi-provider and model routing create operational policy\",\"level\":2},\"tunes\":{}},{\"id\":\"p-route-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Once an application can use several providers or local models, routing becomes an operational policy rather than a simple model string.\"},\"tunes\":{}},{\"id\":\"p-route-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Routing may depend on capability, latency, cost, privacy, context length, availability, tool support or locality. A fallback can preserve uptime while changing answer quality or data-processing assumptions.\"},\"tunes\":{}},{\"id\":\"p-route-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps should therefore log which route was actually selected and evaluate routes independently rather than treat every compatible endpoint as behaviorally interchangeable.\"},\"tunes\":{}},{\"id\":\"h-implementation\",\"type\":\"header\",\"data\":{\"text\":\"Original implementation evidence\",\"level\":2},\"tunes\":{}},{\"id\":\"h-client\",\"type\":\"header\",\"data\":{\"text\":\"Aaasaasa AI Client: provider, model and runtime are separate operational objects\",\"level\":3},\"tunes\":{}},{\"id\":\"p-client-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Aaasaasa AI Client separates agent\u002Fclient, provider, model, runtime location and permissions. Its AI Hub supports Ollama, LM Studio\u002FOpenAI-compatible endpoints and other provider protocols rather than treating “the model” as one global setting.\"},\"tunes\":{}},{\"id\":\"p-client-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The implementation includes dynamic local model discovery, streaming, thinking output and explicit Ollama warm\u002Fload and unload controls. That is operational evidence that local LLM serving introduces resource lifecycle concerns beyond an API model name.\"},\"tunes\":{}},{\"id\":\"p-client-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Provider status is queried through provider adapters, and connection types distinguish local, cloud API, account-backed, remote-agent and web-client paths. These are concrete operational dimensions an LLM-aware platform has to surface.\"},\"tunes\":{}},{\"id\":\"p-client-4\",\"type\":\"paragraph\",\"data\":{\"text\":\"The repository also preserves an important boundary: a local runtime is not automatically local inference. Provider\u002Fmodel\u002Fruntime location are versioned or configurable concerns that affect privacy, latency, cost and availability.\"},\"tunes\":{}},{\"id\":\"h-sot\",\"type\":\"header\",\"data\":{\"text\":\"Source of Truth Research Engine: LLM application state extends beyond the model\",\"level\":3},\"tunes\":{}},{\"id\":\"p-sot-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"The Source of Truth Research Engine combines lexical search, optional embeddings, source snapshots, SHA-256 identity, claims, provenance and contradiction tracking around local model-assisted research.\"},\"tunes\":{}},{\"id\":\"p-sot-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"This is useful LLMOps evidence because changing the model alone does not define the research system. Retrieval, source acquisition, evidence classification and persistent provenance are independent operational artifacts.\"},\"tunes\":{}},{\"id\":\"p-sot-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The implementation deliberately treats semantic similarity as discovery rather than evidence, showing why LLMOps observability should distinguish retrieval behavior from claim validity.\"},\"tunes\":{}},{\"id\":\"impl-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Observed implementation\",\"LLMOps lesson\"],[\"Multiple provider protocols\",\"Provider identity is an operational dependency\"],[\"Dynamic model discovery\",\"Available models can change independently of application code\"],[\"Ollama load\u002Funload controls\",\"Local models have memory\u002Fresource lifecycle\"],[\"Provider health\u002Fstatus adapters\",\"Model availability needs runtime observability\"],[\"Separate runtime and inference location\",\"Deployment topology is not one boolean “local\u002Fcloud”\"],[\"Central permissions\",\"Model capability and tool authority must remain separate\"],[\"Lexical + semantic retrieval pipeline\",\"Retrieval configuration is part of application behavior\"],[\"Source\u002Fprovenance persistence\",\"Operational state and evidence live outside model weights\"]]},\"tunes\":{}},{\"id\":\"impl-boundary\",\"type\":\"callout\",\"data\":{\"variant\":\"note\",\"title\":\"Evidence boundary\",\"body\":\"These projects demonstrate multi-provider\u002Flocal-model operations, permission separation, retrieval infrastructure and evidence persistence. They are not presented as a complete commercial LLMOps platform or proof of large-scale production traffic.\"},\"tunes\":{}},{\"id\":\"h-failures\",\"type\":\"header\",\"data\":{\"text\":\"Common LLMOps failure modes\",\"level\":2},\"tunes\":{}},{\"id\":\"failure-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Failure mode\",\"What actually went wrong\"],[\"Model alias upgraded silently\",\"Behavior changed without controlled release\"],[\"Prompt changed without evals\",\"Behavioral regression passed normal unit tests\"],[\"RAG index stale\",\"Generation model was blamed for retrieval\u002Fdata failure\"],[\"Only final answer is logged\",\"Root cause in retrieval\u002Ftool\u002Fcontext trajectory is invisible\"],[\"Provider fallback is silent\",\"Different model\u002Fdata path changes behavior without attribution\"],[\"Token cost tracked globally\",\"Expensive workflows cannot be localized\"],[\"Judge model changed\",\"Evaluation scores drift without application change\"],[\"Production traces never become tests\",\"Known failures repeatedly return\"],[\"Local model stays loaded indefinitely\",\"VRAM\u002Fresource pressure becomes operational instability\"],[\"Permissions encoded only in prompt\",\"Model behavior is mistaken for authorization\"],[\"One eval score gates everything\",\"Different quality dimensions are collapsed into a misleading number\"],[\"Model registry exists but prompt\u002Findex versions do not\",\"Application lineage remains incomplete\"]]},\"tunes\":{}},{\"id\":\"h-misconceptions\",\"type\":\"header\",\"data\":{\"text\":\"Common misconceptions\",\"level\":2},\"tunes\":{}},{\"id\":\"misconceptions-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Misconception\",\"Correction\"],[\"“LLMOps replaces MLOps.”\",\"LLMOps extends MLOps principles to LLM-specific application behavior.\"],[\"“LLMOps is prompt engineering.”\",\"Prompts are one artifact among models, providers, context, retrieval, tools, evals and runtime.\"],[\"“Hosted APIs remove operations work.”\",\"They remove some model-serving\u002Ftraining work but add provider lifecycle, version and dependency management.\"],[\"“If the API is stable, the app is stable.”\",\"Model behavior and provider\u002Fmodel snapshots can change independently of API schema.\"],[\"“RAG is just data preprocessing.”\",\"In production it has its own ingestion, index, retrieval and freshness lifecycle.\"],[\"“LLM outputs cannot be tested.”\",\"They can be evaluated with deterministic, reference, judge and human criteria.\"],[\"“LLM judges are objective ground truth.”\",\"They are model-based evaluators that also require calibration and version control.\"],[\"“A local model eliminates LLMOps.”\",\"Local serving adds model files, VRAM, load\u002Funload, runtime health and upgrade concerns.\"],[\"“Observability means token counts.”\",\"Useful observability follows prompts, retrievals, tools, model spans and outcomes.\"],[\"“Continuous training is mandatory.”\",\"Many LLM apps use continuous evaluation without training the foundation model.\"]]},\"tunes\":{}},{\"id\":\"h-design\",\"type\":\"header\",\"data\":{\"text\":\"A practical LLMOps design sequence\",\"level\":2},\"tunes\":{}},{\"id\":\"design-flow\",\"type\":\"processFlow\",\"data\":{\"title\":\"Operate the complete behavior-producing system\",\"orientation\":\"auto\",\"steps\":[{\"label\":\"1. Define the behavior unit\",\"description\":\"List every component that can materially change output: model, prompt, retrieval, tools, context and policy.\"},{\"label\":\"2. Establish application lineage\",\"description\":\"Version code, model\u002Fprovider, prompts, eval datasets, retrieval configuration and tool contracts.\"},{\"label\":\"3. Build representative eval datasets\",\"description\":\"Use expected success\u002Ffailure cases from design and production.\"},{\"label\":\"4. Separate deterministic and behavioral tests\",\"description\":\"Keep schema\u002Fsecurity assertions distinct from semantic output evaluation.\"},{\"label\":\"5. Trace end-to-end execution\",\"description\":\"Instrument model, retrieval, reranking, tools and agent\u002Fruntime spans.\"},{\"label\":\"6. Define release gates\",\"description\":\"Set quality, safety, latency and cost thresholds.\"},{\"label\":\"7. Pin or explicitly record model versions\",\"description\":\"Treat model\u002Fprovider changes as release events.\"},{\"label\":\"8. Deploy progressively\",\"description\":\"Use flags, canaries or staged rollout where consequence warrants it.\"},{\"label\":\"9. Evaluate production traces\",\"description\":\"Measure real task behavior and identify recurrent failures.\"},{\"label\":\"10. Feed failures back into eval datasets\",\"description\":\"Turn incidents and corrections into permanent regression coverage.\"},{\"label\":\"11. Monitor provider and data lifecycles\",\"description\":\"Track deprecations, index freshness, source changes and runtime availability.\"},{\"label\":\"12. Retire obsolete versions cleanly\",\"description\":\"Remove old prompts\u002Fmodels\u002Findexes\u002Fcredentials after migration and evidence retention decisions.\"}]},\"tunes\":{}},{\"id\":\"h-checklist\",\"type\":\"header\",\"data\":{\"text\":\"LLMOps architecture checklist\",\"level\":2},\"tunes\":{}},{\"id\":\"checklist-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Question\",\"Expected evidence\"],[\"Which model\u002Fprovider\u002Fversion served the request?\",\"Traceable model identity\"],[\"Which prompt\u002Finstructions were active?\",\"Versioned application code\u002Fconfig\"],[\"Which context reached the model?\",\"Context\u002Fretrieval trace\"],[\"Which corpus\u002Findex version was used?\",\"Retrieval lineage\"],[\"Which tools were available and called?\",\"Tool schema + trajectory trace\"],[\"Which permissions applied?\",\"Runtime authorization record\"],[\"How is quality measured?\",\"Versioned eval dataset + scorers\"],[\"How are model upgrades tested?\",\"Behavioral regression suite\"],[\"How is production quality sampled?\",\"Trace evaluation\u002Ffeedback process\"],[\"Can one failure be reproduced approximately?\",\"Model\u002Fcontext\u002Fprovider\u002Fapplication lineage\"],[\"Where is cost spent?\",\"Per-trace model\u002Ftool\u002Fretrieval attribution\"],[\"What triggers rollback?\",\"Defined quality\u002Fsafety\u002Fcost\u002Favailability threshold\"],[\"How are provider deprecations handled?\",\"Migration\u002Ffallback process\"],[\"How are local models operated?\",\"Health, resource, load\u002Funload and version controls\"]]},\"tunes\":{}},{\"id\":\"h-edge\",\"type\":\"header\",\"data\":{\"text\":\"Edge cases and limitations\",\"level\":2},\"tunes\":{}},{\"id\":\"p-edge-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"A simple application that calls one fixed hosted model with no retrieval or tools may need only lightweight LLMOps: versioned prompt code, evals, model pinning, basic tracing and provider monitoring.\"},\"tunes\":{}},{\"id\":\"p-edge-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"A self-hosted fine-tuned model may require nearly the full classical MLOps stack plus LLM-specific application evaluation, making the boundary between MLOps and LLMOps intentionally blurry.\"},\"tunes\":{}},{\"id\":\"p-edge-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"An agent platform can have minimal model-training operations but substantial runtime operations because failures occur in tool selection, state and orchestration.\"},\"tunes\":{}},{\"id\":\"p-edge-4\",\"type\":\"paragraph\",\"data\":{\"text\":\"A RAG-heavy system can be operationally dominated by document ingestion and retrieval quality rather than model serving.\"},\"tunes\":{}},{\"id\":\"p-edge-5\",\"type\":\"paragraph\",\"data\":{\"text\":\"Terminology will continue to evolve. The durable architecture question is not which “Ops” label wins, but which artifacts produce behavior and therefore must be versioned, evaluated, observed and governed.\"},\"tunes\":{}},{\"id\":\"h-change\",\"type\":\"header\",\"data\":{\"text\":\"What would change this answer?\",\"level\":2},\"tunes\":{}},{\"id\":\"p-change-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"If foundation-model providers standardize perfectly stable model behavior and long-term version support, provider\u002Fsnapshot management could become less operationally significant.\"},\"tunes\":{}},{\"id\":\"p-change-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"If applications increasingly own fine-tuning or training, classical MLOps concerns become more central again.\"},\"tunes\":{}},{\"id\":\"p-change-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The operational principle would remain: every component that can materially change production behavior belongs in lineage, testing, observability and change control.\"},\"tunes\":{}},{\"id\":\"h-related\",\"type\":\"header\",\"data\":{\"text\":\"Related canonical knowledge\",\"level\":2},\"tunes\":{}},{\"id\":\"p-related-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps sits below AI Governance and Enterprise AI Architecture: governance defines which changes require evidence and approval, while LLMOps provides the operational machinery to version, evaluate, deploy and observe those changes.\"},\"tunes\":{}},{\"id\":\"p-related-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Context Engineering and RAG are operational subdomains inside many LLM applications because context and retrieval can change behavior independently of the model.\"},\"tunes\":{}},{\"id\":\"p-related-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Agentic AI extends LLMOps further into trajectory, permissions and tool-runtime operations.\"},\"tunes\":{}},{\"id\":\"ref-memory\",\"type\":\"referralArticle\",\"data\":{\"url\":\"https:\u002F\u002Fstajic.de\u002Fblog\u002Fai-agent-memory-is-not-rag-how-to-separate-memory-retrieval-state-and-context\",\"title\":\"AI Agent Memory Is Not RAG: How to Separate Memory, Retrieval, State and Context\",\"excerpt\":\"Operational reliability improves when memory, retrieval, application state and model context remain separate lifecycle objects.\",\"ctaLabel\":\"Read the architecture article\"},\"tunes\":{}},{\"id\":\"ref-avb\",\"type\":\"referralArticle\",\"data\":{\"url\":\"https:\u002F\u002Fstajic.de\u002Fblog\u002Fthe-answer-validity-boundary-the-missing-layer-between-relevance-and-reliable-ai-answers\",\"title\":\"The Answer Validity Boundary: The Missing Layer Between Relevance and Reliable AI Answers\",\"excerpt\":\"LLMOps evaluation should preserve the version, scope and evidence conditions under which an answer remains supported.\",\"ctaLabel\":\"Read the Answer Validity Boundary\"},\"tunes\":{}},{\"id\":\"h-faq\",\"type\":\"header\",\"data\":{\"text\":\"Frequently asked questions\",\"level\":2},\"tunes\":{}},{\"id\":\"faq\",\"type\":\"faq\",\"data\":{\"title\":\"MLOps vs LLMOps FAQ\",\"items\":[{\"id\":\"faq1\",\"question\":\"What is the difference between MLOps and LLMOps?\",\"answer\":\"MLOps operates machine-learning systems across data, training, deployment and monitoring. LLMOps extends those practices to LLM applications where prompts, context, retrieval, providers, tools and evaluations also materially affect behavior.\"},{\"id\":\"faq2\",\"question\":\"Does LLMOps replace MLOps?\",\"answer\":\"No. LLMOps reuses MLOps disciplines such as CI\u002FCD, lineage, evaluation, deployment and monitoring and adds LLM-specific operational concerns.\"},{\"id\":\"faq3\",\"question\":\"Do LLM applications need continuous training?\",\"answer\":\"Not necessarily. Many use external foundation models and instead rely on continuous evaluation of prompts, models, retrieval and application behavior. Fine-tuned or self-trained systems can still require training pipelines.\"},{\"id\":\"faq4\",\"question\":\"Why are evals so important in LLMOps?\",\"answer\":\"Generative outputs are open-ended and model behavior can change across prompts, snapshots and context. Evals provide repeatable evidence that a release still meets defined quality and safety criteria.\"},{\"id\":\"faq5\",\"question\":\"What should be versioned in LLMOps?\",\"answer\":\"At minimum: application code, model\u002Fprovider\u002Fversion, prompts, eval datasets\u002Fscorers, retrieval configuration\u002Findexes, tool schemas, context rules and relevant safety\u002Fpermission configuration.\"},{\"id\":\"faq6\",\"question\":\"Is prompt versioning enough?\",\"answer\":\"No. The same prompt can behave differently with another model, retrieval set, context order, tool surface or provider.\"},{\"id\":\"faq7\",\"question\":\"What is GenAIOps?\",\"answer\":\"GenAIOps is another industry term for operating generative-AI applications. Some vendors use it interchangeably or as a broader label than LLMOps.\"},{\"id\":\"faq8\",\"question\":\"How do you monitor an LLM application?\",\"answer\":\"Monitor end-to-end traces including model calls, prompts\u002Fcontext, retrieval, tools, latency, token\u002Fcost, quality samples, safety and final task outcomes.\"},{\"id\":\"faq9\",\"question\":\"Can local LLMs use LLMOps practices?\",\"answer\":\"Yes. Local models add their own operational concerns such as model files, hardware\u002FVRAM, load\u002Funload, runtime health, quantization and upgrade management.\"}]},\"tunes\":{}},{\"id\":\"h-glossary\",\"type\":\"header\",\"data\":{\"text\":\"Glossary\",\"level\":2},\"tunes\":{}},{\"id\":\"glossary\",\"type\":\"glossary\",\"data\":{\"title\":\"Key MLOps and LLMOps terms\",\"entries\":[{\"term\":\"MLOps\",\"definition\":\"Engineering practices for building, deploying, monitoring and maintaining machine-learning systems and their data\u002Fmodel lifecycle.\",\"anchor\":\"mlops\"},{\"term\":\"LLMOps\",\"definition\":\"Operational practices for production applications whose behavior materially depends on large language models and surrounding prompts, context, retrieval, tools and runtime.\",\"anchor\":\"llmops\"},{\"term\":\"GenAIOps\",\"definition\":\"Operational discipline for generative-AI applications; often used as a broader or alternate label for LLMOps.\",\"anchor\":\"genaiops\"},{\"term\":\"Continuous training\",\"definition\":\"Automated or repeated retraining and serving of ML models as data or implementations change.\",\"anchor\":\"continuous-training\"},{\"term\":\"Continuous evaluation\",\"definition\":\"Repeated evaluation of candidate and production AI behavior against versioned datasets and criteria.\",\"anchor\":\"continuous-evaluation\"},{\"term\":\"Model snapshot\",\"definition\":\"A concrete version of a hosted or packaged model whose behavior can be tested and referenced.\",\"anchor\":\"model-snapshot\"},{\"term\":\"Application lineage\",\"definition\":\"Traceable relationship among code, model\u002Fprovider, prompts, data\u002Fretrieval, tools, runtime and release configuration.\",\"anchor\":\"application-lineage\"},{\"term\":\"Trace\",\"definition\":\"Structured record of one application execution containing spans such as model calls, retrievals and tool operations.\",\"anchor\":\"trace\"},{\"term\":\"Eval dataset\",\"definition\":\"Versioned set of representative inputs, expectations and optionally traces\u002Foutputs used to measure behavior.\",\"anchor\":\"eval-dataset\"},{\"term\":\"LLM judge\",\"definition\":\"A language model used as an evaluator for qualitative or semantic criteria; it is itself a versioned evaluation dependency.\",\"anchor\":\"llm-judge\"},{\"term\":\"Behavioral regression\",\"definition\":\"A degradation in application output or trajectory despite interfaces and code continuing to execute successfully.\",\"anchor\":\"behavioral-regression\"},{\"term\":\"Provider routing\",\"definition\":\"Policy for selecting among available model providers\u002Fendpoints according to capability, cost, latency, privacy or availability.\",\"anchor\":\"provider-routing\"}]},\"tunes\":{}},{\"id\":\"h-conclusion\",\"type\":\"header\",\"data\":{\"text\":\"Conclusion\",\"level\":2},\"tunes\":{}},{\"id\":\"p-conclusion-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLOps and LLMOps share the same engineering objective: make AI systems reproducible enough, testable enough and observable enough to operate reliably in production.\"},\"tunes\":{}},{\"id\":\"p-conclusion-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The difference is the shape of the system. Classical MLOps often centers on training and serving model artifacts; LLMOps must operate a behavioral stack in which model snapshots, prompts, context, retrieval, tools, permissions and providers can change independently.\"},\"tunes\":{}},{\"id\":\"p-conclusion-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The shortest useful rule is: version, evaluate and observe everything that can materially change the LLM application's behavior — not only the model.\"},\"tunes\":{}},{\"id\":\"h-sources\",\"type\":\"header\",\"data\":{\"text\":\"Primary sources and current documentation\",\"level\":2},\"tunes\":{}},{\"id\":\"p-sources-note\",\"type\":\"paragraph\",\"data\":{\"text\":\"The sources below ground the MLOps baseline and the current operational patterns for LLM and agent applications. Project sections are original implementation evidence and are intentionally narrower than claims about a complete LLMOps platform.\"},\"tunes\":{}},{\"id\":\"src-google-mlops\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdocs.cloud.google.com\u002Farchitecture\u002Fmlops-continuous-delivery-and-automation-pipelines-in-machine-learning\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"Google Cloud — MLOps: Continuous delivery and automation pipelines\",\"description\":\"Reference architecture describing CI, CD, continuous training, model registry, metadata, serving and monitoring for ML systems.\"}},\"tunes\":{}},{\"id\":\"src-aws-lineage\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops02-bp04.html\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"AWS Machine Learning Lens — Model lineage\",\"description\":\"Current guidance for tracking code, data, models, environments and infrastructure across ML releases.\"}},\"tunes\":{}},{\"id\":\"src-aws-monitor\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops06-bp02.html\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"AWS Machine Learning Lens — Model observability and tracking\",\"description\":\"Current guidance for production model monitoring, drift, endpoint health and lineage.\"}},\"tunes\":{}},{\"id\":\"src-azure-llmops\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fmachine-learning\u002Fprompt-flow\u002Fhow-to-end-to-end-llmops-with-prompt-flow\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"Microsoft Azure — GenAIOps \u002F LLMOps lifecycle\",\"description\":\"Official guidance describing GenAIOps, sometimes called LLMOps, across initialization, experimentation, evaluation\u002Frefinement and deployment.\"}},\"tunes\":{}},{\"id\":\"src-mlflow-genai\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002F\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"MLflow — Agents and LLM applications\",\"description\":\"Current GenAI operations documentation covering tracing, evaluation, prompts and production observability for LLM applications and agents.\"}},\"tunes\":{}},{\"id\":\"src-mlflow-traces\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fwww.mlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Feval-monitor\u002Frunning-evaluation\u002Ftraces\u002F\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"MLflow — Evaluating production traces\",\"description\":\"Current guidance for evaluating complete LLM\u002Fagent traces, including retrieval and tool-call trajectories.\"}},\"tunes\":{}},{\"id\":\"src-mlflow-prompt-eval\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Fprompt-registry\u002Fevaluate-prompts\u002F\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"MLflow — Evaluating prompts\",\"description\":\"Current prompt\u002Fmodel evaluation workflow using versioned prompts, datasets, scorers and traces.\"}},\"tunes\":{}},{\"id\":\"src-openai-api\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Freference\u002Foverview\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"OpenAI API — Versioning and model snapshots\",\"description\":\"Current API guidance recommending pinned model versions and evals because prompting behavior can change between snapshots.\"}},\"tunes\":{}},{\"id\":\"src-openai-prompting\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fguides\u002Fprompting\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"OpenAI — Prompting\",\"description\":\"Current guidance to treat production prompts as application code, version them through source control and cover changes with tests and evaluation checks.\"}},\"tunes\":{}},{\"id\":\"src-openai-deprecations\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fdeprecations\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"OpenAI — Deprecations\",\"description\":\"Current provider lifecycle evidence showing model and platform-surface retirement as an operational dependency.\"}},\"tunes\":{}},{\"id\":\"src-openai-promptfoo\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdevelopers.openai.com\u002Fcookbook\u002Fexamples\u002Fevaluation\u002Fmoving-from-openai-evals-to-promptfoo\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"OpenAI — Moving evaluation workflows to Promptfoo\",\"description\":\"Current 2026 migration guidance illustrating why evaluation assets should remain portable as provider tooling changes.\"}},\"tunes\":{}}],\"version\":\"2.31.6\"}",{"time":2284,"blocks":2285,"version":3391},1791487321430,[2286,2290,2295,2300,2305,2310,2314,2318,2322,2326,2330,2334,2338,2342,2346,2350,2354,2358,2362,2391,2395,2399,2403,2407,2411,2443,2447,2451,2455,2459,2463,2512,2516,2520,2524,2528,2532,2536,2540,2544,2548,2552,2556,2560,2564,2568,2572,2576,2580,2584,2588,2592,2599,2603,2607,2611,2615,2620,2624,2628,2632,2636,2640,2644,2648,2652,2656,2660,2664,2668,2675,2679,2683,2687,2691,2695,2699,2703,2707,2711,2715,2719,2723,2727,2757,2761,2765,2769,2773,2777,2781,2785,2789,2793,2827,2831,2835,2839,2843,2847,2851,2855,2859,2863,2867,2871,2875,2879,2883,2887,2891,2895,2899,2903,2907,2911,2915,2919,2923,2927,2931,2962,2967,2971,3014,3018,3055,3059,3100,3104,3153,3157,3161,3165,3169,3173,3177,3181,3185,3189,3193,3197,3201,3205,3209,3216,3223,3227,3259,3263,3298,3302,3306,3310,3314,3318,3322,3328,3334,3340,3346,3352,3358,3365,3372,3378,3384],{"id":214,"data":2287,"type":217,"tunes":2289},{"text":2288},"MLOps is the engineering discipline for reliably developing, deploying, versioning and operating machine-learning systems; LLMOps extends that discipline to applications built around large language models, where production behavior depends not only on a model artifact but also on prompts, context, retrieval, provider\u002Fmodel versions, tool calls, safety controls and evaluation pipelines. LLMOps does not replace MLOps. It changes the operational unit from “a model plus serving pipeline” toward “an evolving LLM application whose behavior emerges from several independently changing components.”",{},{"id":220,"data":2291,"type":225,"tunes":2294},{"body":2292,"title":2293,"variant":224},"\u003Cstrong>MLOps operates ML systems. LLMOps operates LLM applications.\u003C\u002Fstrong>\u003Cbr>\u003Cbr>Classical MLOps commonly centers on data pipelines, training, validation, model registry, deployment, drift and retraining. LLMOps keeps those disciplines where relevant, but often adds prompt\u002Fcontext versioning, model\u002Fprovider abstraction, RAG indexes, agent\u002Ftool traces, semantic evaluations, safety tests, token\u002Fcost monitoring and regression testing across rapidly changing model snapshots.","Direct answer",{},{"id":228,"data":2296,"type":225,"tunes":2299},{"body":2297,"title":2298,"variant":232},"A production LLM application can fail even when the prompt is unchanged: the provider can change a model snapshot, a RAG corpus can become stale, a reranker can regress, tool permissions can change, context assembly can drop evidence, or an agent can take a wrong trajectory. LLMOps therefore has to observe and version the system around the model, not only prompt text.","LLMOps is not just prompt management",{},{"id":235,"data":2301,"type":225,"tunes":2304},{"body":2302,"title":2303,"variant":239},"\u003Cstrong>LLMOps\u003C\u002Fstrong>, \u003Cstrong>GenAIOps\u003C\u002Fstrong> and related terms are widely used engineering labels, but they are not one universal formal standard with a single canonical lifecycle. Microsoft currently describes GenAIOps as “sometimes called LLMOps,” while MLflow groups operational tooling around agents and LLM applications. This article uses LLMOps as a practical architecture term for operating production systems whose behavior materially depends on LLMs.","Terminology boundary",{},{"id":242,"data":2306,"type":225,"tunes":2309},{"body":2307,"title":2308,"variant":239},"The operational surface is changing quickly. OpenAI currently recommends pinning model snapshots and running evals because prompting behavior can change between snapshots, and several older platform-specific prompt\u002Feval surfaces are being retired in 2026. The stable architectural lesson is to keep prompts, tests and evals portable and versioned with the application rather than depend on one provider's dashboard object model.","Current-source note — 8 October 2026",{},{"id":248,"data":2311,"type":253,"tunes":2313},{"title":2312,"maxLevel":251,"minLevel":252},"Contents",{},{"id":256,"data":2315,"type":41,"tunes":2317},{"text":2316,"level":252},"What MLOps really means",{},{"id":261,"data":2319,"type":217,"tunes":2321},{"text":2320},"MLOps applies software-engineering and operational discipline to machine-learning systems. The production challenge is broader than training a model: data collection, data validation, experimentation, reproducibility, model evaluation, deployment, infrastructure and monitoring all have to work together.",{},{"id":266,"data":2323,"type":217,"tunes":2325},{"text":2324},"Google's MLOps architecture guidance frames the discipline around continuous integration, continuous delivery and continuous training. CI validates not only code but also data, schemas and models; CD deploys ML pipelines and prediction services; CT can retrain and redeploy models as data or implementations change.",{},{"id":271,"data":2327,"type":217,"tunes":2329},{"text":2328},"AWS guidance adds the same operational concerns from another angle: model lineage, model\u002Fversion traceability, drift monitoring and production-quality monitoring are core parts of keeping ML systems reliable after deployment.",{},{"id":276,"data":2331,"type":41,"tunes":2333},{"text":2332,"level":252},"What changes when the model is an LLM",{},{"id":281,"data":2335,"type":217,"tunes":2337},{"text":2336},"Large language models change the production problem because the application often does not own the complete model-training lifecycle. A team may call a hosted model API, run an open model locally, switch between providers or use several models for different tasks.",{},{"id":286,"data":2339,"type":217,"tunes":2341},{"text":2340},"The model is therefore only one versioned dependency inside a larger behavioral system. Prompts, retrieval results, context order, tools, model snapshot, temperature\u002Freasoning settings, safety filters and runtime orchestration can all change the output.",{},{"id":291,"data":2343,"type":217,"tunes":2345},{"text":2344},"This creates a broader operational question: which combination of model, context, data, prompt, tools and runtime produced this behavior? LLMOps exists to make that question answerable and the answer reproducible enough for engineering work.",{},{"id":296,"data":2347,"type":41,"tunes":2349},{"text":2348,"level":252},"The simplest example",{},{"id":301,"data":2351,"type":217,"tunes":2353},{"text":2352},"Suppose an application answers internal policy questions.",{},{"id":306,"data":2355,"type":217,"tunes":2357},{"text":2356},"In a classical ML framing, you might version a trained classifier, deploy it and monitor prediction quality. In an LLM application, the answer might depend on a hosted model snapshot, a system prompt, an embedding model, a vector index, retrieval filters, a reranker and the final selected context.",{},{"id":311,"data":2359,"type":217,"tunes":2361},{"text":2360},"Changing any one of those components can change the final answer even though the application endpoint and user question stay identical.",{},{"id":316,"data":2363,"type":345,"tunes":2390},{"steps":2364,"title":2389,"orientation":344},[2365,2368,2371,2374,2377,2380,2383,2386],{"label":2366,"description":2367},"1. Change one component","Prompt, model, provider, retrieval setting, tool schema or application code changes.",{"label":2369,"description":2370},"2. Run deterministic tests","Validate schemas, permissions, tool contracts, retrieval filters and application behavior.",{"label":2372,"description":2373},"3. Run behavioral evals","Compare representative outputs, retrieval quality and agent\u002Ftool trajectories against acceptance criteria.",{"label":2375,"description":2376},"4. Compare cost and latency","Measure token use, model calls, retrieval\u002Ftool overhead and response latency.",{"label":2378,"description":2379},"5. Deploy controlled version","Ship the concrete application configuration with model\u002Fprovider versions recorded.",{"label":2381,"description":2382},"6. Trace production behavior","Capture relevant model, retrieval, tool and runtime spans.",{"label":2384,"description":2385},"7. Evaluate production traces","Sample real executions for quality, grounding, safety and task success.",{"label":2387,"description":2388},"8. Roll back or iterate","Use regression evidence and operational signals to decide the next release.","A typical LLMOps release path",{},{"id":348,"data":2392,"type":41,"tunes":2394},{"text":2393,"level":252},"Where the simple example stops",{},{"id":353,"data":2396,"type":217,"tunes":2398},{"text":2397},"Some LLM systems still train or fine-tune their own models, so traditional MLOps practices such as training pipelines, model registry and data lineage remain directly relevant.",{},{"id":358,"data":2400,"type":217,"tunes":2402},{"text":2401},"Other systems use only external foundation-model APIs and never run continuous training. Their main operational workload is application evaluation, model\u002Fprovider change management, prompt\u002Fcontext versioning, retrieval quality and observability.",{},{"id":363,"data":2404,"type":217,"tunes":2406},{"text":2405},"There is therefore no single universal “LLMOps pipeline.” The exact lifecycle depends on whether you train, fine-tune, self-host, retrieve external knowledge, run agents or depend on managed model APIs.",{},{"id":368,"data":2408,"type":41,"tunes":2410},{"text":2409,"level":252},"MLOps vs LLMOps",{},{"id":373,"data":2412,"type":418,"tunes":2442},{"rows":2413,"title":2438,"layout":410,"columns":2439},[2414,2417,2420,2423,2426,2429,2432,2435],{"id":377,"label":2415,"values":2416},"Primary operational unit",[380,380],{"id":382,"label":2418,"values":2419},"Model ownership",[380,380],{"id":386,"label":2421,"values":2422},"Typical change",[380,380],{"id":390,"label":2424,"values":2425},"Evaluation",[380,380],{"id":394,"label":2427,"values":2428},"Production monitoring",[380,380],{"id":398,"label":2430,"values":2431},"Continuous training",[380,380],{"id":402,"label":2433,"values":2434},"Versioned artifacts",[380,380],{"id":406,"label":2436,"values":2437},"Rollback target",[380,380],"What stays the same and what expands",[2440,2441],{"id":413,"label":414},{"id":416,"label":417},{},{"id":421,"data":2444,"type":41,"tunes":2446},{"text":2445,"level":252},"LLMOps extends MLOps rather than replacing it",{},{"id":426,"data":2448,"type":217,"tunes":2450},{"text":2449},"The core operational principles do not disappear: source control, CI\u002FCD, reproducibility, lineage, deployment controls, monitoring, rollback and measurable acceptance criteria remain essential.",{},{"id":431,"data":2452,"type":217,"tunes":2454},{"text":2453},"The extension is that more behavior-defining artifacts now sit outside the model weights. A managed foundation model can change behavior through snapshot upgrades, while application output can change through prompt or retrieval changes without any model retraining.",{},{"id":436,"data":2456,"type":217,"tunes":2458},{"text":2457},"This is why the useful hierarchy is usually DevOps → MLOps → LLMOps\u002FGenAIOps as increasingly specialized operational concerns, not three mutually exclusive practices.",{},{"id":441,"data":2460,"type":41,"tunes":2462},{"text":2461,"level":252},"What has to be versioned in LLMOps?",{},{"id":446,"data":2464,"type":410,"tunes":2511},{"content":2465,"stretched":42,"withHeadings":13},[2466,2469,2472,2475,2478,2481,2484,2487,2490,2493,2496,2499,2502,2505,2508],[2467,2468],"Artifact","Why it matters",[2470,2471],"Application code","Defines orchestration, validation, retries and business behavior",[2473,2474],"Model family + snapshot\u002Fversion","Different snapshots can produce different behavior",[2476,2477],"Provider \u002F endpoint","Changes data flow, latency, limits, pricing and availability",[2479,2480],"Prompt\u002Finstruction code","Changes model behavior even with same model",[2482,2483],"Generation\u002Freasoning parameters","Can alter determinism, latency, depth and cost",[2485,2486],"Eval dataset","Defines what “good enough” is tested against",[2488,2489],"Scorers \u002F graders","Define how quality is measured",[2491,2492],"Embedding model","Changes vector representation and retrieval behavior",[2494,2495],"Chunking\u002Findex configuration","Changes what can be retrieved",[2497,2498],"Reranker \u002F retrieval fusion","Changes result ordering",[2500,2501],"Tool schemas","Change what the model can request and how",[2503,2504],"Permission profile","Changes what tool actions may actually execute",[2506,2507],"Context assembly rules","Change what evidence and state reach the model",[2509,2510],"Safety\u002Fguardrail configuration","Changes allowed or blocked behavior",{},{"id":496,"data":2513,"type":41,"tunes":2515},{"text":2514,"level":252},"Model snapshots become release dependencies",{},{"id":501,"data":2517,"type":217,"tunes":2519},{"text":2518},"With hosted LLMs, the team may not control model training, but it still controls which model or snapshot the application calls.",{},{"id":506,"data":2521,"type":217,"tunes":2523},{"text":2522},"OpenAI's current API guidance explicitly warns that prompting behavior can change between model snapshots and recommends pinning production applications to specific snapshots where consistency matters, then running evals when upgrading.",{},{"id":511,"data":2525,"type":217,"tunes":2527},{"text":2526},"The operational consequence is straightforward: model upgrades should be treated as application releases, not invisible infrastructure maintenance.",{},{"id":516,"data":2529,"type":41,"tunes":2531},{"text":2530,"level":252},"Provider lifecycle becomes part of operations",{},{"id":521,"data":2533,"type":217,"tunes":2535},{"text":2534},"LLM applications often depend on provider rate limits, deprecation schedules, API semantics, context limits, data-handling rules and pricing.",{},{"id":526,"data":2537,"type":217,"tunes":2539},{"text":2538},"A provider can deprecate a model while your application code remains unchanged. OpenAI's current deprecation schedule, for example, includes 2026 retirement dates for older model snapshots and platform surfaces.",{},{"id":531,"data":2541,"type":217,"tunes":2543},{"text":2542},"LLMOps therefore needs provider lifecycle tracking, migration testing and fallback decisions in addition to model-quality monitoring.",{},{"id":536,"data":2545,"type":41,"tunes":2547},{"text":2546,"level":252},"Prompts behave like production code",{},{"id":541,"data":2549,"type":217,"tunes":2551},{"text":2550},"Prompts are executable behavioral configuration. Small changes can alter output quality, tool selection and policy interpretation.",{},{"id":546,"data":2553,"type":217,"tunes":2555},{"text":2554},"OpenAI's current guidance recommends storing production prompts in application code, reviewing prompt changes through pull requests, using typed inputs and covering changes with tests and evaluation checks.",{},{"id":551,"data":2557,"type":217,"tunes":2559},{"text":2558},"That makes prompt versioning less like editing marketing copy and more like changing a function whose output is probabilistic and model-dependent.",{},{"id":556,"data":2561,"type":41,"tunes":2563},{"text":2562,"level":252},"Context engineering becomes an operational concern",{},{"id":561,"data":2565,"type":217,"tunes":2567},{"text":2566},"The production model rarely receives only a static prompt. It may receive conversation history, retrieved documents, tool outputs, memory, current application state and policy instructions.",{},{"id":566,"data":2569,"type":217,"tunes":2571},{"text":2570},"LLMOps must therefore observe context assembly: which evidence was selected, which state version was current, whether truncation occurred and whether important instructions survived compaction.",{},{"id":571,"data":2573,"type":217,"tunes":2575},{"text":2574},"A model regression and a context regression can look identical at the final answer. Tracing the actual context path is what lets the team separate them.",{},{"id":576,"data":2577,"type":41,"tunes":2579},{"text":2578,"level":252},"RAG creates its own operational lifecycle",{},{"id":581,"data":2581,"type":217,"tunes":2583},{"text":2582},"A RAG system introduces a second production pipeline beside model inference: ingestion, extraction, chunking, metadata, embeddings, indexes, retrieval, reranking and context selection.",{},{"id":586,"data":2585,"type":217,"tunes":2587},{"text":2586},"The knowledge corpus can change every day even when the model and prompt do not. A stale index or broken metadata filter can therefore degrade answer quality without any model drift.",{},{"id":591,"data":2589,"type":217,"tunes":2591},{"text":2590},"LLMOps for RAG should track corpus\u002Findex version, embedding model, chunking policy, retrieval configuration, source freshness and retrieval metrics separately from generation quality.",{},{"id":596,"data":2593,"type":602,"tunes":2598},{"url":2594,"title":2595,"excerpt":2596,"ctaLabel":2597},"https:\u002F\u002Fstajic.de\u002Fblog\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method","RAG Failed — But Which Layer Actually Failed? A Diagnostic Method","A production LLM pipeline needs separate observability for source coverage, retrieval, ranking, context assembly and generation.","Read the RAG diagnostic method",{},{"id":605,"data":2600,"type":41,"tunes":2602},{"text":2601,"level":252},"Evals replace “looks good to me” with release evidence",{},{"id":610,"data":2604,"type":217,"tunes":2606},{"text":2605},"Generative outputs are often open-ended, so exact-match tests are insufficient for many tasks. LLMOps adds evaluation datasets and scorers that can measure task success, correctness, safety, groundedness, style or domain-specific acceptance criteria.",{},{"id":615,"data":2608,"type":217,"tunes":2610},{"text":2609},"MLflow's current GenAI evaluation stack supports versioned evaluation datasets, prompt\u002Fmodel comparisons, custom scorers and evaluation over complete traces.",{},{"id":620,"data":2612,"type":217,"tunes":2614},{"text":2613},"The strongest practice is evaluation-driven development: define representative cases and acceptance criteria before or alongside changes, then compare releases against the same evidence.",{},{"id":625,"data":2616,"type":225,"tunes":2619},{"body":2617,"title":2618,"variant":629},"A deployment should not be considered equivalent merely because the API contract still works. If the prompt, model, retrieval or tools changed, the behavioral regression suite should run again.","Behavioral changes need behavioral tests",{},{"id":632,"data":2621,"type":41,"tunes":2623},{"text":2622,"level":252},"LLM-as-a-judge is useful but not ground truth",{},{"id":637,"data":2625,"type":217,"tunes":2627},{"text":2626},"LLM judges can scale evaluation for qualities that are expensive to encode as deterministic assertions, such as relevance, tone or groundedness.",{},{"id":642,"data":2629,"type":217,"tunes":2631},{"text":2630},"However, the judge is another model with its own bias, version and prompt. Judge configuration should therefore be versioned and calibrated against human or deterministic reference cases where consequence matters.",{},{"id":647,"data":2633,"type":217,"tunes":2635},{"text":2634},"A production eval can mix deterministic checks, reference-based metrics, model judges and human review rather than asking one metric to represent every quality dimension.",{},{"id":652,"data":2637,"type":41,"tunes":2639},{"text":2638,"level":252},"Tracing becomes more important than endpoint logs",{},{"id":657,"data":2641,"type":217,"tunes":2643},{"text":2642},"Traditional API logs can tell you that a request took two seconds and returned HTTP 200. They cannot tell you which retrieved chunks were selected, which tool the agent called or which model span consumed most tokens.",{},{"id":662,"data":2645,"type":217,"tunes":2647},{"text":2646},"MLflow's current GenAI tracing captures prompts, retrievals, tool calls and application spans, and its production evaluation flow can score intermediate trajectory information rather than only final text.",{},{"id":667,"data":2649,"type":217,"tunes":2651},{"text":2650},"This is a major LLMOps shift: observability follows the behavioral graph of the application, not only the serving endpoint.",{},{"id":672,"data":2653,"type":41,"tunes":2655},{"text":2654,"level":252},"Agents expand LLMOps into runtime operations",{},{"id":677,"data":2657,"type":217,"tunes":2659},{"text":2658},"An agentic application can perform several model calls, tool invocations and state transitions before producing a result.",{},{"id":682,"data":2661,"type":217,"tunes":2663},{"text":2662},"Operating agents therefore requires step counts, tool-call traces, permission denials, retries, loop detection, human approvals and verified final state in addition to ordinary model latency and token metrics.",{},{"id":687,"data":2665,"type":217,"tunes":2667},{"text":2666},"A correct final answer can hide a bad trajectory, so agent evaluation must inspect the path as well as the result.",{},{"id":692,"data":2669,"type":602,"tunes":2674},{"url":2670,"title":2671,"excerpt":2672,"ctaLabel":2673},"https:\u002F\u002Fstajic.de\u002Fblog\u002Fai-agent-reliability-why-the-final-answer-is-not-enough","AI Agent Reliability: Why the Final Answer Is Not Enough","Why agent production evaluation must include tool calls, state transitions, approvals and recoverability.","Read the agent reliability article",{},{"id":700,"data":2676,"type":41,"tunes":2678},{"text":2677,"level":252},"Tokens, model calls and context become cost variables",{},{"id":705,"data":2680,"type":217,"tunes":2682},{"text":2681},"Classical ML inference cost is often dominated by serving infrastructure or per-prediction compute. LLM applications can add provider token pricing, repeated agent calls, embedding calls, reranking and tool\u002Fruntime overhead.",{},{"id":710,"data":2684,"type":217,"tunes":2686},{"text":2685},"Cost therefore has to be attributed to task or trace, not only to one endpoint. A workflow that makes eight hidden model calls can be functionally correct but operationally unacceptable.",{},{"id":715,"data":2688,"type":217,"tunes":2690},{"text":2689},"Latency behaves the same way: model latency, retrieval, reranking and external tools compose into end-to-end user latency.",{},{"id":720,"data":2692,"type":41,"tunes":2694},{"text":2693,"level":252},"Caching becomes semantic, not only technical",{},{"id":725,"data":2696,"type":217,"tunes":2698},{"text":2697},"LLM systems can cache prompts, embeddings, retrieval results or full responses, but the cache key must reflect the semantics that can change the result.",{},{"id":730,"data":2700,"type":217,"tunes":2702},{"text":2701},"A response cache that ignores model version, tenant, permissions or source freshness can return a technically valid but semantically invalid answer.",{},{"id":735,"data":2704,"type":217,"tunes":2706},{"text":2705},"LLMOps therefore treats cache invalidation as part of model\u002Fcontext\u002Fdata versioning rather than only infrastructure optimization.",{},{"id":740,"data":2708,"type":41,"tunes":2710},{"text":2709,"level":252},"Safety and permissions become release criteria",{},{"id":745,"data":2712,"type":217,"tunes":2714},{"text":2713},"Generative systems can produce unbounded text and agents can trigger external actions. Safety testing therefore sits closer to ordinary CI\u002FCD than in many classical predictive ML systems.",{},{"id":750,"data":2716,"type":217,"tunes":2718},{"text":2717},"Permission checks, prompt-injection tests, tenant-isolation tests and side-effect approvals should be reproducible regression tests where those risks exist.",{},{"id":755,"data":2720,"type":217,"tunes":2722},{"text":2721},"The model may suggest an operation, but the runtime still has to enforce authorization. LLMOps owns the evidence that those controls continue to work after model, prompt or tool changes.",{},{"id":760,"data":2724,"type":41,"tunes":2726},{"text":2725,"level":252},"What CI looks like in LLMOps",{},{"id":765,"data":2728,"type":410,"tunes":2756},{"content":2729,"stretched":42,"withHeadings":13},[2730,2733,2735,2737,2740,2742,2744,2747,2750,2753],[2731,2732],"CI layer","Example checks",[772,2734],"Unit tests, type checks, schema validation",[775,2736],"Template rendering, required variables, policy text, snapshot review",[2738,2739],"Models\u002Fproviders","Compatibility, output schema, capability and regression tests",[781,2741],"Chunking fixtures, filter tests, Recall@k, reranker regression",[784,2743],"Input\u002Foutput schema tests, permission tests, idempotency tests",[2745,2746],"Agents","Trajectory fixtures, loop limits, handoff\u002Ftool-selection tests",[2748,2749],"Security","Prompt injection, unauthorized tools, cross-tenant negative tests",[2751,2752],"Behavioral evals","Task success, correctness, grounding, safety, domain criteria",[2754,2755],"Operational","Latency, token\u002Fcost budgets, timeout\u002Ffallback behavior",{},{"id":800,"data":2758,"type":41,"tunes":2760},{"text":2759,"level":252},"What CD looks like in LLMOps",{},{"id":805,"data":2762,"type":217,"tunes":2764},{"text":2763},"A production release may deploy no new model artifact at all. It may simply ship a new prompt, retrieval configuration, tool set or provider mapping.",{},{"id":810,"data":2766,"type":217,"tunes":2768},{"text":2767},"The release bundle should therefore identify the complete behavior-defining configuration rather than only the application container image.",{},{"id":815,"data":2770,"type":217,"tunes":2772},{"text":2771},"Feature flags, staged rollout, shadow evaluation, canary traffic and rollback are useful because LLM behavior can regress in ways that static contract tests do not detect.",{},{"id":820,"data":2774,"type":41,"tunes":2776},{"text":2775,"level":252},"Continuous training becomes optional; continuous evaluation becomes central",{},{"id":825,"data":2778,"type":217,"tunes":2780},{"text":2779},"Traditional MLOps often emphasizes continuous training when new data or drift justifies retraining.",{},{"id":830,"data":2782,"type":217,"tunes":2784},{"text":2783},"Many LLM applications never train the foundation model. Their equivalent continuous loop is continuous evaluation: collect failures and representative production cases, add them to evaluation datasets, test candidate prompt\u002Fmodel\u002Fretrieval changes and redeploy only when evidence improves.",{},{"id":835,"data":2786,"type":217,"tunes":2788},{"text":2787},"Fine-tuning can reintroduce a training lifecycle, but it should sit inside the same broader evaluation and release process.",{},{"id":840,"data":2790,"type":41,"tunes":2792},{"text":2791,"level":252},"What should be monitored in production?",{},{"id":845,"data":2794,"type":410,"tunes":2826},{"content":2795,"stretched":42,"withHeadings":13},[2796,2799,2802,2805,2808,2811,2814,2816,2818,2820,2823],[2797,2798],"Signal class","Examples",[2800,2801],"System health","Errors, timeouts, endpoint availability",[2803,2804],"Model\u002Fprovider","Model ID, snapshot, rate limits, provider errors",[2806,2807],"Latency","End-to-end, model, retrieval, tool and reranker spans",[2809,2810],"Cost","Input\u002Foutput tokens, embeddings, tool\u002FAPI spend",[2812,2813],"Quality","Sampled task success, correctness, relevance, groundedness",[781,2815],"Retrieval recall proxies, empty retrieval, stale sources, citation coverage",[2745,2817],"Tool selection, retries, loops, handoffs, approval frequency",[2748,2819],"Denied actions, prompt-injection indicators, tenant-boundary failures",[2821,2822],"User feedback","Corrections, abandonment, escalation, explicit ratings",[2824,2825],"Change drift","Provider\u002Fmodel\u002Fconfig changes relative to approved release",{},{"id":880,"data":2828,"type":41,"tunes":2830},{"text":2829,"level":252},"Production traces can become evaluation data",{},{"id":885,"data":2832,"type":217,"tunes":2834},{"text":2833},"One of the most useful modern LLMOps patterns is to turn sampled production traces into evaluation records.",{},{"id":890,"data":2836,"type":217,"tunes":2838},{"text":2837},"MLflow currently supports retrieving production traces and scoring not only outputs but intermediate spans such as retrieval or tool-call trajectories.",{},{"id":895,"data":2840,"type":217,"tunes":2842},{"text":2841},"This closes the loop between observability and development: real failures can become regression cases in the next release rather than disappear inside logs.",{},{"id":900,"data":2844,"type":41,"tunes":2846},{"text":2845,"level":252},"Reproducibility becomes conditional rather than exact",{},{"id":905,"data":2848,"type":217,"tunes":2850},{"text":2849},"Classical ML reproducibility often aims to recreate a model from versioned code, data, environment and training parameters.",{},{"id":910,"data":2852,"type":217,"tunes":2854},{"text":2853},"Hosted LLM applications cannot always reproduce identical output token-for-token because generation is probabilistic and providers may control infrastructure.",{},{"id":915,"data":2856,"type":217,"tunes":2858},{"text":2857},"LLMOps therefore aims for behavioral reproducibility: record enough model\u002Fprovider\u002Fversion, prompt, context inputs, retrieval state and runtime configuration to reproduce the conditions and validate behavior within expected tolerances.",{},{"id":920,"data":2860,"type":41,"tunes":2862},{"text":2861,"level":252},"Lineage expands from model lineage to application lineage",{},{"id":925,"data":2864,"type":217,"tunes":2866},{"text":2865},"AWS's MLOps guidance treats model lineage as the history of code, data, model and infrastructure artifacts needed for diagnosis and reproducibility.",{},{"id":930,"data":2868,"type":217,"tunes":2870},{"text":2869},"For LLM applications, lineage should additionally connect prompts, eval datasets, retrieval\u002Findex versions, tool schemas, agent\u002Fruntime configuration and provider\u002Fmodel snapshots.",{},{"id":935,"data":2872,"type":217,"tunes":2874},{"text":2873},"The target question becomes: Which exact application configuration produced this trace?",{},{"id":940,"data":2876,"type":41,"tunes":2878},{"text":2877,"level":252},"Multi-provider and model routing create operational policy",{},{"id":945,"data":2880,"type":217,"tunes":2882},{"text":2881},"Once an application can use several providers or local models, routing becomes an operational policy rather than a simple model string.",{},{"id":950,"data":2884,"type":217,"tunes":2886},{"text":2885},"Routing may depend on capability, latency, cost, privacy, context length, availability, tool support or locality. A fallback can preserve uptime while changing answer quality or data-processing assumptions.",{},{"id":955,"data":2888,"type":217,"tunes":2890},{"text":2889},"LLMOps should therefore log which route was actually selected and evaluate routes independently rather than treat every compatible endpoint as behaviorally interchangeable.",{},{"id":960,"data":2892,"type":41,"tunes":2894},{"text":2893,"level":252},"Original implementation evidence",{},{"id":965,"data":2896,"type":41,"tunes":2898},{"text":2897,"level":251},"Aaasaasa AI Client: provider, model and runtime are separate operational objects",{},{"id":970,"data":2900,"type":217,"tunes":2902},{"text":2901},"Aaasaasa AI Client separates agent\u002Fclient, provider, model, runtime location and permissions. Its AI Hub supports Ollama, LM Studio\u002FOpenAI-compatible endpoints and other provider protocols rather than treating “the model” as one global setting.",{},{"id":975,"data":2904,"type":217,"tunes":2906},{"text":2905},"The implementation includes dynamic local model discovery, streaming, thinking output and explicit Ollama warm\u002Fload and unload controls. That is operational evidence that local LLM serving introduces resource lifecycle concerns beyond an API model name.",{},{"id":980,"data":2908,"type":217,"tunes":2910},{"text":2909},"Provider status is queried through provider adapters, and connection types distinguish local, cloud API, account-backed, remote-agent and web-client paths. These are concrete operational dimensions an LLM-aware platform has to surface.",{},{"id":985,"data":2912,"type":217,"tunes":2914},{"text":2913},"The repository also preserves an important boundary: a local runtime is not automatically local inference. Provider\u002Fmodel\u002Fruntime location are versioned or configurable concerns that affect privacy, latency, cost and availability.",{},{"id":990,"data":2916,"type":41,"tunes":2918},{"text":2917,"level":251},"Source of Truth Research Engine: LLM application state extends beyond the model",{},{"id":995,"data":2920,"type":217,"tunes":2922},{"text":2921},"The Source of Truth Research Engine combines lexical search, optional embeddings, source snapshots, SHA-256 identity, claims, provenance and contradiction tracking around local model-assisted research.",{},{"id":1000,"data":2924,"type":217,"tunes":2926},{"text":2925},"This is useful LLMOps evidence because changing the model alone does not define the research system. Retrieval, source acquisition, evidence classification and persistent provenance are independent operational artifacts.",{},{"id":1005,"data":2928,"type":217,"tunes":2930},{"text":2929},"The implementation deliberately treats semantic similarity as discovery rather than evidence, showing why LLMOps observability should distinguish retrieval behavior from claim validity.",{},{"id":1010,"data":2932,"type":410,"tunes":2961},{"content":2933,"stretched":42,"withHeadings":13},[2934,2937,2940,2943,2946,2949,2952,2955,2958],[2935,2936],"Observed implementation","LLMOps lesson",[2938,2939],"Multiple provider protocols","Provider identity is an operational dependency",[2941,2942],"Dynamic model discovery","Available models can change independently of application code",[2944,2945],"Ollama load\u002Funload controls","Local models have memory\u002Fresource lifecycle",[2947,2948],"Provider health\u002Fstatus adapters","Model availability needs runtime observability",[2950,2951],"Separate runtime and inference location","Deployment topology is not one boolean “local\u002Fcloud”",[2953,2954],"Central permissions","Model capability and tool authority must remain separate",[2956,2957],"Lexical + semantic retrieval pipeline","Retrieval configuration is part of application behavior",[2959,2960],"Source\u002Fprovenance persistence","Operational state and evidence live outside model weights",{},{"id":1042,"data":2963,"type":225,"tunes":2966},{"body":2964,"title":2965,"variant":239},"These projects demonstrate multi-provider\u002Flocal-model operations, permission separation, retrieval infrastructure and evidence persistence. They are not presented as a complete commercial LLMOps platform or proof of large-scale production traffic.","Evidence boundary",{},{"id":1048,"data":2968,"type":41,"tunes":2970},{"text":2969,"level":252},"Common LLMOps failure modes",{},{"id":1053,"data":2972,"type":410,"tunes":3013},{"content":2973,"stretched":42,"withHeadings":13},[2974,2977,2980,2983,2986,2989,2992,2995,2998,3001,3004,3007,3010],[2975,2976],"Failure mode","What actually went wrong",[2978,2979],"Model alias upgraded silently","Behavior changed without controlled release",[2981,2982],"Prompt changed without evals","Behavioral regression passed normal unit tests",[2984,2985],"RAG index stale","Generation model was blamed for retrieval\u002Fdata failure",[2987,2988],"Only final answer is logged","Root cause in retrieval\u002Ftool\u002Fcontext trajectory is invisible",[2990,2991],"Provider fallback is silent","Different model\u002Fdata path changes behavior without attribution",[2993,2994],"Token cost tracked globally","Expensive workflows cannot be localized",[2996,2997],"Judge model changed","Evaluation scores drift without application change",[2999,3000],"Production traces never become tests","Known failures repeatedly return",[3002,3003],"Local model stays loaded indefinitely","VRAM\u002Fresource pressure becomes operational instability",[3005,3006],"Permissions encoded only in prompt","Model behavior is mistaken for authorization",[3008,3009],"One eval score gates everything","Different quality dimensions are collapsed into a misleading number",[3011,3012],"Model registry exists but prompt\u002Findex versions do not","Application lineage remains incomplete",{},{"id":1097,"data":3015,"type":41,"tunes":3017},{"text":3016,"level":252},"Common misconceptions",{},{"id":1102,"data":3019,"type":410,"tunes":3054},{"content":3020,"stretched":42,"withHeadings":13},[3021,3024,3027,3030,3033,3036,3039,3042,3045,3048,3051],[3022,3023],"Misconception","Correction",[3025,3026],"“LLMOps replaces MLOps.”","LLMOps extends MLOps principles to LLM-specific application behavior.",[3028,3029],"“LLMOps is prompt engineering.”","Prompts are one artifact among models, providers, context, retrieval, tools, evals and runtime.",[3031,3032],"“Hosted APIs remove operations work.”","They remove some model-serving\u002Ftraining work but add provider lifecycle, version and dependency management.",[3034,3035],"“If the API is stable, the app is stable.”","Model behavior and provider\u002Fmodel snapshots can change independently of API schema.",[3037,3038],"“RAG is just data preprocessing.”","In production it has its own ingestion, index, retrieval and freshness lifecycle.",[3040,3041],"“LLM outputs cannot be tested.”","They can be evaluated with deterministic, reference, judge and human criteria.",[3043,3044],"“LLM judges are objective ground truth.”","They are model-based evaluators that also require calibration and version control.",[3046,3047],"“A local model eliminates LLMOps.”","Local serving adds model files, VRAM, load\u002Funload, runtime health and upgrade concerns.",[3049,3050],"“Observability means token counts.”","Useful observability follows prompts, retrievals, tools, model spans and outcomes.",[3052,3053],"“Continuous training is mandatory.”","Many LLM apps use continuous evaluation without training the foundation model.",{},{"id":1140,"data":3056,"type":41,"tunes":3058},{"text":3057,"level":252},"A practical LLMOps design sequence",{},{"id":1145,"data":3060,"type":345,"tunes":3099},{"steps":3061,"title":3098,"orientation":344},[3062,3065,3068,3071,3074,3077,3080,3083,3086,3089,3092,3095],{"label":3063,"description":3064},"1. Define the behavior unit","List every component that can materially change output: model, prompt, retrieval, tools, context and policy.",{"label":3066,"description":3067},"2. Establish application lineage","Version code, model\u002Fprovider, prompts, eval datasets, retrieval configuration and tool contracts.",{"label":3069,"description":3070},"3. Build representative eval datasets","Use expected success\u002Ffailure cases from design and production.",{"label":3072,"description":3073},"4. Separate deterministic and behavioral tests","Keep schema\u002Fsecurity assertions distinct from semantic output evaluation.",{"label":3075,"description":3076},"5. Trace end-to-end execution","Instrument model, retrieval, reranking, tools and agent\u002Fruntime spans.",{"label":3078,"description":3079},"6. Define release gates","Set quality, safety, latency and cost thresholds.",{"label":3081,"description":3082},"7. Pin or explicitly record model versions","Treat model\u002Fprovider changes as release events.",{"label":3084,"description":3085},"8. Deploy progressively","Use flags, canaries or staged rollout where consequence warrants it.",{"label":3087,"description":3088},"9. Evaluate production traces","Measure real task behavior and identify recurrent failures.",{"label":3090,"description":3091},"10. Feed failures back into eval datasets","Turn incidents and corrections into permanent regression coverage.",{"label":3093,"description":3094},"11. Monitor provider and data lifecycles","Track deprecations, index freshness, source changes and runtime availability.",{"label":3096,"description":3097},"12. Retire obsolete versions cleanly","Remove old prompts\u002Fmodels\u002Findexes\u002Fcredentials after migration and evidence retention decisions.","Operate the complete behavior-producing system",{},{"id":1187,"data":3101,"type":41,"tunes":3103},{"text":3102,"level":252},"LLMOps architecture checklist",{},{"id":1192,"data":3105,"type":410,"tunes":3152},{"content":3106,"stretched":42,"withHeadings":13},[3107,3110,3113,3116,3119,3122,3125,3128,3131,3134,3137,3140,3143,3146,3149],[3108,3109],"Question","Expected evidence",[3111,3112],"Which model\u002Fprovider\u002Fversion served the request?","Traceable model identity",[3114,3115],"Which prompt\u002Finstructions were active?","Versioned application code\u002Fconfig",[3117,3118],"Which context reached the model?","Context\u002Fretrieval trace",[3120,3121],"Which corpus\u002Findex version was used?","Retrieval lineage",[3123,3124],"Which tools were available and called?","Tool schema + trajectory trace",[3126,3127],"Which permissions applied?","Runtime authorization record",[3129,3130],"How is quality measured?","Versioned eval dataset + scorers",[3132,3133],"How are model upgrades tested?","Behavioral regression suite",[3135,3136],"How is production quality sampled?","Trace evaluation\u002Ffeedback process",[3138,3139],"Can one failure be reproduced approximately?","Model\u002Fcontext\u002Fprovider\u002Fapplication lineage",[3141,3142],"Where is cost spent?","Per-trace model\u002Ftool\u002Fretrieval attribution",[3144,3145],"What triggers rollback?","Defined quality\u002Fsafety\u002Fcost\u002Favailability threshold",[3147,3148],"How are provider deprecations handled?","Migration\u002Ffallback process",[3150,3151],"How are local models operated?","Health, resource, load\u002Funload and version controls",{},{"id":1242,"data":3154,"type":41,"tunes":3156},{"text":3155,"level":252},"Edge cases and limitations",{},{"id":1247,"data":3158,"type":217,"tunes":3160},{"text":3159},"A simple application that calls one fixed hosted model with no retrieval or tools may need only lightweight LLMOps: versioned prompt code, evals, model pinning, basic tracing and provider monitoring.",{},{"id":1252,"data":3162,"type":217,"tunes":3164},{"text":3163},"A self-hosted fine-tuned model may require nearly the full classical MLOps stack plus LLM-specific application evaluation, making the boundary between MLOps and LLMOps intentionally blurry.",{},{"id":1257,"data":3166,"type":217,"tunes":3168},{"text":3167},"An agent platform can have minimal model-training operations but substantial runtime operations because failures occur in tool selection, state and orchestration.",{},{"id":1262,"data":3170,"type":217,"tunes":3172},{"text":3171},"A RAG-heavy system can be operationally dominated by document ingestion and retrieval quality rather than model serving.",{},{"id":1267,"data":3174,"type":217,"tunes":3176},{"text":3175},"Terminology will continue to evolve. The durable architecture question is not which “Ops” label wins, but which artifacts produce behavior and therefore must be versioned, evaluated, observed and governed.",{},{"id":1272,"data":3178,"type":41,"tunes":3180},{"text":3179,"level":252},"What would change this answer?",{},{"id":1277,"data":3182,"type":217,"tunes":3184},{"text":3183},"If foundation-model providers standardize perfectly stable model behavior and long-term version support, provider\u002Fsnapshot management could become less operationally significant.",{},{"id":1282,"data":3186,"type":217,"tunes":3188},{"text":3187},"If applications increasingly own fine-tuning or training, classical MLOps concerns become more central again.",{},{"id":1287,"data":3190,"type":217,"tunes":3192},{"text":3191},"The operational principle would remain: every component that can materially change production behavior belongs in lineage, testing, observability and change control.",{},{"id":1292,"data":3194,"type":41,"tunes":3196},{"text":3195,"level":252},"Related canonical knowledge",{},{"id":1297,"data":3198,"type":217,"tunes":3200},{"text":3199},"LLMOps sits below AI Governance and Enterprise AI Architecture: governance defines which changes require evidence and approval, while LLMOps provides the operational machinery to version, evaluate, deploy and observe those changes.",{},{"id":1302,"data":3202,"type":217,"tunes":3204},{"text":3203},"Context Engineering and RAG are operational subdomains inside many LLM applications because context and retrieval can change behavior independently of the model.",{},{"id":1307,"data":3206,"type":217,"tunes":3208},{"text":3207},"Agentic AI extends LLMOps further into trajectory, permissions and tool-runtime operations.",{},{"id":1312,"data":3210,"type":602,"tunes":3215},{"url":3211,"title":3212,"excerpt":3213,"ctaLabel":3214},"https:\u002F\u002Fstajic.de\u002Fblog\u002Fai-agent-memory-is-not-rag-how-to-separate-memory-retrieval-state-and-context","AI Agent Memory Is Not RAG: How to Separate Memory, Retrieval, State and Context","Operational reliability improves when memory, retrieval, application state and model context remain separate lifecycle objects.","Read the architecture article",{},{"id":1320,"data":3217,"type":602,"tunes":3222},{"url":3218,"title":3219,"excerpt":3220,"ctaLabel":3221},"https:\u002F\u002Fstajic.de\u002Fblog\u002Fthe-answer-validity-boundary-the-missing-layer-between-relevance-and-reliable-ai-answers","The Answer Validity Boundary: The Missing Layer Between Relevance and Reliable AI Answers","LLMOps evaluation should preserve the version, scope and evidence conditions under which an answer remains supported.","Read the Answer Validity Boundary",{},{"id":1328,"data":3224,"type":41,"tunes":3226},{"text":3225,"level":252},"Frequently asked questions",{},{"id":1333,"data":3228,"type":1333,"tunes":3258},{"items":3229,"title":3257},[3230,3233,3236,3239,3242,3245,3248,3251,3254],{"id":1337,"answer":3231,"question":3232},"MLOps operates machine-learning systems across data, training, deployment and monitoring. LLMOps extends those practices to LLM applications where prompts, context, retrieval, providers, tools and evaluations also materially affect behavior.","What is the difference between MLOps and LLMOps?",{"id":1341,"answer":3234,"question":3235},"No. LLMOps reuses MLOps disciplines such as CI\u002FCD, lineage, evaluation, deployment and monitoring and adds LLM-specific operational concerns.","Does LLMOps replace MLOps?",{"id":1345,"answer":3237,"question":3238},"Not necessarily. Many use external foundation models and instead rely on continuous evaluation of prompts, models, retrieval and application behavior. Fine-tuned or self-trained systems can still require training pipelines.","Do LLM applications need continuous training?",{"id":1349,"answer":3240,"question":3241},"Generative outputs are open-ended and model behavior can change across prompts, snapshots and context. Evals provide repeatable evidence that a release still meets defined quality and safety criteria.","Why are evals so important in LLMOps?",{"id":1353,"answer":3243,"question":3244},"At minimum: application code, model\u002Fprovider\u002Fversion, prompts, eval datasets\u002Fscorers, retrieval configuration\u002Findexes, tool schemas, context rules and relevant safety\u002Fpermission configuration.","What should be versioned in LLMOps?",{"id":1357,"answer":3246,"question":3247},"No. The same prompt can behave differently with another model, retrieval set, context order, tool surface or provider.","Is prompt versioning enough?",{"id":1361,"answer":3249,"question":3250},"GenAIOps is another industry term for operating generative-AI applications. Some vendors use it interchangeably or as a broader label than LLMOps.","What is GenAIOps?",{"id":1365,"answer":3252,"question":3253},"Monitor end-to-end traces including model calls, prompts\u002Fcontext, retrieval, tools, latency, token\u002Fcost, quality samples, safety and final task outcomes.","How do you monitor an LLM application?",{"id":1369,"answer":3255,"question":3256},"Yes. Local models add their own operational concerns such as model files, hardware\u002FVRAM, load\u002Funload, runtime health, quantization and upgrade management.","Can local LLMs use LLMOps practices?","MLOps vs LLMOps FAQ",{},{"id":1375,"data":3260,"type":41,"tunes":3262},{"text":3261,"level":252},"Glossary",{},{"id":1380,"data":3264,"type":1380,"tunes":3297},{"title":3265,"entries":3266},"Key MLOps and LLMOps terms",[3267,3269,3271,3273,3275,3278,3281,3284,3286,3288,3291,3294],{"term":414,"anchor":413,"definition":3268},"Engineering practices for building, deploying, monitoring and maintaining machine-learning systems and their data\u002Fmodel lifecycle.",{"term":417,"anchor":416,"definition":3270},"Operational practices for production applications whose behavior materially depends on large language models and surrounding prompts, context, retrieval, tools and runtime.",{"term":1389,"anchor":1390,"definition":3272},"Operational discipline for generative-AI applications; often used as a broader or alternate label for LLMOps.",{"term":2430,"anchor":1393,"definition":3274},"Automated or repeated retraining and serving of ML models as data or implementations change.",{"term":3276,"anchor":1397,"definition":3277},"Continuous evaluation","Repeated evaluation of candidate and production AI behavior against versioned datasets and criteria.",{"term":3279,"anchor":1401,"definition":3280},"Model snapshot","A concrete version of a hosted or packaged model whose behavior can be tested and referenced.",{"term":3282,"anchor":1405,"definition":3283},"Application lineage","Traceable relationship among code, model\u002Fprovider, prompts, data\u002Fretrieval, tools, runtime and release configuration.",{"term":1408,"anchor":1409,"definition":3285},"Structured record of one application execution containing spans such as model calls, retrievals and tool operations.",{"term":2485,"anchor":1412,"definition":3287},"Versioned set of representative inputs, expectations and optionally traces\u002Foutputs used to measure behavior.",{"term":3289,"anchor":1416,"definition":3290},"LLM judge","A language model used as an evaluator for qualitative or semantic criteria; it is itself a versioned evaluation dependency.",{"term":3292,"anchor":1420,"definition":3293},"Behavioral regression","A degradation in application output or trajectory despite interfaces and code continuing to execute successfully.",{"term":3295,"anchor":1424,"definition":3296},"Provider routing","Policy for selecting among available model providers\u002Fendpoints according to capability, cost, latency, privacy or availability.",{},{"id":1428,"data":3299,"type":41,"tunes":3301},{"text":3300,"level":252},"Conclusion",{},{"id":1433,"data":3303,"type":217,"tunes":3305},{"text":3304},"MLOps and LLMOps share the same engineering objective: make AI systems reproducible enough, testable enough and observable enough to operate reliably in production.",{},{"id":1438,"data":3307,"type":217,"tunes":3309},{"text":3308},"The difference is the shape of the system. Classical MLOps often centers on training and serving model artifacts; LLMOps must operate a behavioral stack in which model snapshots, prompts, context, retrieval, tools, permissions and providers can change independently.",{},{"id":1443,"data":3311,"type":217,"tunes":3313},{"text":3312},"The shortest useful rule is: version, evaluate and observe everything that can materially change the LLM application's behavior — not only the model.",{},{"id":1448,"data":3315,"type":41,"tunes":3317},{"text":3316,"level":252},"Primary sources and current documentation",{},{"id":1453,"data":3319,"type":217,"tunes":3321},{"text":3320},"The sources below ground the MLOps baseline and the current operational patterns for LLM and agent applications. Project sections are original implementation evidence and are intentionally narrower than claims about a complete LLMOps platform.",{},{"id":1458,"data":3323,"type":1465,"tunes":3327},{"link":1460,"meta":3324},{"image":3325,"title":1463,"description":3326},{"url":380},"Reference architecture describing CI, CD, continuous training, model registry, metadata, serving and monitoring for ML systems.",{},{"id":1468,"data":3329,"type":1465,"tunes":3333},{"link":1470,"meta":3330},{"image":3331,"title":1473,"description":3332},{"url":380},"Current guidance for tracking code, data, models, environments and infrastructure across ML releases.",{},{"id":1477,"data":3335,"type":1465,"tunes":3339},{"link":1479,"meta":3336},{"image":3337,"title":1482,"description":3338},{"url":380},"Current guidance for production model monitoring, drift, endpoint health and lineage.",{},{"id":1486,"data":3341,"type":1465,"tunes":3345},{"link":1488,"meta":3342},{"image":3343,"title":1491,"description":3344},{"url":380},"Official guidance describing GenAIOps, sometimes called LLMOps, across initialization, experimentation, evaluation\u002Frefinement and deployment.",{},{"id":1495,"data":3347,"type":1465,"tunes":3351},{"link":1497,"meta":3348},{"image":3349,"title":1500,"description":3350},{"url":380},"Current GenAI operations documentation covering tracing, evaluation, prompts and production observability for LLM applications and agents.",{},{"id":1504,"data":3353,"type":1465,"tunes":3357},{"link":1506,"meta":3354},{"image":3355,"title":1509,"description":3356},{"url":380},"Current guidance for evaluating complete LLM\u002Fagent traces, including retrieval and tool-call trajectories.",{},{"id":1513,"data":3359,"type":1465,"tunes":3364},{"link":1515,"meta":3360},{"image":3361,"title":3362,"description":3363},{"url":380},"MLflow — Evaluating prompts","Current prompt\u002Fmodel evaluation workflow using versioned prompts, datasets, scorers and traces.",{},{"id":1522,"data":3366,"type":1465,"tunes":3371},{"link":1524,"meta":3367},{"image":3368,"title":3369,"description":3370},{"url":380},"OpenAI API — Versioning and model snapshots","Current API guidance recommending pinned model versions and evals because prompting behavior can change between snapshots.",{},{"id":1531,"data":3373,"type":1465,"tunes":3377},{"link":1533,"meta":3374},{"image":3375,"title":1536,"description":3376},{"url":380},"Current guidance to treat production prompts as application code, version them through source control and cover changes with tests and evaluation checks.",{},{"id":1540,"data":3379,"type":1465,"tunes":3383},{"link":1542,"meta":3380},{"image":3381,"title":1545,"description":3382},{"url":380},"Current provider lifecycle evidence showing model and platform-surface retirement as an operational dependency.",{},{"id":1549,"data":3385,"type":1465,"tunes":3390},{"link":1551,"meta":3386},{"image":3387,"title":3388,"description":3389},{"url":380},"OpenAI — Moving evaluation workflows to Promptfoo","Current 2026 migration guidance illustrating why evaluation assets should remain portable as provider tooling changes.",{},"2.31.6","MLOps operates machine-learning systems; LLMOps extends those practices to prompts, context, retrieval, providers, tools, evaluations and runtime behavior around large language models.","Post erfolgreich abgerufen",{"items":3395,"source":3476,"manualIds":3477,"manualMatchedIds":3478},[3396,3403,3410,3417,3424,3430,3437,3444,3450,3457,3463,3469],{"id":3397,"slug":3398,"title":3399,"excerpt":3400,"featuredImage":3401,"publishedAt":3402},"475","managed-agent-harness-vs-self-hosted-agent-loop-what-you-gain-what-you-lose","Managed Agent Harness vs. Self-Hosted Agent Loop: Was Sie gewinnen, was Sie verlieren","“Selbst gehosteter Agent” kann sehr unterschiedliche Architekturen bedeuten. Dieser Leitfaden unterscheidet zwischen dem Managed Harness, der selbst gehosteten Ausführungsumgebung und dem vollständig selbst betriebenen Agent-Loop—und zeigt, welche Kontrollgrenze Teams tatsächlich benötigen.","\u002Fuploads\u002F2026\u002F09\u002Fmanaged-agent-harness-vs-self-hosted-agent-loop-what-you-gain-what-you-lose-1790352403475-kj10jh.webp","2026-09-25T12:05:00.000Z",{"id":3404,"slug":3405,"title":3406,"excerpt":3407,"featuredImage":3408,"publishedAt":3409},"488","what-is-context-engineering-what-the-model-receives-before-it-answers","Was ist Context Engineering? Was das Modell erhält, bevor es antwortet","Context Engineering gestaltet, welche Informationen ein KI-Modell vor der Inferenz erhält, einschließlich Prompts, Retrieval, Speicher, Anwendungszustand, Tool-Ergebnissen und Konversationsverlauf.","\u002Fuploads\u002F2026\u002F10\u002Fwhat-is-context-engineering-what-the-model-receives-before-it-answers-1791480653258-018kcv.webp","2026-10-08T13:29:00.000Z",{"id":3411,"slug":3412,"title":3413,"excerpt":3414,"featuredImage":3415,"publishedAt":3416},"364","tipps-fuer-die-verbesserung-der-seo-suchmaschinenoptimierung","Meistern des SEO-Workflows: Essenzielle Optimierungsstrategien für organisches Wachstum","Ein strukturierter SEO-Workflow ist entscheidend für nachhaltiges organisches Wachstum. Lerne die zehn grundlegenden Strategien, von der Keyword-Recherche und technischen Optimierung bis hin zur Content-Qualität und Performance-Analyse.","\u002Fuploads\u002F2026\u002F03\u002Ftipps-fuer-die-verbesserung-der-seo-suchmaschinenoptimierung-1774866098131-hwkzrg.webp","2024-01-26T06:35:00.000Z",{"id":3418,"slug":3419,"title":3420,"excerpt":3421,"featuredImage":3422,"publishedAt":3423},"445","qwen-3-6-in-production-release-runbook-ai-rollback-and-llmops-versioning","Qwen 3.6 in der Produktion: Release-Runbook, KI-Rollback und LLMOps-Versionierung","Qwen 3.6 ist nicht nur ein weiteres Modell-Upgrade. Es ist gleichzeitig ein Release-Ereignis, ein Rollback-Szenario und ein Versionierungsproblem. Dieser Artikel erklärt, wie Qwen 3.6 in der Produktion durch LLMOps-Disziplin, Prompt- und Modell-Rückverfolgbarkeit, kontrollierten Rollout und evidenzbasierte Rollback-Bereitschaft gehandhabt werden sollte.","\u002Fuploads\u002F2026\u002F02\u002Fnew-qwen-3-5-plus-1771515512741-dcbi9p.webp","2026-05-04T02:49:00.000Z",{"id":3425,"slug":1586,"title":3426,"excerpt":3427,"featuredImage":3428,"publishedAt":3429},"434","Umfassender Leitfaden zum Evaluation Harness: LLM-Leistungsbewertung meistern","Dieser Leitfaden bietet eine detaillierte Einführung in Evaluation Harness, ein unverzichtbares Framework zur strengen Bewertung der Fähigkeiten von Large Language Models (LLMs) in Enterprise-LLMOps-Pipelines. Erfahren Sie mehr über Einrichtung, Best Practices und fortgeschrittene Techniken, um ein zuverlässiges Modell-Benchmarking und eine Optimierung zu gewährleisten.","\u002Fuploads\u002F2026\u002F04\u002Fevaluation-harness-1775466944495-4s0xv2.webp","2026-03-01T17:50:00.000Z",{"id":3431,"slug":3432,"title":3433,"excerpt":3434,"featuredImage":3435,"publishedAt":3436},"477","computer-use-agents-why-a-successful-demo-can-still-be-an-unreliable-system","Computer-Use-Agenten: Warum eine erfolgreiche Demo dennoch ein unzuverlässiges System sein kann","Computer-Use-Agenten können mittlerweile beeindruckende Browser- und Desktop-Workflows abschließen, aber ein erfolgreicher Durchlauf beweist Fähigkeit—nicht Zuverlässigkeit. Dieser Artikel zeigt, wie man Wiederholbarkeit, Umgebungsrobustheit, Steuerung über lange Zeithorizonte, Zustandsbewusstsein, Ergebnisüberprüfung und sichere Zielhandhabung testet.","\u002Fuploads\u002F2026\u002F09\u002Fcomputer-use-agents-why-a-successful-demo-can-still-be-an-unreliable-system-1790352854690-75qnrg.webp","2026-09-25T12:13:00.000Z",{"id":3438,"slug":3439,"title":3440,"excerpt":3441,"featuredImage":3442,"publishedAt":3443},"480","when-should-an-ai-stop-trusting-its-own-knowledge-the-retrieval-trigger","Wann sollte eine KI aufhören, ihrem eigenen Wissen zu vertrauen? — Der Retrieval-Trigger","Ein KI-Modell benötigt nicht für jede Frage einen Retrieval. Das wichtige Problem ist zu erkennen, wann sein internes Wissen nicht mehr ausreicht. Der Retrieval-Trigger ist eine praktische Entscheidungsgrenze, die bestimmt, wann ein KI-System aufhören sollte, sich allein auf das Modellwissen zu verlassen, und vor der Beantwortung externe Evidenz einholen sollte.","\u002Fuploads\u002F2026\u002F09\u002Fwhen-should-an-ai-stop-trusting-its-own-knowledge-the-retrieval-trigger-1790574991244-f4rpyg.webp","2026-09-28T01:49:00.000Z",{"id":3445,"slug":3446,"title":3447,"excerpt":3448,"featuredImage":3422,"publishedAt":3449},"384","new-qwen-3-5-plus","Neues Qwen 3.5-Plus: Open-Source-KI macht jetzt Ernst","Entdecken Sie die bahnbrechenden Funktionen und Vorteile von Alibabas Qwen 3.5-Plus, einer revolutionären Open-Source-KI für Entwickler.","2026-02-19T10:23:00.000Z",{"id":3451,"slug":3452,"title":3453,"excerpt":3454,"featuredImage":3455,"publishedAt":3456},"478","what-is-rag-the-simplest-explanation-of-how-it-works","Was ist RAG? Die einfachste Erklärung, wie es funktioniert","RAG klingt kompliziert, aber die Idee ist einfach: Bevor eine KI antwortet, sucht sie zunächst nützliche Informationen aus einer Wissensquelle und gibt diese Informationen an das Sprachmodell weiter. Dieser Leitfaden erklärt RAG, LLMs, Zustand, Gedächtnis und Werkzeuge anhand eines einfachen mentalen Modells.","\u002Fuploads\u002F2026\u002F09\u002Fwhat-is-rag-the-simplest-explanation-of-how-it-works-1790377492124-khjagt.webp","2026-09-25T19:03:00.000Z",{"id":3458,"slug":3459,"title":599,"excerpt":3460,"featuredImage":3461,"publishedAt":3462},"469","rag-failed-but-which-layer-actually-failed-a-diagnostic-method","Wenn eine RAG-Antwort falsch ist, ist es zu vage, das Retrieval oder das Modell verantwortlich zu machen. Diese Diagnosemethode isoliert Quellenabdeckung, Query-Konstruktion, Retrieval, Ranking, Kontextzusammenstellung, Generierung, Evidenzzuordnung und Aktualität – sodass der tatsächliche Fehler reproduziert und behoben werden kann.","\u002Fuploads\u002F2026\u002F09\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method-1790350847177-pior4c.webp","2026-09-24T19:39:00.000Z",{"id":3464,"slug":3465,"title":3466,"excerpt":9,"featuredImage":3467,"publishedAt":3468},"369","git-with-automatic-upload-and-synchronization-to-a-production-server","Git with automatic upload and synchronization to a production server","\u002Fuploads\u002F2024\u002F05\u002Fstep-by-step-guide-illustration-showing-the-process-of-setting-up-Git-with-auto-upload-and-synchronization-to-a-production-server-large.webp","2024-05-28T22:48:00.000Z",{"id":3470,"slug":3471,"title":3472,"excerpt":3473,"featuredImage":3474,"publishedAt":3475},"479","where-does-an-llm-get-its-data-rag-data-sources-in-python","Woher bezieht ein LLM seine Daten? RAG-Datenquellen in Python","Ein LLM kennt deine Dateien, Datenbanken oder APIs nicht auf magische Weise. Diese praktische Fortsetzung der RAG-Reihe zeigt mit einfachem Python, wie externe Daten zu abrufbaren Belegen werden: von Textdateien und SQL bis hin zu Volltextsuche, Embeddings, Kontextzusammenstellung und dem abschließenden LLM-Aufruf.","\u002Fuploads\u002F2026\u002F09\u002Fwhere-does-an-llm-get-its-data-rag-data-sources-in-python-1790517200521-nfsi5i.webp","2026-09-27T05:51:00.000Z","fallback",[],[]]