[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"portal-settings:stajic:sr":3,"public-menus:all":38,"post:mlops-vs-llmops-what-changes-when-the-model-is-an-llm:sr":205,"related:post:mlops-vs-llmops-what-changes-when-the-model-is-an-llm:sr:1":3405},{"statusCode":4,"data":5,"message":37},200,{"tenantId":6,"lang":7,"defaultLang":8,"siteUrl":9,"contactEmail":10,"brandName":11,"logoUrl":12,"siteName":11,"siteDescription":13,"ogImage":10,"robotsIndex":14,"socialLinks":10,"reservedSlugs":10,"seoPolicy":15},"stajic","sr","de","https:\u002F\u002Fstajic.de",null,"Stajic Platform","\u002FLogo_Planet.svg","Stajic Portal",true,{"branding":16,"relatedContent":17,"crossDomainLinks":18},{"logoUrl":12},{"enabled":14},[19,22,25,28,31,34],{"url":20,"label":21,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Ffigure.rocks","figure.rocks",{"url":23,"label":24,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Floving.rocks","loving.rocks",{"url":26,"label":27,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Fbazify.com","bazify.com",{"url":29,"label":30,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Fbazify.de","bazify.de",{"url":32,"label":33,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Fbazify.at","bazify.at",{"url":35,"label":36,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Fbazify.ba","bazify.ba","Portal settings resolved",[39,45],{"id":40,"name":41,"location":42,"isActive":14,"isDefault":43,"items":44},1,"main-navigation","header",false,[],{"id":46,"name":47,"location":48,"isActive":14,"isDefault":14,"items":49},4,"main-menu","sidebar",[50,66,79,93,103,118,133],{"id":51,"title":52,"url":60,"target":61,"icon":62,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":64,"portfolioId":10,"children":65},"item-18",{"de":53,"en":54,"es":55,"fr":56,"it":54,"ru":57,"sr":58,"zh":59},"Startseite","Home","Inicio","Accueil","Главная","Почетна","首页","\u002Ffull-stack-web-developer-munich-performance-seo-and-maintainable-builds","_self","i-lucide-home","page",111,[],{"id":67,"title":68,"url":75,"target":61,"icon":76,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":77,"portfolioId":10,"children":78},"item-22",{"de":69,"en":69,"es":70,"fr":69,"it":71,"ru":72,"sr":73,"zh":74},"Vision","Visión","Visione","Видение","Визија","想象","\u002Fueber-uns-webdesign-muenchen-webaplikation","i-lucide-eye",113,[],{"id":80,"title":81,"url":89,"target":61,"icon":90,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":91,"portfolioId":10,"children":92},"item-19",{"de":82,"en":83,"es":84,"fr":83,"it":85,"ru":86,"sr":87,"zh":88},"Leistungen","Services","Servicios","Servizi","Услуги","Услуге","服务","\u002Fservices-dienstleistungen-muenchen","i-lucide-wrench",116,[],{"id":94,"title":95,"url":99,"target":61,"icon":100,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":101,"portfolioId":10,"children":102},"item-23",{"de":96,"en":96,"es":96,"fr":96,"it":96,"ru":97,"sr":97,"zh":98},"Blog","Блог","博客","\u002Fblog","i-lucide-book-open",112,[],{"id":104,"title":105,"url":114,"target":61,"icon":115,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":116,"portfolioId":10,"children":117},"item-32",{"de":106,"en":107,"es":108,"fr":109,"it":110,"ru":111,"sr":112,"zh":113},"Neue Technologien","New Technologies","Nuevas tecnologías","Nouvelles technologies","Nuove tecnologie","Новые технологии","Нове технологије","新技术！","\u002Fneue-webtechnologien","i-lucide-sparkles",122,[],{"id":119,"title":120,"url":129,"target":61,"icon":130,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":131,"portfolioId":10,"children":132},"item-20",{"de":121,"en":122,"es":123,"fr":124,"it":125,"ru":126,"sr":127,"zh":128},"Kontakt","Contact us!","Contacto","Contact","Contatto","Контакт","Контактирајте нас","联系我们！","\u002Fcontact","i-lucide-mail",115,[],{"id":134,"title":135,"url":144,"target":61,"icon":145,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":146,"portfolioId":10,"children":147},"item-21",{"de":136,"en":137,"es":138,"fr":139,"it":140,"ru":141,"sr":142,"zh":143},"Unsere Arbeit","Our Work","Nuestro trabajo","Nos réalisations","I nostri lavori","Наши работы","Наши радови","文件夹","\u002Fportfolio","i-lucide-briefcase",114,[148,161,175,181,193],{"id":149,"title":150,"url":144,"target":61,"icon":159,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":146,"portfolioId":10,"children":160},"item-24",{"de":151,"en":152,"es":153,"fr":154,"it":155,"ru":156,"sr":157,"zh":158},"Alle Projekte","All Projects","Todos los proyectos","Tous les projets","Tutti i progetti","Все проекты","Сви пројекти","所有项目","i-lucide-grid-3x3",[],{"id":162,"title":163,"url":171,"target":61,"icon":172,"isActive":14,"type":173,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":10,"portfolioId":10,"children":174},"item-29",{"de":164,"en":165,"es":166,"fr":167,"it":168,"ru":169,"sr":170,"zh":143},"Local Roots, Global Reach","Local Roots - Global Reach","Empresa local ","Entreprise locale","Azienda locale","Местная компания","Локално предузеће глобално тржиште","\u002Fportfolio\u002Flocal-roots-global-reach-communication-media-systems-for-modern-business","i-lucide-folder","custom",[],{"id":176,"title":177,"url":179,"target":61,"icon":172,"isActive":14,"type":173,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":10,"portfolioId":10,"children":180},"item-28",{"de":178,"en":178,"es":178,"fr":178,"it":178,"ru":178,"sr":178,"zh":178},"Solr Suggester","\u002Fportfolio\u002Fsolr-fuzzy-suggester-und-solr-infix-suggester-abfrage-ueber-ajax-und-filterung",[],{"id":182,"title":183,"url":191,"target":61,"icon":172,"isActive":14,"type":173,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":10,"portfolioId":10,"children":192},"item-27",{"de":184,"en":185,"es":186,"fr":187,"it":188,"ru":189,"sr":190,"zh":185},"Firmenwebseite SEO","Company Website SEO","Sitio web corporativo SEO","Site web d’entreprise SEO","Sito web aziendale SEO","Корпоративный сайт SEO","Пословна веб-страница SEO","\u002Fportfolio\u002Fseo-sem-branding-mobile-webseite-muenchen",[],{"id":194,"title":195,"url":203,"target":61,"icon":172,"isActive":14,"type":173,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":10,"portfolioId":10,"children":204},"item-31",{"de":196,"en":197,"es":198,"fr":199,"it":200,"ru":201,"sr":202,"zh":197},"Digitalisierungsportal","Digitalization Portal","Portal de digitalización","Portail de numérisation","Portale di digitalizzazione","Портал цифровизации","Портал за дигитализацију","\u002Fportfolio\u002Fdigitalisierungsportal-archiv-museum-bibliothek-ead-lido-mets-mods",[],{"statusCode":4,"data":206,"message":3404},{"id":207,"title":208,"slug":209,"content":210,"contentJson":211,"excerpt":1560,"featuredImage":1561,"featuredImageAlt":1562,"featuredImageCaption":10,"featuredImageTitle":10,"featuredImageCopyright":10,"featuredImageAuthor":10,"featuredImageSourceUrl":10,"featuredImageLicense":10,"featuredImageIsAiGenerated":43,"status":1563,"publishedAt":1564,"createdAt":1565,"updatedAt":1566,"seoLocalePaths":1567,"categories":1576,"author":1593,"translations":1598},"493","MLOps vs LLMOps: Šta se menja kada je model LLM","mlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u003Cp>MLOps je inženjerska disciplina za pouzdan razvoj, implementaciju, verzionisanje i upravljanje sistemima mašinskog učenja; LLMOps proširuje tu disciplinu na aplikacije izgrađene oko velikih jezičkih modela, gde ponašanje u produkciji zavisi ne samo od artefakta modela već i od upita, konteksta, pretraživanja, verzija provajdera\u002Fmodela, poziva alata, sigurnosnih kontrola i pipeline-ova za evaluaciju. LLMOps ne zamenjuje MLOps. On menja operativnu jedinicu sa „model plus pipeline za serviranje“ na „LLM aplikaciju koja se razvija i čije ponašanje proizlazi iz nekoliko nezavisno promenljivih komponenti“.\u003C\u002Fp>\n\u003Caside class=\"editorjs-callout editorjs-callout--info my-6 rounded-xl border p-5 border-blue-300 bg-blue-50 dark:border-blue-900 dark:bg-blue-950\u002F20\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Direktan odgovor\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">\u003Cstrong>MLOps upravlja ML sistemima. LLMOps upravlja LLM aplikacijama.\u003C\u002Fstrong>\u003Cbr>\u003Cbr>Klasičan MLOps se obično fokusira na data pipeline-ove, treniranje, validaciju, registar modela, implementaciju, drift i ponovno treniranje. LLMOps zadržava te discipline tamo gde su relevantne, ali često dodaje verzionisanje upita\u002Fkonteksta, apstrakciju modela\u002Fprovajdera, RAG indekse, tragove agenata\u002Falata, semantičke evaluacije, sigurnosne testove, praćenje tokena\u002Ftroškova i regresiono testiranje kroz brzo promenljive snimke modela.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Caside class=\"editorjs-callout editorjs-callout--warning my-6 rounded-xl border p-5 border-amber-300 bg-amber-50 dark:border-amber-900 dark:bg-amber-950\u002F20\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">LLMOps nije samo upravljanje upitima\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">Produkcijska LLM aplikacija može da zakaže čak i kada upit ostane nepromenjen: provajder može da promeni snimak modela, RAG korpus može da postane zastareo, reranker može da regresira, dozvole alata se mogu promeniti, sastavljanje konteksta može da izgubi dokaze ili agent može da zauzme pogrešnu putanju. LLMOps zato mora da posmatra i verzionira sistem oko modela, a ne samo tekst upita.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Caside class=\"editorjs-callout editorjs-callout--note my-6 rounded-xl border p-5 border-gray-300 bg-gray-50 dark:border-gray-700 dark:bg-gray-900\u002F40\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Terminološka granica\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">\u003Cstrong>LLMOps\u003C\u002Fstrong>, \u003Cstrong>GenAIOps\u003C\u002Fstrong> i srodni termini su široko korišćene inženjerske oznake, ali nisu jedinstveni univerzalni formalni standard sa jednim kanonskim životnim ciklusom. Microsoft trenutno opisuje GenAIOps kao „ponekad nazvan LLMOps“, dok MLflow grupiše operativne alate oko agenata i LLM aplikacija. Ovaj članak koristi LLMOps kao praktičan arhitektonski termin za upravljanje produkcijskim sistemima čije ponašanje materijalno zavisi od LLM-ova.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Caside class=\"editorjs-callout editorjs-callout--note my-6 rounded-xl border p-5 border-gray-300 bg-gray-50 dark:border-gray-700 dark:bg-gray-900\u002F40\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Napomena o aktuelnim izvorima — 8. oktobar 2026.\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">Operativna površina se brzo menja. OpenAI trenutno preporučuje fiksiranje snimaka modela i pokretanje evaluacija jer se ponašanje upita može promeniti između snimaka, a nekoliko starijih platformski specifičnih površina za upite\u002Fevaluacije se ukida u 2026. Stabilna arhitektonska lekcija je da upiti, testovi i evaluacije ostanu prenosivi i verzionisani uz aplikaciju, umesto da zavise od objektnog modela kontrolne table jednog provajdera.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Cnav class=\"editorjs-toc\" data-editorjs-toc=\"true\" aria-label=\"Sadržaj\">\u003Cstrong class=\"editorjs-toc__title\">Sadržaj\u003C\u002Fstrong>\u003Col class=\"editorjs-toc__list editorjs-toc__list--depth-0\">\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-7\" class=\"editorjs-toc__link\">Šta MLOps zaista znači\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-11\" class=\"editorjs-toc__link\">Šta se menja kada je model LLM\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-15\" class=\"editorjs-toc__link\">Najjednostavniji primer\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-20\" class=\"editorjs-toc__link\">Gde se jednostavan primer zaustavlja\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-24\" class=\"editorjs-toc__link\">MLOps vs LLMOps\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-26\" class=\"editorjs-toc__link\">LLMOps proširuje MLOps, a ne zamenjuje ga\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-30\" class=\"editorjs-toc__link\">Šta se mora verzionisati u LLMOps-u?\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-32\" class=\"editorjs-toc__link\">Snimci modela postaju zavisnosti izdanja\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-36\" class=\"editorjs-toc__link\">Životni ciklus provajdera postaje deo operacija\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-40\" class=\"editorjs-toc__link\">Promptovi se ponašaju kao produkcioni kod\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-44\" class=\"editorjs-toc__link\">Inženjering konteksta postaje operativna briga\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-48\" class=\"editorjs-toc__link\">RAG stvara sopstveni operativni životni ciklus\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-53\" class=\"editorjs-toc__link\">Evaluacije zamenjuju „izgleda dobro“ dokazima o izdanju\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-58\" class=\"editorjs-toc__link\">LLM kao sudija je koristan, ali nije osnovna istina\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-62\" class=\"editorjs-toc__link\">Praćenje postaje važnije od logova krajnjih tačaka\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-66\" class=\"editorjs-toc__link\">Agenti proširuju LLMOps na operacije u vreme izvršavanja\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-71\" class=\"editorjs-toc__link\">Tokeni, pozivi modela i kontekst postaju varijable troškova\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-75\" class=\"editorjs-toc__link\">Keširanje postaje semantičko, ne samo tehničko\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-79\" class=\"editorjs-toc__link\">Bezbednost i dozvole postaju kriterijumi za izdanje\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-83\" class=\"editorjs-toc__link\">Kako CI izgleda u LLMOps-u\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-85\" class=\"editorjs-toc__link\">Kako CD izgleda u LLMOps-u\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-89\" class=\"editorjs-toc__link\">Kontinuirano treniranje postaje opciono; kontinuirana evaluacija postaje centralna\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-93\" class=\"editorjs-toc__link\">Šta treba pratiti u produkciji?\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-95\" class=\"editorjs-toc__link\">Produkcijski tragovi mogu postati podaci za evaluaciju\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-99\" class=\"editorjs-toc__link\">Reproduktivnost postaje uslovna, a ne egzaktna\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-103\" class=\"editorjs-toc__link\">Loza se širi od loze modela do loze aplikacije\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-107\" class=\"editorjs-toc__link\">Usmeravanje ka više provajdera i modela stvara operativnu politiku\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-111\" class=\"editorjs-toc__link\">Dokazi iz originalne implementacije\u003C\u002Fa>\u003Col class=\"editorjs-toc__list editorjs-toc__list--depth-1\">\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-112\" class=\"editorjs-toc__link\">Aaasaasa AI Client: provajder, model i runtime su odvojeni operativni objekti\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-117\" class=\"editorjs-toc__link\">Source of Truth Research Engine: stanje LLM aplikacije se proteže izvan modela\u003C\u002Fa>\u003C\u002Fli>\u003C\u002Fol>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-123\" class=\"editorjs-toc__link\">Uobičajeni režimi neuspeha u LLMOps-u\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-125\" class=\"editorjs-toc__link\">Uobičajene zablude\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-127\" class=\"editorjs-toc__link\">Praktičan redosled dizajna LLMOps-a\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-129\" class=\"editorjs-toc__link\">Kontrolna lista arhitekture LLMOps-a\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-131\" class=\"editorjs-toc__link\">Rubni slučajevi i ograničenja\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-137\" class=\"editorjs-toc__link\">Šta bi promenilo ovaj odgovor?\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-141\" class=\"editorjs-toc__link\">Povezano kanonsko znanje\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-147\" class=\"editorjs-toc__link\">Često postavljana pitanja\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-149\" class=\"editorjs-toc__link\">Pojmovnik\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-151\" class=\"editorjs-toc__link\">Zaključak\u003C\u002Fa>\u003C\u002Fli>\u003Cli class=\"editorjs-toc__item\">\u003Ca href=\"#section-155\" class=\"editorjs-toc__link\">Primarni izvori i trenutna dokumentacija\u003C\u002Fa>\u003C\u002Fli>\u003C\u002Fol>\u003C\u002Fnav>\n\u003Ch2 id=\"section-7\">Šta MLOps zaista znači\u003C\u002Fh2>\n\u003Cp>MLOps primenjuje softversko-inženjersku i operativnu disciplinu na sisteme mašinskog učenja. Produkcijski izazov je širi od treniranja modela: prikupljanje podataka, validacija podataka, eksperimentisanje, reproduktivnost, evaluacija modela, implementacija, infrastruktura i monitoring moraju da rade zajedno.\u003C\u002Fp>\n\u003Cp>Google-ove smernice za MLOps arhitekturu definišu disciplinu oko kontinuirane integracije, kontinuirane isporuke i kontinuiranog treniranja. CI validira ne samo kod već i podatke, šeme i modele; CD implementira ML pipeline-ove i servise za predikciju; CT može ponovo da trenira i implementira modele kada se podaci ili implementacije promene.\u003C\u002Fp>\n\u003Cp>AWS smernice dodaju iste operativne brige iz drugog ugla: poreklo modela, sledljivost modela\u002Fverzija, praćenje drifta i praćenje produkcijskog kvaliteta su ključni delovi održavanja pouzdanosti ML sistema nakon implementacije.\u003C\u002Fp>\n\u003Ch2 id=\"section-11\">Šta se menja kada je model LLM\u003C\u002Fh2>\n\u003Cp>Veliki jezički modeli menjaju produkcijski problem jer aplikacija često ne poseduje kompletan životni ciklus treniranja modela. Tim može da poziva hostovanu API uslugu modela, pokreće otvoreni model lokalno, menja provajdere ili koristi nekoliko modela za različite zadatke.\u003C\u002Fp>\n\u003Cp>Model je stoga samo jedna verzionisana zavisnost unutar većeg bihevioralnog sistema. Upiti, rezultati pretraživanja, redosled konteksta, alati, snimak modela, podešavanja temperature\u002Fzaključivanja, sigurnosni filteri i runtime orkestracija mogu svi da promene izlaz.\u003C\u002Fp>\n\u003Cp>To stvara šire operativno pitanje: koja kombinacija modela, konteksta, podataka, upita, alata i runtime-a je proizvela ovo ponašanje? LLMOps postoji da učini to pitanje odgovorivim, a odgovor dovoljno reproduktivnim za inženjerski rad.\u003C\u002Fp>\n\u003Ch2 id=\"section-15\">Najjednostavniji primer\u003C\u002Fh2>\n\u003Cp>Pretpostavimo da aplikacija odgovara na pitanja o internim politikama.\u003C\u002Fp>\n\u003Cp>U klasičnom ML okviru, mogli biste da verzionirate trenirani klasifikator, implementirate ga i pratite kvalitet predikcije. U LLM aplikaciji, odgovor može da zavisi od snimka hostovanog modela, sistemskog upita, modela za ugrađivanje, vektorskog indeksa, filtera pretraživanja, rerankera i konačno izabranog konteksta.\u003C\u002Fp>\n\u003Cp>Promena bilo koje od tih komponenti može da promeni konačni odgovor iako krajnja tačka aplikacije i korisničko pitanje ostaju identični.\u003C\u002Fp>\n\u003Csection class=\"editorjs-process my-6\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">Tipičan put izdavanja u LLMOps-u\u003C\u002Fh3>\u003Cdiv class=\"grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4\">\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">1\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">1. Promenite jednu komponentu\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Prompt, model, provajder, podešavanje pretrage, šema alata ili izmene koda aplikacije.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">2\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">2. Pokrenite determinističke testove\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Validirajte šeme, dozvole, ugovore alata, filtere pretrage i ponašanje aplikacije.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">3\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">3. Pokrenite bihevioralne evaluacije\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Uporedite reprezentativne izlaze, kvalitet pretrage i trajektorije agenata\u002Falata u odnosu na kriterijume prihvatanja.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">4\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">4. Uporedite cenu i latenciju\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Izmerite korišćenje tokena, pozive modela, troškove pretrage\u002Falata i latenciju odgovora.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">5\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">5. Implementirajte kontrolisanu verziju\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Isporučite konkretnu konfiguraciju aplikacije sa zabeleženim verzijama modela\u002Fprovajdera.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">6\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">6. Pratite ponašanje u produkciji\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Snimite relevantne opsege modela, pretrage, alata i izvršnog okruženja.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">7\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">7. Evaluirajte produkcijske tragove\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Uzmite uzorke stvarnih izvršavanja radi kvaliteta, utemeljenosti, bezbednosti i uspeha zadatka.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">8\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">8. Vratite ili iterirajte\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Koristite dokaze o regresiji i operativne signale da odlučite o sledećem izdanju.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fsection>\n\u003Ch2 id=\"section-20\">Gde se jednostavan primer zaustavlja\u003C\u002Fh2>\n\u003Cp>Neki LLM sistemi i dalje treniraju ili fino podešavaju sopstvene modele, tako da tradicionalne MLOps prakse kao što su pipeline-ovi za treniranje, registar modela i poreklo podataka ostaju direktno relevantne.\u003C\u002Fp>\n\u003Cp>Drugi sistemi koriste samo eksterne API-je foundation modela i nikada ne pokreću kontinuirano treniranje. Njihov glavni operativni teret je evaluacija aplikacije, upravljanje promenama modela\u002Fprovajdera, verzionisanje prompta\u002Fkonteksta, kvalitet pretrage i observabilnost.\u003C\u002Fp>\n\u003Cp>Zato ne postoji jedinstveni univerzalni „LLMOps pipeline“. Tačan životni ciklus zavisi od toga da li trenirate, fino podešavate, samostalno hostujete, preuzimate eksterno znanje, pokrećete agente ili zavisite od upravljanih API-ja modela.\u003C\u002Fp>\n\u003Ch2 id=\"section-24\">MLOps vs LLMOps\u003C\u002Fh2>\n\u003Csection class=\"editorjs-comparison my-6\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">Šta ostaje isto, a šta se širi\u003C\u002Fh3>\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 bg-gray-50 px-4 py-3 text-left dark:border-gray-700 dark:bg-gray-900\">\u003C\u002Fth>\u003Cth class=\"border border-gray-300 bg-gray-50 px-4 py-3 text-left font-semibold dark:border-gray-700 dark:bg-gray-900\">MLOps\u003C\u002Fth>\u003Cth class=\"border border-gray-300 bg-gray-50 px-4 py-3 text-left font-semibold dark:border-gray-700 dark:bg-gray-900\">LLMOps\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Primarna operativna jedinica\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Vlasništvo nad modelom\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Tipična promena\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Evaluacija\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Praćenje u produkciji\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Kontinuirano treniranje\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Verzionisani artefakti\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-3 text-left font-semibold dark:border-gray-700\">Meta vraćanja\u003C\u002Fth>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-3 dark:border-gray-700\">\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\u003C\u002Fsection>\n\u003Ch2 id=\"section-26\">LLMOps proširuje MLOps, a ne zamenjuje ga\u003C\u002Fh2>\n\u003Cp>Osnovni operativni principi ne nestaju: kontrola izvornog koda, CI\u002FCD, reproduktivnost, poreklo, kontrole implementacije, praćenje, vraćanje i merljivi kriterijumi prihvatanja ostaju suštinski.\u003C\u002Fp>\n\u003Cp>Proširenje je u tome što više artefakata koji definišu ponašanje sada stoji izvan težina modela. Upravljani foundation model može promeniti ponašanje kroz nadogradnje snimaka, dok se izlaz aplikacije može promeniti kroz izmene prompta ili pretrage bez ikakvog ponovnog treniranja modela.\u003C\u002Fp>\n\u003Cp>Zato je korisna hijerarhija obično DevOps → MLOps → LLMOps\u002FGenAIOps kao sve specijalizovanije operativne brige, a ne tri međusobno isključive prakse.\u003C\u002Fp>\n\u003Ch2 id=\"section-30\">Šta se mora verzionisati u LLMOps-u?\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Artefakt\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Zašto je važan\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kod aplikacije\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Definiše orkestraciju, validaciju, ponovne pokušaje i poslovno ponašanje\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Porodica modela + snimak\u002Fverzija\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Različiti snimci mogu proizvesti različito ponašanje\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Provajder \u002F endpoint\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menja tok podataka, latenciju, ograničenja, cene i dostupnost\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kod prompta\u002Finstrukcija\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menja ponašanje modela čak i sa istim modelom\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Parametri generisanja\u002Fzaključivanja\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Mogu promeniti determinizam, latenciju, dubinu i cenu\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Skup podataka za evaluaciju\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Definiše prema čemu se testira „dovoljno dobro“\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ocenjivači \u002F graderi\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Definišu kako se meri kvalitet\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Model za ugrađivanje\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menja vektorsku reprezentaciju i ponašanje pretrage\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Konfiguracija deljenja\u002Findeksa\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menja šta se može pronaći\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Reranker \u002F fuzija pretrage\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menja redosled rezultata\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Šeme alata\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menjaju šta model može zahtevati i kako\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Profil dozvola\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menja koje se akcije alata mogu stvarno izvršiti\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Pravila sastavljanja konteksta\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menjaju koji dokazi i stanje stižu do modela\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Konfiguracija bezbednosti\u002Fzaštitnih ograda\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Menja dozvoljeno ili blokirano ponašanje\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-32\">Snimci modela postaju zavisnosti izdanja\u003C\u002Fh2>\n\u003Cp>Sa hostovanim LLM-ovima, tim možda ne kontroliše treniranje modela, ali i dalje kontroliše koji model ili snimak aplikacija poziva.\u003C\u002Fp>\n\u003Cp>OpenAI-jeva trenutna API smernica eksplicitno upozorava da se ponašanje prompta može promeniti između snimaka modela i preporučuje fiksiranje produkcijskih aplikacija na određene snimke gde je konzistentnost važna, a zatim pokretanje evaluacija prilikom nadogradnje.\u003C\u002Fp>\n\u003Cp>Operativna posledica je jasna: nadogradnje modela treba tretirati kao izdanja aplikacije, a ne kao nevidljivo održavanje infrastrukture.\u003C\u002Fp>\n\u003Ch2 id=\"section-36\">Životni ciklus provajdera postaje deo operacija\u003C\u002Fh2>\n\u003Cp>LLM aplikacije često zavise od ograničenja brzine provajdera, rasporeda ukidanja, API semantike, ograničenja konteksta, pravila rukovanja podacima i cena.\u003C\u002Fp>\n\u003Cp>Provajder može ukinuti model dok vaš aplikacijski kod ostaje nepromenjen. OpenAI-jev trenutni raspored ukidanja, na primer, uključuje datume povlačenja 2026. za starije snimke modela i platforme.\u003C\u002Fp>\n\u003Cp>LLMOps stoga zahteva praćenje životnog ciklusa provajdera, testiranje migracije i odluke o rezervnim rešenjima pored praćenja kvaliteta modela.\u003C\u002Fp>\n\u003Ch2 id=\"section-40\">Promptovi se ponašaju kao produkcioni kod\u003C\u002Fh2>\n\u003Cp>Promptovi su izvršna bihevioralna konfiguracija. Male izmene mogu promeniti kvalitet izlaza, izbor alata i tumačenje politike.\u003C\u002Fp>\n\u003Cp>OpenAI-jeva trenutna uputstva preporučuju čuvanje produkcionih promptova u aplikacijskom kodu, pregled izmena promptova kroz pull zahteve, korišćenje tipizovanih ulaza i pokrivanje izmena testovima i proverama evaluacije.\u003C\u002Fp>\n\u003Cp>To čini verzionisanje promptova manje sličnim uređivanju marketinškog teksta, a više sličnim menjanju funkcije čiji je izlaz probabilistički i zavisi od modela.\u003C\u002Fp>\n\u003Ch2 id=\"section-44\">Inženjering konteksta postaje operativna briga\u003C\u002Fh2>\n\u003Cp>Produkcioni model retko prima samo statički prompt. Može primiti istoriju razgovora, preuzete dokumente, izlaze alata, memoriju, trenutno stanje aplikacije i instrukcije politike.\u003C\u002Fp>\n\u003Cp>LLMOps stoga mora posmatrati sastavljanje konteksta: koji dokazi su izabrani, koja verzija stanja je bila aktuelna, da li je došlo do skraćivanja i da li su važne instrukcije preživele sažimanje.\u003C\u002Fp>\n\u003Cp>Regresija modela i regresija konteksta mogu izgledati identično u konačnom odgovoru. Praćenje stvarne putanje konteksta je ono što omogućava timu da ih razdvoji.\u003C\u002Fp>\n\u003Ch2 id=\"section-48\">RAG stvara sopstveni operativni životni ciklus\u003C\u002Fh2>\n\u003Cp>RAG sistem uvodi drugi produkcioni pipeline pored inferencije modela: unos, ekstrakcija, deljenje na delove, metapodaci, ugrađivanja, indeksi, pretraga, ponovno rangiranje i izbor konteksta.\u003C\u002Fp>\n\u003Cp>Korpus znanja može se menjati svakog dana čak i kada se model i prompt ne menjaju. Zastareo indeks ili neispravan filter metapodataka može stoga pogoršati kvalitet odgovora bez ikakvog odstupanja modela.\u003C\u002Fp>\n\u003Cp>LLMOps za RAG treba da prati verziju korpusa\u002Findeksa, model ugrađivanja, politiku deljenja na delove, konfiguraciju pretrage, svežinu izvora i metrike pretrage odvojeno od kvaliteta generisanja.\u003C\u002Fp>\n\u003Caside class=\"editorjs-referral my-6\">\u003Ca href=\"https:\u002F\u002Fstajic.de\u002Fsr\u002Fblog\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method\" class=\"flex flex-col sm:flex-row gap-4 rounded-xl border border-gray-200 dark:border-gray-700 p-4 transition hover:border-primary-500\">\u003Cdiv class=\"min-w-0 flex-1\">\u003Cstrong class=\"block text-lg text-gray-900 dark:text-gray-100\">RAG nije uspeo — ali koji sloj je zapravo zakazao? Dijagnostička metoda\u003C\u002Fstrong>\u003Cp class=\"mt-2 text-sm text-gray-600 dark:text-gray-300\">Produkcioni LLM pipeline zahteva odvojeno posmatranje pokrivenosti izvora, pretrage, rangiranja, sastavljanja konteksta i generisanja.\u003C\u002Fp>\u003Cspan class=\"mt-3 inline-flex text-sm font-medium text-primary-600 dark:text-primary-400\">Pročitajte RAG dijagnostičku metodu →\u003C\u002Fspan>\u003C\u002Fdiv>\u003C\u002Fa>\u003C\u002Faside>\n\u003Ch2 id=\"section-53\">Evaluacije zamenjuju „izgleda dobro“ dokazima o izdanju\u003C\u002Fh2>\n\u003Cp>Generativni izlazi su često otvoreni, pa su testovi tačnog poklapanja nedovoljni za mnoge zadatke. LLMOps dodaje skupove podataka za evaluaciju i ocenjivače koji mogu meriti uspeh zadatka, tačnost, bezbednost, utemeljenost, stil ili kriterijume prihvatanja specifične za domen.\u003C\u002Fp>\n\u003Cp>MLflow-ov trenutni GenAI stack za evaluaciju podržava verzionisane skupove podataka za evaluaciju, poređenja promptova\u002Fmodela, prilagođene ocenjivače i evaluaciju nad kompletnim tragovima.\u003C\u002Fp>\n\u003Cp>Najjača praksa je razvoj vođen evaluacijom: definišite reprezentativne slučajeve i kriterijume prihvatanja pre ili uporedo sa izmenama, a zatim uporedite izdanja sa istim dokazima.\u003C\u002Fp>\n\u003Caside class=\"editorjs-callout editorjs-callout--success my-6 rounded-xl border p-5 border-emerald-300 bg-emerald-50 dark:border-emerald-900 dark:bg-emerald-950\u002F20\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Promene u ponašanju zahtevaju testove ponašanja\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">Implementacija ne treba da se smatra ekvivalentnom samo zato što API ugovor i dalje funkcioniše. Ako su se prompt, model, pretraga ili alati promenili, skup regresionih testova ponašanja treba ponovo pokrenuti.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Ch2 id=\"section-58\">LLM kao sudija je koristan, ali nije osnovna istina\u003C\u002Fh2>\n\u003Cp>LLM sudije mogu da skaliraju evaluaciju za kvalitete koje je skupo kodirati kao determinističke tvrdnje, kao što su relevantnost, ton ili utemeljenost.\u003C\u002Fp>\n\u003Cp>Međutim, sudija je drugi model sa sopstvenom pristrasnošću, verzijom i promptom. Konfiguracija sudije stoga treba da bude verzionisana i kalibrisana prema ljudskim ili determinističkim referentnim slučajevima tamo gde su posledice važne.\u003C\u002Fp>\n\u003Cp>Produkciona evaluacija može da kombinuje determinističke provere, metrike zasnovane na referencama, model sudije i ljudski pregled, umesto da zahteva da jedna metrika predstavlja svaku dimenziju kvaliteta.\u003C\u002Fp>\n\u003Ch2 id=\"section-62\">Praćenje postaje važnije od logova krajnjih tačaka\u003C\u002Fh2>\n\u003Cp>Tradicionalni API logovi mogu da vam kažu da je zahtev trajao dve sekunde i vratio HTTP 200. Ne mogu da vam kažu koji su izvučeni delovi izabrani, koji je alat agent pozvao ili koji je model segment potrošio najviše tokena.\u003C\u002Fp>\n\u003Cp>MLflow-ovo trenutno GenAI praćenje beleži promptove, pretrage, pozive alata i segmente aplikacije, a njegov tok produkcione evaluacije može da ocenjuje informacije o srednjoj putanji, a ne samo konačni tekst.\u003C\u002Fp>\n\u003Cp>Ovo je veliki pomak u LLMOps-u: observabilnost prati graf ponašanja aplikacije, a ne samo servisnu krajnju tačku.\u003C\u002Fp>\n\u003Ch2 id=\"section-66\">Agenti proširuju LLMOps na operacije u vreme izvršavanja\u003C\u002Fh2>\n\u003Cp>Agentna aplikacija može da izvrši nekoliko poziva modela, poziva alata i prelaza stanja pre nego što proizvede rezultat.\u003C\u002Fp>\n\u003Cp>Operativni agenti stoga zahtevaju brojanje koraka, tragove poziva alata, odbijanja dozvola, ponovne pokušaje, detekciju petlji, ljudska odobrenja i verifikovano konačno stanje pored uobičajenih metrika latencije modela i tokena.\u003C\u002Fp>\n\u003Cp>Tačan konačni odgovor može da sakrije lošu putanju, pa evaluacija agenta mora da pregleda i putanju i rezultat.\u003C\u002Fp>\n\u003Caside class=\"editorjs-referral my-6\">\u003Ca href=\"https:\u002F\u002Fstajic.de\u002Fsr\u002Fblog\u002Fai-agent-reliability-why-the-final-answer-is-not-enough\" class=\"flex flex-col sm:flex-row gap-4 rounded-xl border border-gray-200 dark:border-gray-700 p-4 transition hover:border-primary-500\">\u003Cdiv class=\"min-w-0 flex-1\">\u003Cstrong class=\"block text-lg text-gray-900 dark:text-gray-100\">Pouzdanost AI agenata: Zašto konačni odgovor nije dovoljan\u003C\u002Fstrong>\u003Cp class=\"mt-2 text-sm text-gray-600 dark:text-gray-300\">Zašto produkciona evaluacija agenata mora da uključi pozive alata, prelaze stanja, odobrenja i mogućnost oporavka.\u003C\u002Fp>\u003Cspan class=\"mt-3 inline-flex text-sm font-medium text-primary-600 dark:text-primary-400\">Pročitajte članak o pouzdanosti agenata →\u003C\u002Fspan>\u003C\u002Fdiv>\u003C\u002Fa>\u003C\u002Faside>\n\u003Ch2 id=\"section-71\">Tokeni, pozivi modela i kontekst postaju varijable troškova\u003C\u002Fh2>\n\u003Cp>Trošak klasičnog ML zaključivanja često je dominiran infrastrukturom za serviranje ili računanjem po predikciji. LLM aplikacije mogu da dodaju cene tokena provajdera, ponovljene pozive agenata, pozive za ugrađivanje, ponovno rangiranje i troškove alata\u002Fvremena izvršavanja.\u003C\u002Fp>\n\u003Cp>Trošak se stoga mora pripisati zadatku ili tragu, a ne samo jednom endpointu. Radni tok koji pravi osam skrivenih poziva modela može biti funkcionalno ispravan, ali operativno neprihvatljiv.\u003C\u002Fp>\n\u003Cp>Latencija se ponaša na isti način: latencija modela, pretraga, rerangiranje i eksterni alati se kombinuju u end-to-end latenciju za korisnika.\u003C\u002Fp>\n\u003Ch2 id=\"section-75\">Keširanje postaje semantičko, ne samo tehničko\u003C\u002Fh2>\n\u003Cp>LLM sistemi mogu keširati promptove, embeddinge, rezultate pretrage ili pune odgovore, ali ključ keša mora odražavati semantiku koja može promeniti rezultat.\u003C\u002Fp>\n\u003Cp>Keš odgovora koji ignoriše verziju modela, tenanta, dozvole ili svežinu izvora može vratiti tehnički validan, ali semantički nevalidan odgovor.\u003C\u002Fp>\n\u003Cp>LLMOps stoga tretira invalidaciju keša kao deo verzionisanja modela\u002Fkonteksta\u002Fpodataka, a ne samo kao infrastrukturnu optimizaciju.\u003C\u002Fp>\n\u003Ch2 id=\"section-79\">Bezbednost i dozvole postaju kriterijumi za izdanje\u003C\u002Fh2>\n\u003Cp>Generativni sistemi mogu proizvesti neograničen tekst, a agenti mogu pokrenuti eksterne akcije. Testiranje bezbednosti stoga je bliže običnom CI\u002FCD-u nego u mnogim klasičnim sistemima prediktivnog ML-a.\u003C\u002Fp>\n\u003Cp>Provere dozvola, testovi prompt-injection-a, testovi izolacije tenanta i odobrenja za sporedne efekte treba da budu reproduktivni regresioni testovi tamo gde ti rizici postoje.\u003C\u002Fp>\n\u003Cp>Model može predložiti operaciju, ali runtime i dalje mora sprovesti autorizaciju. LLMOps poseduje dokaze da ti kontroli nastavljaju da rade nakon promena modela, prompta ili alata.\u003C\u002Fp>\n\u003Ch2 id=\"section-83\">Kako CI izgleda u LLMOps-u\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">CI sloj\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Primeri provera\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kod\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Unit testovi, provere tipova, validacija šeme\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Promptovi\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Renderovanje šablona, obavezne promenljive, tekst politike, pregled snimaka\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Modeli\u002Fprovajderi\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kompatibilnost, izlazna šema, testovi sposobnosti i regresije\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">RAG\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Fiksture za chunking, testovi filtera, Recall@k, regresija rerangera\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Alati\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Testovi ulazno\u002Fizlazne šeme, testovi dozvola, testovi idempotentnosti\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Agenti\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Fiksture trajektorije, ograničenja petlje, testovi predaje\u002Fizbora alata\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Bezbednost\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Prompt injection, neovlašćeni alati, negativni testovi između tenanata\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Bihevioralne evaluacije\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Uspeh zadatka, tačnost, utemeljenost, bezbednost, domen-specifični kriterijumi\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Operativno\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Latencija, budžeti za tokene\u002Ftroškove, ponašanje pri timeout-u\u002Ffallback-u\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-85\">Kako CD izgleda u LLMOps-u\u003C\u002Fh2>\n\u003Cp>Produkcijsko izdanje možda uopšte neće deploy-ovati novi artefakt modela. Može jednostavno isporučiti novi prompt, konfiguraciju pretrage, skup alata ili mapiranje provajdera.\u003C\u002Fp>\n\u003Cp>Paket izdanja stoga treba da identifikuje kompletnu konfiguraciju koja definiše ponašanje, a ne samo sliku kontejnera aplikacije.\u003C\u002Fp>\n\u003Cp>Feature flag-ovi, postepeno uvođenje, shadow evaluacija, canary saobraćaj i rollback su korisni jer LLM ponašanje može regresirati na načine koje statički testovi ugovora ne detektuju.\u003C\u002Fp>\n\u003Ch2 id=\"section-89\">Kontinuirano treniranje postaje opciono; kontinuirana evaluacija postaje centralna\u003C\u002Fh2>\n\u003Cp>Tradicionalni MLOps često naglašava kontinuirano treniranje kada novi podaci ili drift opravdavaju ponovno treniranje.\u003C\u002Fp>\n\u003Cp>Mnoge LLM aplikacije nikada ne treniraju osnovni model. Njihova ekvivalentna neprekidna petlja je kontinuirana evaluacija: prikupljaju neuspehe i reprezentativne produkcijske slučajeve, dodaju ih u skupove podataka za evaluaciju, testiraju izmene kandidata za prompt\u002Fmodel\u002Fpretragu i ponovo objavljuju samo kada se dokazi poboljšaju.\u003C\u002Fp>\n\u003Cp>Fino podešavanje može ponovo uvesti životni ciklus treniranja, ali treba da bude deo istog šireg procesa evaluacije i objavljivanja.\u003C\u002Fp>\n\u003Ch2 id=\"section-93\">Šta treba pratiti u produkciji?\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Klasa signala\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Primeri\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Zdravlje sistema\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Greške, isteci vremena, dostupnost endpointa\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Model\u002Fprovider\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">ID modela, snimak, ograničenja brzine, greške provajdera\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Latencija\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Krajnja do krajnje, model, pretraga, alat i reranker opsezi\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Trošak\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ulazni\u002Fizlazni tokeni, embeddingzi, troškovi alata\u002FAPI-ja\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kvalitet\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Uzorkovani uspeh zadatka, tačnost, relevantnost, utemeljenost\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">RAG\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Proksiji za obuhvat pretrage, prazna pretraga, zastareli izvori, pokrivenost citata\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Agenti\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Izbor alata, ponovni pokušaji, petlje, predaje, učestalost odobrenja\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Bezbednost\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Odbijene radnje, indikatori prompt-injection napada, neuspesi na granicama tenanta\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Povratne informacije korisnika\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ispravke, napuštanje, eskalacija, eksplicitne ocene\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Drift promena\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Promene provajdera\u002Fmodela\u002Fkonfiguracije u odnosu na odobreno izdanje\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-95\">Produkcijski tragovi mogu postati podaci za evaluaciju\u003C\u002Fh2>\n\u003Cp>Jedan od najkorisnijih modernih LLMOps obrazaca je pretvaranje uzorkovanih produkcijskih tragova u zapise za evaluaciju.\u003C\u002Fp>\n\u003Cp>MLflow trenutno podržava preuzimanje produkcijskih tragova i ocenjivanje ne samo izlaza već i međukoraka kao što su pretraga ili putanje poziva alata.\u003C\u002Fp>\n\u003Cp>Ovo zatvara petlju između observabilnosti i razvoja: stvarni neuspesi mogu postati regresioni slučajevi u sledećem izdanju umesto da nestanu u logovima.\u003C\u002Fp>\n\u003Ch2 id=\"section-99\">Reproduktivnost postaje uslovna, a ne egzaktna\u003C\u002Fh2>\n\u003Cp>Klasična ML reproduktivnost često ima za cilj da ponovo stvori model iz verzionisanog koda, podataka, okruženja i parametara treniranja.\u003C\u002Fp>\n\u003Cp>Hostovane LLM aplikacije ne mogu uvek da reprodukuju identičan izlaz token po token jer je generisanje probabilističko, a provajderi mogu kontrolisati infrastrukturu.\u003C\u002Fp>\n\u003Cp>LLMOps stoga teži bihevioralnoj reproduktivnosti: beleži dovoljno modela\u002Fprovajdera\u002Fverzije, prompta, kontekstualnih ulaza, stanja pretrage i konfiguracije izvršavanja da bi se reprodukovali uslovi i validiralo ponašanje u okviru očekivanih tolerancija.\u003C\u002Fp>\n\u003Ch2 id=\"section-103\">Loza se širi od loze modela do loze aplikacije\u003C\u002Fh2>\n\u003Cp>AWS-ove MLOps smernice tretiraju lozu modela kao istoriju artefakata koda, podataka, modela i infrastrukture potrebnih za dijagnostiku i reproduktivnost.\u003C\u002Fp>\n\u003Cp>Za LLM aplikacije, loza treba dodatno da poveže promptove, skupove podataka za evaluaciju, verzije pretrage\u002Findeksa, šeme alata, konfiguraciju agenta\u002Fvremena izvršavanja i snimke provajdera\u002Fmodela.\u003C\u002Fp>\n\u003Cp>Ciljno pitanje postaje: Koja tačno konfiguracija aplikacije je proizvela ovaj trag?\u003C\u002Fp>\n\u003Ch2 id=\"section-107\">Usmeravanje ka više provajdera i modela stvara operativnu politiku\u003C\u002Fh2>\n\u003Cp>Kada aplikacija može da koristi nekoliko provajdera ili lokalnih modela, usmeravanje postaje operativna politika, a ne jednostavan string modela.\u003C\u002Fp>\n\u003Cp>Rutiranje može zavisiti od sposobnosti, latencije, cene, privatnosti, dužine konteksta, dostupnosti, podrške za alate ili lokaliteta. Rezervna opcija može očuvati vreme rada dok menja kvalitet odgovora ili pretpostavke obrade podataka.\u003C\u002Fp>\n\u003Cp>LLMOps bi stoga trebalo da beleži koja je ruta zaista izabrana i da nezavisno procenjuje rute, umesto da svaku kompatibilnu krajnju tačku tretira kao bihevioralno zamenljivu.\u003C\u002Fp>\n\u003Ch2 id=\"section-111\">Dokazi iz originalne implementacije\u003C\u002Fh2>\n\u003Ch3 id=\"section-112\">Aaasaasa AI Client: provajder, model i runtime su odvojeni operativni objekti\u003C\u002Fh3>\n\u003Cp>Aaasaasa AI Client razdvaja agenta\u002Fklijenta, provajdera, model, lokaciju runtime-a i dozvole. Njegov AI Hub podržava Ollama, LM Studio\u002FOpenAI-kompatibilne krajnje tačke i druge protokole provajdera, umesto da „model“ tretira kao jedno globalno podešavanje.\u003C\u002Fp>\n\u003Cp>Implementacija uključuje dinamičko otkrivanje lokalnih modela, strimovanje, izlaz razmišljanja i eksplicitne Ollama kontrole zagrevanja\u002Fučitavanja i istovara. To je operativni dokaz da lokalno serviranje LLM-a uvodi brige o životnom ciklusu resursa koje prevazilaze ime API modela.\u003C\u002Fp>\n\u003Cp>Status provajdera se dobija preko adaptera provajdera, a tipovi veze razlikuju lokalne, cloud API, putanje vezane za nalog, udaljene agente i web klijente. To su konkretne operativne dimenzije koje platforma svesna LLM-a mora da izloži.\u003C\u002Fp>\n\u003Cp>Repozitorijum takođe čuva važnu granicu: lokalni runtime nije automatski lokalna inferencija. Lokacija provajdera\u002Fmodela\u002Fruntime-a su pitanja koja se verzioniraju ili konfigurišu i koja utiču na privatnost, latenciju, cenu i dostupnost.\u003C\u002Fp>\n\u003Ch3 id=\"section-117\">Source of Truth Research Engine: stanje LLM aplikacije se proteže izvan modela\u003C\u002Fh3>\n\u003Cp>Source of Truth Research Engine kombinuje leksičku pretragu, opcione embedding-e, snimke izvora, SHA-256 identitet, tvrdnje, poreklo i praćenje kontradikcija oko istraživanja uz pomoć lokalnog modela.\u003C\u002Fp>\n\u003Cp>Ovo je koristan LLMOps dokaz jer samo promena modela ne definiše istraživački sistem. Pretraga, pribavljanje izvora, klasifikacija dokaza i trajno poreklo su nezavisni operativni artefakti.\u003C\u002Fp>\n\u003Cp>Implementacija namerno tretira semantičku sličnost kao otkrivanje, a ne kao dokaz, pokazujući zašto LLMOps observabilnost treba da razlikuje ponašanje pretrage od validnosti tvrdnji.\u003C\u002Fp>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Uočena implementacija\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">LLMOps lekcija\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Više protokola provajdera\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Identitet provajdera je operativna zavisnost\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Dinamičko otkrivanje modela\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Dostupni modeli se mogu menjati nezavisno od koda aplikacije\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ollama kontrole učitavanja\u002Fistovara\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Lokalni modeli imaju životni ciklus memorije\u002Fresursa\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Adapteri za zdravlje\u002Fstatus provajdera\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Dostupnost modela zahteva runtime observabilnost\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Odvojena lokacija runtime-a i inferencije\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Topologija postavljanja nije jedna bulova vrednost „lokalno\u002Fcloud“\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Centralne dozvole\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Sposobnost modela i ovlašćenje alata moraju ostati odvojeni\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Leksički + semantički pipeline pretrage\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Konfiguracija pretrage je deo ponašanja aplikacije\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Trajnost izvora\u002Fporekla\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Operativno stanje i dokazi žive izvan težina modela\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Caside class=\"editorjs-callout editorjs-callout--note my-6 rounded-xl border p-5 border-gray-300 bg-gray-50 dark:border-gray-700 dark:bg-gray-900\u002F40\" role=\"note\">\u003Cstrong class=\"block mb-2 text-gray-900 dark:text-gray-100\">Granica dokaza\u003C\u002Fstrong>\u003Cdiv class=\"text-gray-700 dark:text-gray-200\">Ovi projekti demonstriraju operacije sa više provajdera\u002Flokalnih modela, razdvajanje dozvola, infrastrukturu pretrage i trajnost dokaza. Nisu predstavljeni kao kompletna komercijalna LLMOps platforma niti kao dokaz velikog obima produkcionog saobraćaja.\u003C\u002Fdiv>\u003C\u002Faside>\n\u003Ch2 id=\"section-123\">Uobičajeni režimi neuspeha u LLMOps-u\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Režim neuspeha\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Šta je zapravo pošlo naopako\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Alias modela tiho nadograđen\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ponašanje se promenilo bez kontrolisanog izdanja\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Prompt promenjen bez evaluacija\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Regresija ponašanja prošla je normalne unit testove\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">RAG indeks zastareo\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Model generisanja je okrivljen za neuspeh pretrage\u002Fpodataka\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Beleži se samo konačni odgovor\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Osnovni uzrok u putanji pretrage\u002Falata\u002Fkonteksta je nevidljiv\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Rezervna opcija provajdera je tiha\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Drugačiji model\u002Fputanja podataka menja ponašanje bez pripisivanja\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Trošak tokena se prati globalno\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Skupi tokovi rada ne mogu se lokalizovati\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Model procenjivač promenjen\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ocene evaluacije driftuju bez promene aplikacije\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Produkcioni tragovi nikada ne postaju testovi\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Poznati neuspesi se ponovo vraćaju\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Lokalni model ostaje učitan neograničeno\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Pritisak na VRAM\u002Fresurse postaje operativna nestabilnost\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Dozvole kodirane samo u promptu\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ponašanje modela se pogrešno smatra autorizacijom\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Jedna ocena evaluacije određuje sve\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Različite dimenzije kvaliteta se svode na obmanjujući broj\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Registar modela postoji, ali verzije prompta\u002Findeksa ne\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Rodoslov aplikacije ostaje nepotpun\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-125\">Uobičajene zablude\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Zabluda\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Ispravka\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„LLMOps zamenjuje MLOps.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">LLMOps proširuje MLOps principe na ponašanje aplikacije specifično za LLM.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„LLMOps je prompt inženjering.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Promptovi su jedan artefakt među modelima, provajderima, kontekstom, pretragom, alatima, evaluacijama i runtime-om.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Hostovani API-ji uklanjaju operativni rad.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Oni uklanjaju deo rada na serviranju\u002Ftreniranju modela, ali dodaju upravljanje životnim ciklusom provajdera, verzijama i zavisnostima.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Ako je API stabilan, aplikacija je stabilna.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Ponašanje modela i snimci provajdera\u002Fmodela mogu se menjati nezavisno od API šeme.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„RAG je samo predobrada podataka.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">U produkciji ima sopstveni životni ciklus unosa, indeksa, pretrage i svežine.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„LLM izlazi ne mogu se testirati.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Mogu se evaluirati determinističkim, referentnim, procenjivačkim i ljudskim kriterijumima.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„LLM procenjivači su objektivna osnovna istina.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Oni su evaluatori zasnovani na modelu koji takođe zahtevaju kalibraciju i kontrolu verzija.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Lokalni model eliminiše LLMOps.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Lokalno serviranje dodaje fajlove modela, VRAM, učitavanje\u002Fistovar, zdravlje runtime-a i brige o nadogradnji.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Observabilnost znači brojanje tokena.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Korisna observabilnost prati promptove, pretrage, alate, raspone modela i ishode.\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">„Kontinuirano treniranje je obavezno.“\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Mnoge LLM aplikacije koriste kontinuiranu evaluaciju bez treniranja osnovnog modela.\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-127\">Praktičan redosled dizajna LLMOps-a\u003C\u002Fh2>\n\u003Csection class=\"editorjs-process my-6\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">Upravljajte kompletnim sistemom koji proizvodi ponašanje\u003C\u002Fh3>\u003Cdiv class=\"grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4\">\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">1\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">1. Definišite jedinicu ponašanja\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Navedite svaku komponentu koja može materijalno da promeni izlaz: model, uputstvo, pretragu, alate, kontekst i politiku.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">2\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">2. Uspostavite lozu aplikacije\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Verzionirajte kod, model\u002Fprovajdera, uputstva, skupove podataka za evaluaciju, konfiguraciju pretrage i ugovore alata.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">3\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">3. Izgradite reprezentativne skupove podataka za evaluaciju\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Koristite očekivane slučajeve uspeha\u002Fneuspeha iz dizajna i produkcije.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">4\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">4. Razdvojite determinističke i bihevioralne testove\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Držite provere šeme\u002Fbezbednosti odvojeno od semantičke evaluacije izlaza.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">5\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">5. Pratite izvršavanje od početka do kraja\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Instrumentišite model, pretragu, rangiranje, alate i opsege agenta\u002Fruntime-a.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">6\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">6. Definišite kapije za izdanja\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Postavite pragove kvaliteta, bezbednosti, latencije i troškova.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">7\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">7. Fiksirajte ili eksplicitno zabeležite verzije modela\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Tretirajte promene modela\u002Fprovajdera kao događaje izdanja.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">8\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">8. Rasporedite postepeno\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Koristite zastavice, kanarinac ili fazno uvođenje gde posledice to opravdavaju.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">9\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">9. Evaluirajte produkcijske tragove\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Izmerite stvarno ponašanje zadatka i identifikujte ponavljajuće se neuspehe.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">10\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">10. Vratite neuspehe nazad u skupove podataka za evaluaciju\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Pretvorite incidente i ispravke u trajnu pokrivenost regresije.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">11\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">11. Nadgledajte životne cikluse provajdera i podataka\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Pratite ukidanja, svežinu indeksa, promene izvora i dostupnost runtime-a.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv class=\"editorjs-process__step min-w-0  rounded-xl border border-gray-200 dark:border-gray-700 p-4\">\u003Cdiv class=\"text-xs font-semibold text-gray-500 dark:text-gray-400\">12\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 font-semibold text-gray-900 dark:text-gray-100\">12. Uklonite zastarele verzije čisto\u003C\u002Fdiv>\u003Cdiv class=\"mt-1 text-sm text-gray-600 dark:text-gray-300\">Uklonite stare uputstva\u002Fmodele\u002Findekse\u002Fakreditive nakon migracije i odluka o zadržavanju dokaza.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fsection>\n\u003Ch2 id=\"section-129\">Kontrolna lista arhitekture LLMOps-a\u003C\u002Fh2>\n\u003Cdiv class=\"overflow-x-auto\">\u003Ctable class=\"w-full border-collapse\">\u003Cthead>\u003Ctr>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Pitanje\u003C\u002Fth>\u003Cth class=\"border border-gray-300 px-4 py-2 text-left font-semibold\">Očekivani dokaz\u003C\u002Fth>\u003C\u002Ftr>\u003C\u002Fthead>\u003Ctbody>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Koji model\u002Fprovajder\u002Fverzija je poslužio zahtev?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Identitet modela koji se može pratiti\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Koje uputstvo\u002Finstrukcije su bile aktivne?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verzionirani kod\u002Fkonfiguracija aplikacije\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Koji kontekst je stigao do modela?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Trag konteksta\u002Fpretrage\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Koja verzija korpusa\u002Findeksa je korišćena?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Loza pretrage\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Koji alati su bili dostupni i pozvani?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Šema alata + trag trajektorije\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Koje dozvole su primenjene?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Zapis o autorizaciji u runtime-u\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kako se meri kvalitet?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Verzionirani skup podataka za evaluaciju + ocenjivači\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kako se testiraju nadogradnje modela?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Bihevioralni paket regresionih testova\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kako se uzorkuje kvalitet u produkciji?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Proces evaluacije tragova\u002Fpovratnih informacija\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Može li se jedan neuspeh približno reprodukovati?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Loza modela\u002Fkonteksta\u002Fprovajdera\u002Faplikacije\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Gde se troše troškovi?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Atribucija modela\u002Falata\u002Fpretrage po tragu\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Šta pokreće vraćanje na prethodno stanje?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Definisani prag kvaliteta\u002Fbezbednosti\u002Ftroškova\u002Fdostupnosti\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kako se rukuje ukidanjem provajdera?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Proces migracije\u002Frezervnog rešenja\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kako se upravlja lokalnim modelima?\u003C\u002Ftd>\u003Ctd class=\"border border-gray-300 px-4 py-2\">Kontrole zdravlja, resursa, učitavanja\u002Fistovara i verzija\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftbody>\u003C\u002Ftable>\u003C\u002Fdiv>\n\u003Ch2 id=\"section-131\">Rubni slučajevi i ograničenja\u003C\u002Fh2>\n\u003Cp>Jednostavna aplikacija koja poziva jedan fiksni hostovani model bez pretrage ili alata može zahtevati samo lagani LLMOps: verzionisani kod uputstava, evaluacije, fiksiranje modela, osnovno praćenje i nadgledanje provajdera.\u003C\u002Fp>\n\u003Cp>Samostalno hostovani fino podešeni model može zahtevati skoro pun klasični MLOps stek plus LLM-specifičnu evaluaciju aplikacije, čineći granicu između MLOps-a i LLMOps-a namerno zamućenom.\u003C\u002Fp>\n\u003Cp>Platforma agenata može imati minimalne operacije obuke modela, ali značajne runtime operacije jer se neuspesi javljaju u izboru alata, stanju i orkestraciji.\u003C\u002Fp>\n\u003Cp>Sistem sa puno RAG-a može biti operativno dominiran unosom dokumenata i kvalitetom pretrage, a ne posluživanjem modela.\u003C\u002Fp>\n\u003Cp>Terminologija će nastaviti da se razvija. Trajno arhitektonsko pitanje nije koja oznaka „Ops“ pobeđuje, već koji artefakti proizvode ponašanje i stoga moraju biti verzionisani, evaluirani, posmatrani i upravljani.\u003C\u002Fp>\n\u003Ch2 id=\"section-137\">Šta bi promenilo ovaj odgovor?\u003C\u002Fh2>\n\u003Cp>Ako provajderi osnovnih modela standardizuju savršeno stabilno ponašanje modela i dugoročnu podršku verzijama, upravljanje provajderima\u002Fsnimcima moglo bi postati operativno manje značajno.\u003C\u002Fp>\n\u003Cp>Ako aplikacije sve više preuzimaju fino podešavanje ili obuku, klasične MLOps brige ponovo postaju centralne.\u003C\u002Fp>\n\u003Cp>Operativni princip bi ostao: svaka komponenta koja može materijalno da promeni produkcijsko ponašanje pripada lozi, testiranju, observabilnosti i kontroli promena.\u003C\u002Fp>\n\u003Ch2 id=\"section-141\">Povezano kanonsko znanje\u003C\u002Fh2>\n\u003Cp>LLMOps se nalazi ispod AI upravljanja i arhitekture preduzeća AI: upravljanje definiše koje promene zahtevaju dokaz i odobrenje, dok LLMOps pruža operativnu mašineriju za verzionisanje, evaluaciju, raspoređivanje i posmatranje tih promena.\u003C\u002Fp>\n\u003Cp>Inženjering konteksta i RAG su operativni poddomeni unutar mnogih LLM aplikacija jer kontekst i pretraga mogu promeniti ponašanje nezavisno od modela.\u003C\u002Fp>\n\u003Cp>Agentni AI proširuje LLMOps dalje u operacije trajektorije, dozvola i runtime-a alata.\u003C\u002Fp>\n\u003Caside class=\"editorjs-referral my-6\">\u003Ca href=\"https:\u002F\u002Fstajic.de\u002Fsr\u002Fblog\u002Fai-agent-memory-is-not-rag-how-to-separate-memory-retrieval-state-and-context\" class=\"flex flex-col sm:flex-row gap-4 rounded-xl border border-gray-200 dark:border-gray-700 p-4 transition hover:border-primary-500\">\u003Cdiv class=\"min-w-0 flex-1\">\u003Cstrong class=\"block text-lg text-gray-900 dark:text-gray-100\">Memorija AI agenata nije RAG: Kako razdvojiti memoriju, pretragu, stanje i kontekst\u003C\u002Fstrong>\u003Cp class=\"mt-2 text-sm text-gray-600 dark:text-gray-300\">Operativna pouzdanost se poboljšava kada memorija, pretraga, stanje aplikacije i kontekst modela ostanu odvojeni objekti životnog ciklusa.\u003C\u002Fp>\u003Cspan class=\"mt-3 inline-flex text-sm font-medium text-primary-600 dark:text-primary-400\">Pročitajte članak o arhitekturi →\u003C\u002Fspan>\u003C\u002Fdiv>\u003C\u002Fa>\u003C\u002Faside>\n\u003Caside class=\"editorjs-referral my-6\">\u003Ca href=\"https:\u002F\u002Fstajic.de\u002Fsr\u002Fblog\u002Fthe-answer-validity-boundary-the-missing-layer-between-relevance-and-reliable-ai-answers\" class=\"flex flex-col sm:flex-row gap-4 rounded-xl border border-gray-200 dark:border-gray-700 p-4 transition hover:border-primary-500\">\u003Cdiv class=\"min-w-0 flex-1\">\u003Cstrong class=\"block text-lg text-gray-900 dark:text-gray-100\">Granica validnosti odgovora: Nedostajući sloj između relevantnosti i pouzdanih AI odgovora\u003C\u002Fstrong>\u003Cp class=\"mt-2 text-sm text-gray-600 dark:text-gray-300\">LLMOps evaluacija treba da očuva verziju, obim i uslove dokaza pod kojima odgovor ostaje potkrepljen.\u003C\u002Fp>\u003Cspan class=\"mt-3 inline-flex text-sm font-medium text-primary-600 dark:text-primary-400\">Pročitajte o granici validnosti odgovora →\u003C\u002Fspan>\u003C\u002Fdiv>\u003C\u002Fa>\u003C\u002Faside>\n\u003Ch2 id=\"section-147\">Često postavljana pitanja\u003C\u002Fh2>\n\u003Csection class=\"editorjs-faq my-6 rounded-xl border border-gray-200 p-5 dark:border-gray-700\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">MLOps vs LLMOps FAQ\u003C\u002Fh3>\u003Cdiv id=\"faq1\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Koja je razlika između MLOps i LLMOps?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">MLOps upravlja sistemima mašinskog učenja kroz podatke, obuku, implementaciju i nadzor. LLMOps proširuje te prakse na LLM aplikacije gde promptovi, kontekst, pretraga, provajderi, alati i evaluacije takođe materijalno utiču na ponašanje.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq2\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Da li LLMOps zamenjuje MLOps?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Ne. LLMOps ponovo koristi MLOps discipline kao što su CI\u002FCD, poreklo, evaluacija, implementacija i nadzor i dodaje operativne brige specifične za LLM.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq3\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Da li LLM aplikacije zahtevaju kontinuiranu obuku?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Ne nužno. Mnoge koriste eksterne temeljne modele i umesto toga se oslanjaju na kontinuiranu evaluaciju promptova, modela, pretrage i ponašanja aplikacije. Fino podešeni ili samostalno obučeni sistemi i dalje mogu zahtevati pipeline za obuku.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq4\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Zašto su evaluacije toliko važne u LLMOps?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Generativni izlazi su otvoreni i ponašanje modela može se promeniti kroz promptove, snimke i kontekst. Evaluacije pružaju ponovljive dokaze da izdanje i dalje zadovoljava definisane kriterijume kvaliteta i bezbednosti.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq5\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Šta treba verzionisati u LLMOps?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Najmanje: kod aplikacije, model\u002Fprovajder\u002Fverziju, promptove, skupove podataka za evaluaciju\u002Focenjivače, konfiguraciju pretrage\u002Findekse, šeme alata, pravila konteksta i relevantnu konfiguraciju bezbednosti\u002Fdozvola.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq6\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Da li je verzionisanje promptova dovoljno?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Ne. Isti prompt može se ponašati drugačije sa drugim modelom, skupom pretrage, redosledom konteksta, površinom alata ili provajderom.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq7\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Šta je GenAIOps?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">GenAIOps je još jedan industrijski termin za upravljanje generativnim AI aplikacijama. Neki prodavci ga koriste naizmenično ili kao širu oznaku od LLMOps.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq8\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Kako nadzirati LLM aplikaciju?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Nadzirite tragove od početka do kraja uključujući pozive modela, promptove\u002Fkontekst, pretragu, alate, latenciju, tokene\u002Ftroškove, uzorke kvaliteta, bezbednost i konačne ishode zadataka.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003Cdiv id=\"faq9\" class=\"border-t border-gray-200 py-4 first:border-t-0 dark:border-gray-700\">\u003Ch4 class=\"font-semibold text-gray-900 dark:text-gray-100\">Mogu li lokalni LLM-ovi koristiti LLMOps prakse?\u003C\u002Fh4>\u003Cdiv class=\"mt-2 text-gray-600 dark:text-gray-300\">Da. Lokalni modeli dodaju sopstvene operativne brige kao što su datoteke modela, hardver\u002FVRAM, učitavanje\u002Fistovar, zdravlje okruženja, kvantizacija i upravljanje nadogradnjama.\u003C\u002Fdiv>\u003C\u002Fdiv>\u003C\u002Fsection>\n\u003Ch2 id=\"section-149\">Pojmovnik\u003C\u002Fh2>\n\u003Csection class=\"editorjs-glossary my-6 rounded-xl border border-gray-200 dark:border-gray-700 p-5\">\u003Ch3 class=\"mb-3 text-lg font-semibold\">Ključni MLOps i LLMOps pojmovi\u003C\u002Fh3>\u003Cdl>\u003Cdiv id=\"mlops\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">MLOps\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Inženjerske prakse za izgradnju, implementaciju, nadzor i održavanje sistema mašinskog učenja i njihovog životnog ciklusa podataka\u002Fmodela.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"llmops\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">LLMOps\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Operativne prakse za produkcijske aplikacije čije ponašanje materijalno zavisi od velikih jezičkih modela i okolnih promptova, konteksta, pretrage, alata i okruženja.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"genaiops\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">GenAIOps\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Operativna disciplina za generativne AI aplikacije; često se koristi kao šira ili alternativna oznaka za LLMOps.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"continuous-training\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Kontinuirana obuka\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Automatizovano ili ponovljeno ponovno obučavanje i serviranje ML modela kako se podaci ili implementacije menjaju.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"continuous-evaluation\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Kontinuirana evaluacija\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Ponovljena evaluacija ponašanja kandidata i produkcijskih AI sistema prema verzionisanim skupovima podataka i kriterijumima.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"model-snapshot\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Snimak modela\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Konkretna verzija hostovanog ili pakovanog modela čije ponašanje može biti testirano i referencirano.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"application-lineage\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Poreklo aplikacije\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Sledljiv odnos između koda, modela\u002Fprovajdera, promptova, podataka\u002Fpretrage, alata, okruženja i konfiguracije izdanja.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"trace\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Trag\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Strukturirani zapis jednog izvršavanja aplikacije koji sadrži segmente kao što su pozivi modela, pretrage i operacije alata.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"eval-dataset\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Skup podataka za evaluaciju\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Verzionisani skup reprezentativnih ulaza, očekivanja i opciono tragova\u002Fizlaza koji se koristi za merenje ponašanja.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"llm-judge\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">LLM sudija\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Jezički model koji se koristi kao evaluator za kvalitativne ili semantičke kriterijume; i sam je verzionisana zavisnost evaluacije.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"behavioral-regression\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Regresija ponašanja\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Degradacija izlaza ili putanje aplikacije uprkos tome što interfejsi i kod nastavljaju uspešno da se izvršavaju.\u003C\u002Fdd>\u003C\u002Fdiv>\u003Cdiv id=\"provider-routing\" class=\"border-t border-gray-200 dark:border-gray-700 py-3 first:border-t-0\">\u003Cdt class=\"font-semibold text-gray-900 dark:text-gray-100\">Usmeravanje provajdera\u003C\u002Fdt>\u003Cdd class=\"mt-1 text-gray-600 dark:text-gray-300\">Politika za izbor između dostupnih provajdera\u002Fendpointa modela prema sposobnosti, trošku, latenciji, privatnosti ili dostupnosti.\u003C\u002Fdd>\u003C\u002Fdiv>\u003C\u002Fdl>\u003C\u002Fsection>\n\u003Ch2 id=\"section-151\">Zaključak\u003C\u002Fh2>\n\u003Cp>MLOps i LLMOps dele isti inženjerski cilj: učiniti AI sisteme dovoljno reproduktivnim, dovoljno testabilnim i dovoljno opservabilnim da pouzdano rade u produkciji.\u003C\u002Fp>\n\u003Cp>Razlika je u obliku sistema. Klasični MLOps se često fokusira na obuku i serviranje artefakata modela; LLMOps mora upravljati bihevioralnim stekom u kojem se snimci modela, promptovi, kontekst, pretraga, alati, dozvole i provajderi mogu menjati nezavisno.\u003C\u002Fp>\n\u003Cp>Najkraće korisno pravilo je: verzionisati, evaluirati i posmatrati sve što može materijalno promeniti ponašanje LLM aplikacije — ne samo model.\u003C\u002Fp>\n\u003Ch2 id=\"section-155\">Primarni izvori i trenutna dokumentacija\u003C\u002Fh2>\n\u003Cp>Izvori u nastavku postavljaju MLOps osnovu i trenutne operativne obrasce za LLM i agentske aplikacije. Projektni odeljci su originalni dokazi implementacije i namerno su uži od tvrdnji o kompletnoj LLMOps platformi.\u003C\u002Fp>\n\u003Ca href=\"https:\u002F\u002Fdocs.cloud.google.com\u002Farchitecture\u002Fmlops-continuous-delivery-and-automation-pipelines-in-machine-learning\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">Google Cloud — MLOps: Pipeline za kontinuiranu isporuku i automatizaciju\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Referentna arhitektura koja opisuje CI, CD, kontinuiranu obuku, registar modela, metapodatke, serviranje i nadzor za ML sisteme.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops02-bp04.html\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">AWS Machine Learning Lens — Poreklo modela\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutne smernice za praćenje koda, podataka, modela, okruženja i infrastrukture kroz ML izdanja.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops06-bp02.html\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">AWS Machine Learning Lens — Opservabilnost i praćenje modela\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutne smernice za nadzor produkcijskih modela, drift, zdravlje endpointa i poreklo.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fmachine-learning\u002Fprompt-flow\u002Fhow-to-end-to-end-llmops-with-prompt-flow\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">Microsoft Azure — GenAIOps \u002F LLMOps životni ciklus\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Zvanične smernice koje opisuju GenAIOps, ponekad nazvan LLMOps, kroz inicijalizaciju, eksperimentisanje, evaluaciju\u002Frafiniranje i implementaciju.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002F\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">MLflow — Agenti i LLM aplikacije\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutna GenAI operativna dokumentacija koja pokriva praćenje, evaluaciju, promptove i produkcijsku opservabilnost za LLM aplikacije i agente.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fwww.mlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Feval-monitor\u002Frunning-evaluation\u002Ftraces\u002F\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">MLflow — Evaluacija produkcijskih tragova\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutne smernice za evaluaciju kompletnih LLM\u002Fagentskih tragova, uključujući pretragu i putanje poziva alata.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Fprompt-registry\u002Fevaluate-prompts\u002F\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">MLflow — Evaluacija promptova\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutni tok rada za evaluaciju promptova\u002Fmodela koristeći verzionisane promptove, skupove podataka, ocenjivače i tragove.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Freference\u002Foverview\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">OpenAI API — Verzionisanje i snimci modela\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutne smernice API-ja koje preporučuju fiksirane verzije modela i evaluacije jer se ponašanje promptovanja može promeniti između snimaka.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fguides\u002Fprompting\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">OpenAI — Promptovanje\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutne smernice da se produkcijski promptovi tretiraju kao aplikacijski kod, verzioniraju kroz kontrolu izvornog koda i da se promene pokriju testovima i proverama evaluacije.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fdeprecations\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">OpenAI — Ukidanja\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutni dokazi o životnom ciklusu provajdera koji pokazuju ukidanje modela i platformskih površina kao operativnu zavisnost.\u003C\u002Fp>\u003C\u002Fa>\n\u003Ca href=\"https:\u002F\u002Fdevelopers.openai.com\u002Fcookbook\u002Fexamples\u002Fevaluation\u002Fmoving-from-openai-evals-to-promptfoo\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"editorjs-link-tool block border border-gray-200 dark:border-gray-700 rounded-lg p-4 transition text-gray-900 dark:text-gray-100 hover:border-primary-500 hover:bg-primary-50 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100\">\u003Cstrong class=\"block font-semibold\">OpenAI — Premeštanje tokova evaluacije na Promptfoo\u003C\u002Fstrong>\u003Cp class=\"text-sm text-gray-600 dark:text-gray-400\">Trenutne smernice za migraciju iz 2026. koje ilustruju zašto sredstva za evaluaciju treba da ostanu prenosiva kada se alat provajdera menja.\u003C\u002Fp>\u003C\u002Fa>",{"time":212,"blocks":213,"version":1559},1791487465803,[214,220,228,235,242,248,256,261,266,271,276,281,286,291,296,301,306,311,316,348,353,358,363,368,373,421,426,431,436,441,446,496,501,506,511,516,521,526,531,536,541,546,551,556,561,566,571,576,581,586,591,596,605,610,615,620,625,632,637,642,647,652,657,662,667,672,677,682,687,692,700,705,710,715,720,725,730,735,740,745,750,755,760,765,800,805,810,815,820,825,830,835,840,845,880,885,890,895,900,905,910,915,920,925,930,935,940,945,950,955,960,965,970,975,980,985,990,995,1000,1005,1010,1042,1048,1053,1097,1102,1140,1145,1187,1192,1242,1247,1252,1257,1262,1267,1272,1277,1282,1287,1292,1297,1302,1307,1312,1320,1328,1333,1375,1380,1429,1434,1439,1444,1449,1454,1459,1469,1478,1487,1496,1505,1514,1523,1532,1541,1550],{"id":215,"data":216,"type":218,"tunes":219},"intro",{"text":217},"MLOps je inženjerska disciplina za pouzdan razvoj, implementaciju, verzionisanje i upravljanje sistemima mašinskog učenja; LLMOps proširuje tu disciplinu na aplikacije izgrađene oko velikih jezičkih modela, gde ponašanje u produkciji zavisi ne samo od artefakta modela već i od upita, konteksta, pretraživanja, verzija provajdera\u002Fmodela, poziva alata, sigurnosnih kontrola i pipeline-ova za evaluaciju. LLMOps ne zamenjuje MLOps. On menja operativnu jedinicu sa „model plus pipeline za serviranje“ na „LLM aplikaciju koja se razvija i čije ponašanje proizlazi iz nekoliko nezavisno promenljivih komponenti“.","paragraph",{},{"id":221,"data":222,"type":226,"tunes":227},"direct",{"body":223,"title":224,"variant":225},"\u003Cstrong>MLOps upravlja ML sistemima. LLMOps upravlja LLM aplikacijama.\u003C\u002Fstrong>\u003Cbr>\u003Cbr>Klasičan MLOps se obično fokusira na data pipeline-ove, treniranje, validaciju, registar modela, implementaciju, drift i ponovno treniranje. LLMOps zadržava te discipline tamo gde su relevantne, ali često dodaje verzionisanje upita\u002Fkonteksta, apstrakciju modela\u002Fprovajdera, RAG indekse, tragove agenata\u002Falata, semantičke evaluacije, sigurnosne testove, praćenje tokena\u002Ftroškova i regresiono testiranje kroz brzo promenljive snimke modela.","Direktan odgovor","info","callout",{},{"id":229,"data":230,"type":226,"tunes":234},"boundary",{"body":231,"title":232,"variant":233},"Produkcijska LLM aplikacija može da zakaže čak i kada upit ostane nepromenjen: provajder može da promeni snimak modela, RAG korpus može da postane zastareo, reranker može da regresira, dozvole alata se mogu promeniti, sastavljanje konteksta može da izgubi dokaze ili agent može da zauzme pogrešnu putanju. LLMOps zato mora da posmatra i verzionira sistem oko modela, a ne samo tekst upita.","LLMOps nije samo upravljanje upitima","warning",{},{"id":236,"data":237,"type":226,"tunes":241},"term-note",{"body":238,"title":239,"variant":240},"\u003Cstrong>LLMOps\u003C\u002Fstrong>, \u003Cstrong>GenAIOps\u003C\u002Fstrong> i srodni termini su široko korišćene inženjerske oznake, ali nisu jedinstveni univerzalni formalni standard sa jednim kanonskim životnim ciklusom. Microsoft trenutno opisuje GenAIOps kao „ponekad nazvan LLMOps“, dok MLflow grupiše operativne alate oko agenata i LLM aplikacija. Ovaj članak koristi LLMOps kao praktičan arhitektonski termin za upravljanje produkcijskim sistemima čije ponašanje materijalno zavisi od LLM-ova.","Terminološka granica","note",{},{"id":243,"data":244,"type":226,"tunes":247},"current",{"body":245,"title":246,"variant":240},"Operativna površina se brzo menja. OpenAI trenutno preporučuje fiksiranje snimaka modela i pokretanje evaluacija jer se ponašanje upita može promeniti između snimaka, a nekoliko starijih platformski specifičnih površina za upite\u002Fevaluacije se ukida u 2026. Stabilna arhitektonska lekcija je da upiti, testovi i evaluacije ostanu prenosivi i verzionisani uz aplikaciju, umesto da zavise od objektnog modela kontrolne table jednog provajdera.","Napomena o aktuelnim izvorima — 8. oktobar 2026.",{},{"id":249,"data":250,"type":254,"tunes":255},"toc",{"title":251,"maxLevel":252,"minLevel":253},"Sadržaj",3,2,"tableOfContents",{},{"id":257,"data":258,"type":42,"tunes":260},"h-meaning",{"text":259,"level":253},"Šta MLOps zaista znači",{},{"id":262,"data":263,"type":218,"tunes":265},"p-mlops-1",{"text":264},"MLOps primenjuje softversko-inženjersku i operativnu disciplinu na sisteme mašinskog učenja. Produkcijski izazov je širi od treniranja modela: prikupljanje podataka, validacija podataka, eksperimentisanje, reproduktivnost, evaluacija modela, implementacija, infrastruktura i monitoring moraju da rade zajedno.",{},{"id":267,"data":268,"type":218,"tunes":270},"p-mlops-2",{"text":269},"Google-ove smernice za MLOps arhitekturu definišu disciplinu oko kontinuirane integracije, kontinuirane isporuke i kontinuiranog treniranja. CI validira ne samo kod već i podatke, šeme i modele; CD implementira ML pipeline-ove i servise za predikciju; CT može ponovo da trenira i implementira modele kada se podaci ili implementacije promene.",{},{"id":272,"data":273,"type":218,"tunes":275},"p-mlops-3",{"text":274},"AWS smernice dodaju iste operativne brige iz drugog ugla: poreklo modela, sledljivost modela\u002Fverzija, praćenje drifta i praćenje produkcijskog kvaliteta su ključni delovi održavanja pouzdanosti ML sistema nakon implementacije.",{},{"id":277,"data":278,"type":42,"tunes":280},"h-llmops",{"text":279,"level":253},"Šta se menja kada je model LLM",{},{"id":282,"data":283,"type":218,"tunes":285},"p-llmops-1",{"text":284},"Veliki jezički modeli menjaju produkcijski problem jer aplikacija često ne poseduje kompletan životni ciklus treniranja modela. Tim može da poziva hostovanu API uslugu modela, pokreće otvoreni model lokalno, menja provajdere ili koristi nekoliko modela za različite zadatke.",{},{"id":287,"data":288,"type":218,"tunes":290},"p-llmops-2",{"text":289},"Model je stoga samo jedna verzionisana zavisnost unutar većeg bihevioralnog sistema. Upiti, rezultati pretraživanja, redosled konteksta, alati, snimak modela, podešavanja temperature\u002Fzaključivanja, sigurnosni filteri i runtime orkestracija mogu svi da promene izlaz.",{},{"id":292,"data":293,"type":218,"tunes":295},"p-llmops-3",{"text":294},"To stvara šire operativno pitanje: koja kombinacija modela, konteksta, podataka, upita, alata i runtime-a je proizvela ovo ponašanje? LLMOps postoji da učini to pitanje odgovorivim, a odgovor dovoljno reproduktivnim za inženjerski rad.",{},{"id":297,"data":298,"type":42,"tunes":300},"h-simple",{"text":299,"level":253},"Najjednostavniji primer",{},{"id":302,"data":303,"type":218,"tunes":305},"p-simple-1",{"text":304},"Pretpostavimo da aplikacija odgovara na pitanja o internim politikama.",{},{"id":307,"data":308,"type":218,"tunes":310},"p-simple-2",{"text":309},"U klasičnom ML okviru, mogli biste da verzionirate trenirani klasifikator, implementirate ga i pratite kvalitet predikcije. U LLM aplikaciji, odgovor može da zavisi od snimka hostovanog modela, sistemskog upita, modela za ugrađivanje, vektorskog indeksa, filtera pretraživanja, rerankera i konačno izabranog konteksta.",{},{"id":312,"data":313,"type":218,"tunes":315},"p-simple-3",{"text":314},"Promena bilo koje od tih komponenti može da promeni konačni odgovor iako krajnja tačka aplikacije i korisničko pitanje ostaju identični.",{},{"id":317,"data":318,"type":346,"tunes":347},"simple-flow",{"steps":319,"title":344,"orientation":345},[320,323,326,329,332,335,338,341],{"label":321,"description":322},"1. Promenite jednu komponentu","Prompt, model, provajder, podešavanje pretrage, šema alata ili izmene koda aplikacije.",{"label":324,"description":325},"2. Pokrenite determinističke testove","Validirajte šeme, dozvole, ugovore alata, filtere pretrage i ponašanje aplikacije.",{"label":327,"description":328},"3. Pokrenite bihevioralne evaluacije","Uporedite reprezentativne izlaze, kvalitet pretrage i trajektorije agenata\u002Falata u odnosu na kriterijume prihvatanja.",{"label":330,"description":331},"4. Uporedite cenu i latenciju","Izmerite korišćenje tokena, pozive modela, troškove pretrage\u002Falata i latenciju odgovora.",{"label":333,"description":334},"5. Implementirajte kontrolisanu verziju","Isporučite konkretnu konfiguraciju aplikacije sa zabeleženim verzijama modela\u002Fprovajdera.",{"label":336,"description":337},"6. Pratite ponašanje u produkciji","Snimite relevantne opsege modela, pretrage, alata i izvršnog okruženja.",{"label":339,"description":340},"7. Evaluirajte produkcijske tragove","Uzmite uzorke stvarnih izvršavanja radi kvaliteta, utemeljenosti, bezbednosti i uspeha zadatka.",{"label":342,"description":343},"8. Vratite ili iterirajte","Koristite dokaze o regresiji i operativne signale da odlučite o sledećem izdanju.","Tipičan put izdavanja u LLMOps-u","auto","processFlow",{},{"id":349,"data":350,"type":42,"tunes":352},"h-stops",{"text":351,"level":253},"Gde se jednostavan primer zaustavlja",{},{"id":354,"data":355,"type":218,"tunes":357},"p-stops-1",{"text":356},"Neki LLM sistemi i dalje treniraju ili fino podešavaju sopstvene modele, tako da tradicionalne MLOps prakse kao što su pipeline-ovi za treniranje, registar modela i poreklo podataka ostaju direktno relevantne.",{},{"id":359,"data":360,"type":218,"tunes":362},"p-stops-2",{"text":361},"Drugi sistemi koriste samo eksterne API-je foundation modela i nikada ne pokreću kontinuirano treniranje. Njihov glavni operativni teret je evaluacija aplikacije, upravljanje promenama modela\u002Fprovajdera, verzionisanje prompta\u002Fkonteksta, kvalitet pretrage i observabilnost.",{},{"id":364,"data":365,"type":218,"tunes":367},"p-stops-3",{"text":366},"Zato ne postoji jedinstveni univerzalni „LLMOps pipeline“. Tačan životni ciklus zavisi od toga da li trenirate, fino podešavate, samostalno hostujete, preuzimate eksterno znanje, pokrećete agente ili zavisite od upravljanih API-ja modela.",{},{"id":369,"data":370,"type":42,"tunes":372},"h-compare",{"text":371,"level":253},"MLOps vs LLMOps",{},{"id":374,"data":375,"type":419,"tunes":420},"main-comparison",{"rows":376,"title":410,"layout":411,"columns":412},[377,382,386,390,394,398,402,406],{"id":378,"label":379,"values":380},"unit","Primarna operativna jedinica",[381,381],"",{"id":383,"label":384,"values":385},"model","Vlasništvo nad modelom",[381,381],{"id":387,"label":388,"values":389},"change","Tipična promena",[381,381],{"id":391,"label":392,"values":393},"eval","Evaluacija",[381,381],{"id":395,"label":396,"values":397},"monitor","Praćenje u produkciji",[381,381],{"id":399,"label":400,"values":401},"training","Kontinuirano treniranje",[381,381],{"id":403,"label":404,"values":405},"registry","Verzionisani artefakti",[381,381],{"id":407,"label":408,"values":409},"rollback","Meta vraćanja",[381,381],"Šta ostaje isto, a šta se širi","table",[413,416],{"id":414,"label":415},"mlops","MLOps",{"id":417,"label":418},"llmops","LLMOps","comparison",{},{"id":422,"data":423,"type":42,"tunes":425},"h-extension",{"text":424,"level":253},"LLMOps proširuje MLOps, a ne zamenjuje ga",{},{"id":427,"data":428,"type":218,"tunes":430},"p-extension-1",{"text":429},"Osnovni operativni principi ne nestaju: kontrola izvornog koda, CI\u002FCD, reproduktivnost, poreklo, kontrole implementacije, praćenje, vraćanje i merljivi kriterijumi prihvatanja ostaju suštinski.",{},{"id":432,"data":433,"type":218,"tunes":435},"p-extension-2",{"text":434},"Proširenje je u tome što više artefakata koji definišu ponašanje sada stoji izvan težina modela. Upravljani foundation model može promeniti ponašanje kroz nadogradnje snimaka, dok se izlaz aplikacije može promeniti kroz izmene prompta ili pretrage bez ikakvog ponovnog treniranja modela.",{},{"id":437,"data":438,"type":218,"tunes":440},"p-extension-3",{"text":439},"Zato je korisna hijerarhija obično DevOps → MLOps → LLMOps\u002FGenAIOps kao sve specijalizovanije operativne brige, a ne tri međusobno isključive prakse.",{},{"id":442,"data":443,"type":42,"tunes":445},"h-artifacts",{"text":444,"level":253},"Šta se mora verzionisati u LLMOps-u?",{},{"id":447,"data":448,"type":411,"tunes":495},"artifact-table",{"content":449,"stretched":43,"withHeadings":14},[450,453,456,459,462,465,468,471,474,477,480,483,486,489,492],[451,452],"Artefakt","Zašto je važan",[454,455],"Kod aplikacije","Definiše orkestraciju, validaciju, ponovne pokušaje i poslovno ponašanje",[457,458],"Porodica modela + snimak\u002Fverzija","Različiti snimci mogu proizvesti različito ponašanje",[460,461],"Provajder \u002F endpoint","Menja tok podataka, latenciju, ograničenja, cene i dostupnost",[463,464],"Kod prompta\u002Finstrukcija","Menja ponašanje modela čak i sa istim modelom",[466,467],"Parametri generisanja\u002Fzaključivanja","Mogu promeniti determinizam, latenciju, dubinu i cenu",[469,470],"Skup podataka za evaluaciju","Definiše prema čemu se testira „dovoljno dobro“",[472,473],"Ocenjivači \u002F graderi","Definišu kako se meri kvalitet",[475,476],"Model za ugrađivanje","Menja vektorsku reprezentaciju i ponašanje pretrage",[478,479],"Konfiguracija deljenja\u002Findeksa","Menja šta se može pronaći",[481,482],"Reranker \u002F fuzija pretrage","Menja redosled rezultata",[484,485],"Šeme alata","Menjaju šta model može zahtevati i kako",[487,488],"Profil dozvola","Menja koje se akcije alata mogu stvarno izvršiti",[490,491],"Pravila sastavljanja konteksta","Menjaju koji dokazi i stanje stižu do modela",[493,494],"Konfiguracija bezbednosti\u002Fzaštitnih ograda","Menja dozvoljeno ili blokirano ponašanje",{},{"id":497,"data":498,"type":42,"tunes":500},"h-model-version",{"text":499,"level":253},"Snimci modela postaju zavisnosti izdanja",{},{"id":502,"data":503,"type":218,"tunes":505},"p-model-version-1",{"text":504},"Sa hostovanim LLM-ovima, tim možda ne kontroliše treniranje modela, ali i dalje kontroliše koji model ili snimak aplikacija poziva.",{},{"id":507,"data":508,"type":218,"tunes":510},"p-model-version-2",{"text":509},"OpenAI-jeva trenutna API smernica eksplicitno upozorava da se ponašanje prompta može promeniti između snimaka modela i preporučuje fiksiranje produkcijskih aplikacija na određene snimke gde je konzistentnost važna, a zatim pokretanje evaluacija prilikom nadogradnje.",{},{"id":512,"data":513,"type":218,"tunes":515},"p-model-version-3",{"text":514},"Operativna posledica je jasna: nadogradnje modela treba tretirati kao izdanja aplikacije, a ne kao nevidljivo održavanje infrastrukture.",{},{"id":517,"data":518,"type":42,"tunes":520},"h-provider",{"text":519,"level":253},"Životni ciklus provajdera postaje deo operacija",{},{"id":522,"data":523,"type":218,"tunes":525},"p-provider-1",{"text":524},"LLM aplikacije često zavise od ograničenja brzine provajdera, rasporeda ukidanja, API semantike, ograničenja konteksta, pravila rukovanja podacima i cena.",{},{"id":527,"data":528,"type":218,"tunes":530},"p-provider-2",{"text":529},"Provajder može ukinuti model dok vaš aplikacijski kod ostaje nepromenjen. OpenAI-jev trenutni raspored ukidanja, na primer, uključuje datume povlačenja 2026. za starije snimke modela i platforme.",{},{"id":532,"data":533,"type":218,"tunes":535},"p-provider-3",{"text":534},"LLMOps stoga zahteva praćenje životnog ciklusa provajdera, testiranje migracije i odluke o rezervnim rešenjima pored praćenja kvaliteta modela.",{},{"id":537,"data":538,"type":42,"tunes":540},"h-prompt",{"text":539,"level":253},"Promptovi se ponašaju kao produkcioni kod",{},{"id":542,"data":543,"type":218,"tunes":545},"p-prompt-1",{"text":544},"Promptovi su izvršna bihevioralna konfiguracija. Male izmene mogu promeniti kvalitet izlaza, izbor alata i tumačenje politike.",{},{"id":547,"data":548,"type":218,"tunes":550},"p-prompt-2",{"text":549},"OpenAI-jeva trenutna uputstva preporučuju čuvanje produkcionih promptova u aplikacijskom kodu, pregled izmena promptova kroz pull zahteve, korišćenje tipizovanih ulaza i pokrivanje izmena testovima i proverama evaluacije.",{},{"id":552,"data":553,"type":218,"tunes":555},"p-prompt-3",{"text":554},"To čini verzionisanje promptova manje sličnim uređivanju marketinškog teksta, a više sličnim menjanju funkcije čiji je izlaz probabilistički i zavisi od modela.",{},{"id":557,"data":558,"type":42,"tunes":560},"h-context",{"text":559,"level":253},"Inženjering konteksta postaje operativna briga",{},{"id":562,"data":563,"type":218,"tunes":565},"p-context-1",{"text":564},"Produkcioni model retko prima samo statički prompt. Može primiti istoriju razgovora, preuzete dokumente, izlaze alata, memoriju, trenutno stanje aplikacije i instrukcije politike.",{},{"id":567,"data":568,"type":218,"tunes":570},"p-context-2",{"text":569},"LLMOps stoga mora posmatrati sastavljanje konteksta: koji dokazi su izabrani, koja verzija stanja je bila aktuelna, da li je došlo do skraćivanja i da li su važne instrukcije preživele sažimanje.",{},{"id":572,"data":573,"type":218,"tunes":575},"p-context-3",{"text":574},"Regresija modela i regresija konteksta mogu izgledati identično u konačnom odgovoru. Praćenje stvarne putanje konteksta je ono što omogućava timu da ih razdvoji.",{},{"id":577,"data":578,"type":42,"tunes":580},"h-rag",{"text":579,"level":253},"RAG stvara sopstveni operativni životni ciklus",{},{"id":582,"data":583,"type":218,"tunes":585},"p-rag-1",{"text":584},"RAG sistem uvodi drugi produkcioni pipeline pored inferencije modela: unos, ekstrakcija, deljenje na delove, metapodaci, ugrađivanja, indeksi, pretraga, ponovno rangiranje i izbor konteksta.",{},{"id":587,"data":588,"type":218,"tunes":590},"p-rag-2",{"text":589},"Korpus znanja može se menjati svakog dana čak i kada se model i prompt ne menjaju. Zastareo indeks ili neispravan filter metapodataka može stoga pogoršati kvalitet odgovora bez ikakvog odstupanja modela.",{},{"id":592,"data":593,"type":218,"tunes":595},"p-rag-3",{"text":594},"LLMOps za RAG treba da prati verziju korpusa\u002Findeksa, model ugrađivanja, politiku deljenja na delove, konfiguraciju pretrage, svežinu izvora i metrike pretrage odvojeno od kvaliteta generisanja.",{},{"id":597,"data":598,"type":603,"tunes":604},"ref-rag-diagnostic",{"url":599,"title":600,"excerpt":601,"ctaLabel":602},"https:\u002F\u002Fstajic.de\u002Fsr\u002Fblog\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method","RAG nije uspeo — ali koji sloj je zapravo zakazao? Dijagnostička metoda","Produkcioni LLM pipeline zahteva odvojeno posmatranje pokrivenosti izvora, pretrage, rangiranja, sastavljanja konteksta i generisanja.","Pročitajte RAG dijagnostičku metodu","referralArticle",{},{"id":606,"data":607,"type":42,"tunes":609},"h-evals",{"text":608,"level":253},"Evaluacije zamenjuju „izgleda dobro“ dokazima o izdanju",{},{"id":611,"data":612,"type":218,"tunes":614},"p-eval-1",{"text":613},"Generativni izlazi su često otvoreni, pa su testovi tačnog poklapanja nedovoljni za mnoge zadatke. LLMOps dodaje skupove podataka za evaluaciju i ocenjivače koji mogu meriti uspeh zadatka, tačnost, bezbednost, utemeljenost, stil ili kriterijume prihvatanja specifične za domen.",{},{"id":616,"data":617,"type":218,"tunes":619},"p-eval-2",{"text":618},"MLflow-ov trenutni GenAI stack za evaluaciju podržava verzionisane skupove podataka za evaluaciju, poređenja promptova\u002Fmodela, prilagođene ocenjivače i evaluaciju nad kompletnim tragovima.",{},{"id":621,"data":622,"type":218,"tunes":624},"p-eval-3",{"text":623},"Najjača praksa je razvoj vođen evaluacijom: definišite reprezentativne slučajeve i kriterijume prihvatanja pre ili uporedo sa izmenama, a zatim uporedite izdanja sa istim dokazima.",{},{"id":626,"data":627,"type":226,"tunes":631},"eval-rule",{"body":628,"title":629,"variant":630},"Implementacija ne treba da se smatra ekvivalentnom samo zato što API ugovor i dalje funkcioniše. Ako su se prompt, model, pretraga ili alati promenili, skup regresionih testova ponašanja treba ponovo pokrenuti.","Promene u ponašanju zahtevaju testove ponašanja","success",{},{"id":633,"data":634,"type":42,"tunes":636},"h-judges",{"text":635,"level":253},"LLM kao sudija je koristan, ali nije osnovna istina",{},{"id":638,"data":639,"type":218,"tunes":641},"p-judge-1",{"text":640},"LLM sudije mogu da skaliraju evaluaciju za kvalitete koje je skupo kodirati kao determinističke tvrdnje, kao što su relevantnost, ton ili utemeljenost.",{},{"id":643,"data":644,"type":218,"tunes":646},"p-judge-2",{"text":645},"Međutim, sudija je drugi model sa sopstvenom pristrasnošću, verzijom i promptom. Konfiguracija sudije stoga treba da bude verzionisana i kalibrisana prema ljudskim ili determinističkim referentnim slučajevima tamo gde su posledice važne.",{},{"id":648,"data":649,"type":218,"tunes":651},"p-judge-3",{"text":650},"Produkciona evaluacija može da kombinuje determinističke provere, metrike zasnovane na referencama, model sudije i ljudski pregled, umesto da zahteva da jedna metrika predstavlja svaku dimenziju kvaliteta.",{},{"id":653,"data":654,"type":42,"tunes":656},"h-tracing",{"text":655,"level":253},"Praćenje postaje važnije od logova krajnjih tačaka",{},{"id":658,"data":659,"type":218,"tunes":661},"p-trace-1",{"text":660},"Tradicionalni API logovi mogu da vam kažu da je zahtev trajao dve sekunde i vratio HTTP 200. Ne mogu da vam kažu koji su izvučeni delovi izabrani, koji je alat agent pozvao ili koji je model segment potrošio najviše tokena.",{},{"id":663,"data":664,"type":218,"tunes":666},"p-trace-2",{"text":665},"MLflow-ovo trenutno GenAI praćenje beleži promptove, pretrage, pozive alata i segmente aplikacije, a njegov tok produkcione evaluacije može da ocenjuje informacije o srednjoj putanji, a ne samo konačni tekst.",{},{"id":668,"data":669,"type":218,"tunes":671},"p-trace-3",{"text":670},"Ovo je veliki pomak u LLMOps-u: observabilnost prati graf ponašanja aplikacije, a ne samo servisnu krajnju tačku.",{},{"id":673,"data":674,"type":42,"tunes":676},"h-agent",{"text":675,"level":253},"Agenti proširuju LLMOps na operacije u vreme izvršavanja",{},{"id":678,"data":679,"type":218,"tunes":681},"p-agent-1",{"text":680},"Agentna aplikacija može da izvrši nekoliko poziva modela, poziva alata i prelaza stanja pre nego što proizvede rezultat.",{},{"id":683,"data":684,"type":218,"tunes":686},"p-agent-2",{"text":685},"Operativni agenti stoga zahtevaju brojanje koraka, tragove poziva alata, odbijanja dozvola, ponovne pokušaje, detekciju petlji, ljudska odobrenja i verifikovano konačno stanje pored uobičajenih metrika latencije modela i tokena.",{},{"id":688,"data":689,"type":218,"tunes":691},"p-agent-3",{"text":690},"Tačan konačni odgovor može da sakrije lošu putanju, pa evaluacija agenta mora da pregleda i putanju i rezultat.",{},{"id":693,"data":694,"type":603,"tunes":699},"ref-agent-reliability",{"url":695,"title":696,"excerpt":697,"ctaLabel":698},"https:\u002F\u002Fstajic.de\u002Fsr\u002Fblog\u002Fai-agent-reliability-why-the-final-answer-is-not-enough","Pouzdanost AI agenata: Zašto konačni odgovor nije dovoljan","Zašto produkciona evaluacija agenata mora da uključi pozive alata, prelaze stanja, odobrenja i mogućnost oporavka.","Pročitajte članak o pouzdanosti agenata",{},{"id":701,"data":702,"type":42,"tunes":704},"h-cost",{"text":703,"level":253},"Tokeni, pozivi modela i kontekst postaju varijable troškova",{},{"id":706,"data":707,"type":218,"tunes":709},"p-cost-1",{"text":708},"Trošak klasičnog ML zaključivanja često je dominiran infrastrukturom za serviranje ili računanjem po predikciji. LLM aplikacije mogu da dodaju cene tokena provajdera, ponovljene pozive agenata, pozive za ugrađivanje, ponovno rangiranje i troškove alata\u002Fvremena izvršavanja.",{},{"id":711,"data":712,"type":218,"tunes":714},"p-cost-2",{"text":713},"Trošak se stoga mora pripisati zadatku ili tragu, a ne samo jednom endpointu. Radni tok koji pravi osam skrivenih poziva modela može biti funkcionalno ispravan, ali operativno neprihvatljiv.",{},{"id":716,"data":717,"type":218,"tunes":719},"p-cost-3",{"text":718},"Latencija se ponaša na isti način: latencija modela, pretraga, rerangiranje i eksterni alati se kombinuju u end-to-end latenciju za korisnika.",{},{"id":721,"data":722,"type":42,"tunes":724},"h-cache",{"text":723,"level":253},"Keširanje postaje semantičko, ne samo tehničko",{},{"id":726,"data":727,"type":218,"tunes":729},"p-cache-1",{"text":728},"LLM sistemi mogu keširati promptove, embeddinge, rezultate pretrage ili pune odgovore, ali ključ keša mora odražavati semantiku koja može promeniti rezultat.",{},{"id":731,"data":732,"type":218,"tunes":734},"p-cache-2",{"text":733},"Keš odgovora koji ignoriše verziju modela, tenanta, dozvole ili svežinu izvora može vratiti tehnički validan, ali semantički nevalidan odgovor.",{},{"id":736,"data":737,"type":218,"tunes":739},"p-cache-3",{"text":738},"LLMOps stoga tretira invalidaciju keša kao deo verzionisanja modela\u002Fkonteksta\u002Fpodataka, a ne samo kao infrastrukturnu optimizaciju.",{},{"id":741,"data":742,"type":42,"tunes":744},"h-safety",{"text":743,"level":253},"Bezbednost i dozvole postaju kriterijumi za izdanje",{},{"id":746,"data":747,"type":218,"tunes":749},"p-safety-1",{"text":748},"Generativni sistemi mogu proizvesti neograničen tekst, a agenti mogu pokrenuti eksterne akcije. Testiranje bezbednosti stoga je bliže običnom CI\u002FCD-u nego u mnogim klasičnim sistemima prediktivnog ML-a.",{},{"id":751,"data":752,"type":218,"tunes":754},"p-safety-2",{"text":753},"Provere dozvola, testovi prompt-injection-a, testovi izolacije tenanta i odobrenja za sporedne efekte treba da budu reproduktivni regresioni testovi tamo gde ti rizici postoje.",{},{"id":756,"data":757,"type":218,"tunes":759},"p-safety-3",{"text":758},"Model može predložiti operaciju, ali runtime i dalje mora sprovesti autorizaciju. LLMOps poseduje dokaze da ti kontroli nastavljaju da rade nakon promena modela, prompta ili alata.",{},{"id":761,"data":762,"type":42,"tunes":764},"h-ci",{"text":763,"level":253},"Kako CI izgleda u LLMOps-u",{},{"id":766,"data":767,"type":411,"tunes":799},"ci-table",{"content":768,"stretched":43,"withHeadings":14},[769,772,775,778,781,784,787,790,793,796],[770,771],"CI sloj","Primeri provera",[773,774],"Kod","Unit testovi, provere tipova, validacija šeme",[776,777],"Promptovi","Renderovanje šablona, obavezne promenljive, tekst politike, pregled snimaka",[779,780],"Modeli\u002Fprovajderi","Kompatibilnost, izlazna šema, testovi sposobnosti i regresije",[782,783],"RAG","Fiksture za chunking, testovi filtera, Recall@k, regresija rerangera",[785,786],"Alati","Testovi ulazno\u002Fizlazne šeme, testovi dozvola, testovi idempotentnosti",[788,789],"Agenti","Fiksture trajektorije, ograničenja petlje, testovi predaje\u002Fizbora alata",[791,792],"Bezbednost","Prompt injection, neovlašćeni alati, negativni testovi između tenanata",[794,795],"Bihevioralne evaluacije","Uspeh zadatka, tačnost, utemeljenost, bezbednost, domen-specifični kriterijumi",[797,798],"Operativno","Latencija, budžeti za tokene\u002Ftroškove, ponašanje pri timeout-u\u002Ffallback-u",{},{"id":801,"data":802,"type":42,"tunes":804},"h-cd",{"text":803,"level":253},"Kako CD izgleda u LLMOps-u",{},{"id":806,"data":807,"type":218,"tunes":809},"p-cd-1",{"text":808},"Produkcijsko izdanje možda uopšte neće deploy-ovati novi artefakt modela. Može jednostavno isporučiti novi prompt, konfiguraciju pretrage, skup alata ili mapiranje provajdera.",{},{"id":811,"data":812,"type":218,"tunes":814},"p-cd-2",{"text":813},"Paket izdanja stoga treba da identifikuje kompletnu konfiguraciju koja definiše ponašanje, a ne samo sliku kontejnera aplikacije.",{},{"id":816,"data":817,"type":218,"tunes":819},"p-cd-3",{"text":818},"Feature flag-ovi, postepeno uvođenje, shadow evaluacija, canary saobraćaj i rollback su korisni jer LLM ponašanje može regresirati na načine koje statički testovi ugovora ne detektuju.",{},{"id":821,"data":822,"type":42,"tunes":824},"h-ct",{"text":823,"level":253},"Kontinuirano treniranje postaje opciono; kontinuirana evaluacija postaje centralna",{},{"id":826,"data":827,"type":218,"tunes":829},"p-ct-1",{"text":828},"Tradicionalni MLOps često naglašava kontinuirano treniranje kada novi podaci ili drift opravdavaju ponovno treniranje.",{},{"id":831,"data":832,"type":218,"tunes":834},"p-ct-2",{"text":833},"Mnoge LLM aplikacije nikada ne treniraju osnovni model. Njihova ekvivalentna neprekidna petlja je kontinuirana evaluacija: prikupljaju neuspehe i reprezentativne produkcijske slučajeve, dodaju ih u skupove podataka za evaluaciju, testiraju izmene kandidata za prompt\u002Fmodel\u002Fpretragu i ponovo objavljuju samo kada se dokazi poboljšaju.",{},{"id":836,"data":837,"type":218,"tunes":839},"p-ct-3",{"text":838},"Fino podešavanje može ponovo uvesti životni ciklus treniranja, ali treba da bude deo istog šireg procesa evaluacije i objavljivanja.",{},{"id":841,"data":842,"type":42,"tunes":844},"h-monitor",{"text":843,"level":253},"Šta treba pratiti u produkciji?",{},{"id":846,"data":847,"type":411,"tunes":879},"monitor-table",{"content":848,"stretched":43,"withHeadings":14},[849,852,855,858,861,864,867,869,871,873,876],[850,851],"Klasa signala","Primeri",[853,854],"Zdravlje sistema","Greške, isteci vremena, dostupnost endpointa",[856,857],"Model\u002Fprovider","ID modela, snimak, ograničenja brzine, greške provajdera",[859,860],"Latencija","Krajnja do krajnje, model, pretraga, alat i reranker opsezi",[862,863],"Trošak","Ulazni\u002Fizlazni tokeni, embeddingzi, troškovi alata\u002FAPI-ja",[865,866],"Kvalitet","Uzorkovani uspeh zadatka, tačnost, relevantnost, utemeljenost",[782,868],"Proksiji za obuhvat pretrage, prazna pretraga, zastareli izvori, pokrivenost citata",[788,870],"Izbor alata, ponovni pokušaji, petlje, predaje, učestalost odobrenja",[791,872],"Odbijene radnje, indikatori prompt-injection napada, neuspesi na granicama tenanta",[874,875],"Povratne informacije korisnika","Ispravke, napuštanje, eskalacija, eksplicitne ocene",[877,878],"Drift promena","Promene provajdera\u002Fmodela\u002Fkonfiguracije u odnosu na odobreno izdanje",{},{"id":881,"data":882,"type":42,"tunes":884},"h-prod-eval",{"text":883,"level":253},"Produkcijski tragovi mogu postati podaci za evaluaciju",{},{"id":886,"data":887,"type":218,"tunes":889},"p-prod-1",{"text":888},"Jedan od najkorisnijih modernih LLMOps obrazaca je pretvaranje uzorkovanih produkcijskih tragova u zapise za evaluaciju.",{},{"id":891,"data":892,"type":218,"tunes":894},"p-prod-2",{"text":893},"MLflow trenutno podržava preuzimanje produkcijskih tragova i ocenjivanje ne samo izlaza već i međukoraka kao što su pretraga ili putanje poziva alata.",{},{"id":896,"data":897,"type":218,"tunes":899},"p-prod-3",{"text":898},"Ovo zatvara petlju između observabilnosti i razvoja: stvarni neuspesi mogu postati regresioni slučajevi u sledećem izdanju umesto da nestanu u logovima.",{},{"id":901,"data":902,"type":42,"tunes":904},"h-repro",{"text":903,"level":253},"Reproduktivnost postaje uslovna, a ne egzaktna",{},{"id":906,"data":907,"type":218,"tunes":909},"p-repro-1",{"text":908},"Klasična ML reproduktivnost često ima za cilj da ponovo stvori model iz verzionisanog koda, podataka, okruženja i parametara treniranja.",{},{"id":911,"data":912,"type":218,"tunes":914},"p-repro-2",{"text":913},"Hostovane LLM aplikacije ne mogu uvek da reprodukuju identičan izlaz token po token jer je generisanje probabilističko, a provajderi mogu kontrolisati infrastrukturu.",{},{"id":916,"data":917,"type":218,"tunes":919},"p-repro-3",{"text":918},"LLMOps stoga teži bihevioralnoj reproduktivnosti: beleži dovoljno modela\u002Fprovajdera\u002Fverzije, prompta, kontekstualnih ulaza, stanja pretrage i konfiguracije izvršavanja da bi se reprodukovali uslovi i validiralo ponašanje u okviru očekivanih tolerancija.",{},{"id":921,"data":922,"type":42,"tunes":924},"h-lineage",{"text":923,"level":253},"Loza se širi od loze modela do loze aplikacije",{},{"id":926,"data":927,"type":218,"tunes":929},"p-lineage-1",{"text":928},"AWS-ove MLOps smernice tretiraju lozu modela kao istoriju artefakata koda, podataka, modela i infrastrukture potrebnih za dijagnostiku i reproduktivnost.",{},{"id":931,"data":932,"type":218,"tunes":934},"p-lineage-2",{"text":933},"Za LLM aplikacije, loza treba dodatno da poveže promptove, skupove podataka za evaluaciju, verzije pretrage\u002Findeksa, šeme alata, konfiguraciju agenta\u002Fvremena izvršavanja i snimke provajdera\u002Fmodela.",{},{"id":936,"data":937,"type":218,"tunes":939},"p-lineage-3",{"text":938},"Ciljno pitanje postaje: Koja tačno konfiguracija aplikacije je proizvela ovaj trag?",{},{"id":941,"data":942,"type":42,"tunes":944},"h-routing",{"text":943,"level":253},"Usmeravanje ka više provajdera i modela stvara operativnu politiku",{},{"id":946,"data":947,"type":218,"tunes":949},"p-route-1",{"text":948},"Kada aplikacija može da koristi nekoliko provajdera ili lokalnih modela, usmeravanje postaje operativna politika, a ne jednostavan string modela.",{},{"id":951,"data":952,"type":218,"tunes":954},"p-route-2",{"text":953},"Rutiranje može zavisiti od sposobnosti, latencije, cene, privatnosti, dužine konteksta, dostupnosti, podrške za alate ili lokaliteta. Rezervna opcija može očuvati vreme rada dok menja kvalitet odgovora ili pretpostavke obrade podataka.",{},{"id":956,"data":957,"type":218,"tunes":959},"p-route-3",{"text":958},"LLMOps bi stoga trebalo da beleži koja je ruta zaista izabrana i da nezavisno procenjuje rute, umesto da svaku kompatibilnu krajnju tačku tretira kao bihevioralno zamenljivu.",{},{"id":961,"data":962,"type":42,"tunes":964},"h-implementation",{"text":963,"level":253},"Dokazi iz originalne implementacije",{},{"id":966,"data":967,"type":42,"tunes":969},"h-client",{"text":968,"level":252},"Aaasaasa AI Client: provajder, model i runtime su odvojeni operativni objekti",{},{"id":971,"data":972,"type":218,"tunes":974},"p-client-1",{"text":973},"Aaasaasa AI Client razdvaja agenta\u002Fklijenta, provajdera, model, lokaciju runtime-a i dozvole. Njegov AI Hub podržava Ollama, LM Studio\u002FOpenAI-kompatibilne krajnje tačke i druge protokole provajdera, umesto da „model“ tretira kao jedno globalno podešavanje.",{},{"id":976,"data":977,"type":218,"tunes":979},"p-client-2",{"text":978},"Implementacija uključuje dinamičko otkrivanje lokalnih modela, strimovanje, izlaz razmišljanja i eksplicitne Ollama kontrole zagrevanja\u002Fučitavanja i istovara. To je operativni dokaz da lokalno serviranje LLM-a uvodi brige o životnom ciklusu resursa koje prevazilaze ime API modela.",{},{"id":981,"data":982,"type":218,"tunes":984},"p-client-3",{"text":983},"Status provajdera se dobija preko adaptera provajdera, a tipovi veze razlikuju lokalne, cloud API, putanje vezane za nalog, udaljene agente i web klijente. To su konkretne operativne dimenzije koje platforma svesna LLM-a mora da izloži.",{},{"id":986,"data":987,"type":218,"tunes":989},"p-client-4",{"text":988},"Repozitorijum takođe čuva važnu granicu: lokalni runtime nije automatski lokalna inferencija. Lokacija provajdera\u002Fmodela\u002Fruntime-a su pitanja koja se verzioniraju ili konfigurišu i koja utiču na privatnost, latenciju, cenu i dostupnost.",{},{"id":991,"data":992,"type":42,"tunes":994},"h-sot",{"text":993,"level":252},"Source of Truth Research Engine: stanje LLM aplikacije se proteže izvan modela",{},{"id":996,"data":997,"type":218,"tunes":999},"p-sot-1",{"text":998},"Source of Truth Research Engine kombinuje leksičku pretragu, opcione embedding-e, snimke izvora, SHA-256 identitet, tvrdnje, poreklo i praćenje kontradikcija oko istraživanja uz pomoć lokalnog modela.",{},{"id":1001,"data":1002,"type":218,"tunes":1004},"p-sot-2",{"text":1003},"Ovo je koristan LLMOps dokaz jer samo promena modela ne definiše istraživački sistem. Pretraga, pribavljanje izvora, klasifikacija dokaza i trajno poreklo su nezavisni operativni artefakti.",{},{"id":1006,"data":1007,"type":218,"tunes":1009},"p-sot-3",{"text":1008},"Implementacija namerno tretira semantičku sličnost kao otkrivanje, a ne kao dokaz, pokazujući zašto LLMOps observabilnost treba da razlikuje ponašanje pretrage od validnosti tvrdnji.",{},{"id":1011,"data":1012,"type":411,"tunes":1041},"impl-table",{"content":1013,"stretched":43,"withHeadings":14},[1014,1017,1020,1023,1026,1029,1032,1035,1038],[1015,1016],"Uočena implementacija","LLMOps lekcija",[1018,1019],"Više protokola provajdera","Identitet provajdera je operativna zavisnost",[1021,1022],"Dinamičko otkrivanje modela","Dostupni modeli se mogu menjati nezavisno od koda aplikacije",[1024,1025],"Ollama kontrole učitavanja\u002Fistovara","Lokalni modeli imaju životni ciklus memorije\u002Fresursa",[1027,1028],"Adapteri za zdravlje\u002Fstatus provajdera","Dostupnost modela zahteva runtime observabilnost",[1030,1031],"Odvojena lokacija runtime-a i inferencije","Topologija postavljanja nije jedna bulova vrednost „lokalno\u002Fcloud“",[1033,1034],"Centralne dozvole","Sposobnost modela i ovlašćenje alata moraju ostati odvojeni",[1036,1037],"Leksički + semantički pipeline pretrage","Konfiguracija pretrage je deo ponašanja aplikacije",[1039,1040],"Trajnost izvora\u002Fporekla","Operativno stanje i dokazi žive izvan težina modela",{},{"id":1043,"data":1044,"type":226,"tunes":1047},"impl-boundary",{"body":1045,"title":1046,"variant":240},"Ovi projekti demonstriraju operacije sa više provajdera\u002Flokalnih modela, razdvajanje dozvola, infrastrukturu pretrage i trajnost dokaza. Nisu predstavljeni kao kompletna komercijalna LLMOps platforma niti kao dokaz velikog obima produkcionog saobraćaja.","Granica dokaza",{},{"id":1049,"data":1050,"type":42,"tunes":1052},"h-failures",{"text":1051,"level":253},"Uobičajeni režimi neuspeha u LLMOps-u",{},{"id":1054,"data":1055,"type":411,"tunes":1096},"failure-table",{"content":1056,"stretched":43,"withHeadings":14},[1057,1060,1063,1066,1069,1072,1075,1078,1081,1084,1087,1090,1093],[1058,1059],"Režim neuspeha","Šta je zapravo pošlo naopako",[1061,1062],"Alias modela tiho nadograđen","Ponašanje se promenilo bez kontrolisanog izdanja",[1064,1065],"Prompt promenjen bez evaluacija","Regresija ponašanja prošla je normalne unit testove",[1067,1068],"RAG indeks zastareo","Model generisanja je okrivljen za neuspeh pretrage\u002Fpodataka",[1070,1071],"Beleži se samo konačni odgovor","Osnovni uzrok u putanji pretrage\u002Falata\u002Fkonteksta je nevidljiv",[1073,1074],"Rezervna opcija provajdera je tiha","Drugačiji model\u002Fputanja podataka menja ponašanje bez pripisivanja",[1076,1077],"Trošak tokena se prati globalno","Skupi tokovi rada ne mogu se lokalizovati",[1079,1080],"Model procenjivač promenjen","Ocene evaluacije driftuju bez promene aplikacije",[1082,1083],"Produkcioni tragovi nikada ne postaju testovi","Poznati neuspesi se ponovo vraćaju",[1085,1086],"Lokalni model ostaje učitan neograničeno","Pritisak na VRAM\u002Fresurse postaje operativna nestabilnost",[1088,1089],"Dozvole kodirane samo u promptu","Ponašanje modela se pogrešno smatra autorizacijom",[1091,1092],"Jedna ocena evaluacije određuje sve","Različite dimenzije kvaliteta se svode na obmanjujući broj",[1094,1095],"Registar modela postoji, ali verzije prompta\u002Findeksa ne","Rodoslov aplikacije ostaje nepotpun",{},{"id":1098,"data":1099,"type":42,"tunes":1101},"h-misconceptions",{"text":1100,"level":253},"Uobičajene zablude",{},{"id":1103,"data":1104,"type":411,"tunes":1139},"misconceptions-table",{"content":1105,"stretched":43,"withHeadings":14},[1106,1109,1112,1115,1118,1121,1124,1127,1130,1133,1136],[1107,1108],"Zabluda","Ispravka",[1110,1111],"„LLMOps zamenjuje MLOps.“","LLMOps proširuje MLOps principe na ponašanje aplikacije specifično za LLM.",[1113,1114],"„LLMOps je prompt inženjering.“","Promptovi su jedan artefakt među modelima, provajderima, kontekstom, pretragom, alatima, evaluacijama i runtime-om.",[1116,1117],"„Hostovani API-ji uklanjaju operativni rad.“","Oni uklanjaju deo rada na serviranju\u002Ftreniranju modela, ali dodaju upravljanje životnim ciklusom provajdera, verzijama i zavisnostima.",[1119,1120],"„Ako je API stabilan, aplikacija je stabilna.“","Ponašanje modela i snimci provajdera\u002Fmodela mogu se menjati nezavisno od API šeme.",[1122,1123],"„RAG je samo predobrada podataka.“","U produkciji ima sopstveni životni ciklus unosa, indeksa, pretrage i svežine.",[1125,1126],"„LLM izlazi ne mogu se testirati.“","Mogu se evaluirati determinističkim, referentnim, procenjivačkim i ljudskim kriterijumima.",[1128,1129],"„LLM procenjivači su objektivna osnovna istina.“","Oni su evaluatori zasnovani na modelu koji takođe zahtevaju kalibraciju i kontrolu verzija.",[1131,1132],"„Lokalni model eliminiše LLMOps.“","Lokalno serviranje dodaje fajlove modela, VRAM, učitavanje\u002Fistovar, zdravlje runtime-a i brige o nadogradnji.",[1134,1135],"„Observabilnost znači brojanje tokena.“","Korisna observabilnost prati promptove, pretrage, alate, raspone modela i ishode.",[1137,1138],"„Kontinuirano treniranje je obavezno.“","Mnoge LLM aplikacije koriste kontinuiranu evaluaciju bez treniranja osnovnog modela.",{},{"id":1141,"data":1142,"type":42,"tunes":1144},"h-design",{"text":1143,"level":253},"Praktičan redosled dizajna LLMOps-a",{},{"id":1146,"data":1147,"type":346,"tunes":1186},"design-flow",{"steps":1148,"title":1185,"orientation":345},[1149,1152,1155,1158,1161,1164,1167,1170,1173,1176,1179,1182],{"label":1150,"description":1151},"1. Definišite jedinicu ponašanja","Navedite svaku komponentu koja može materijalno da promeni izlaz: model, uputstvo, pretragu, alate, kontekst i politiku.",{"label":1153,"description":1154},"2. Uspostavite lozu aplikacije","Verzionirajte kod, model\u002Fprovajdera, uputstva, skupove podataka za evaluaciju, konfiguraciju pretrage i ugovore alata.",{"label":1156,"description":1157},"3. Izgradite reprezentativne skupove podataka za evaluaciju","Koristite očekivane slučajeve uspeha\u002Fneuspeha iz dizajna i produkcije.",{"label":1159,"description":1160},"4. Razdvojite determinističke i bihevioralne testove","Držite provere šeme\u002Fbezbednosti odvojeno od semantičke evaluacije izlaza.",{"label":1162,"description":1163},"5. Pratite izvršavanje od početka do kraja","Instrumentišite model, pretragu, rangiranje, alate i opsege agenta\u002Fruntime-a.",{"label":1165,"description":1166},"6. Definišite kapije za izdanja","Postavite pragove kvaliteta, bezbednosti, latencije i troškova.",{"label":1168,"description":1169},"7. Fiksirajte ili eksplicitno zabeležite verzije modela","Tretirajte promene modela\u002Fprovajdera kao događaje izdanja.",{"label":1171,"description":1172},"8. Rasporedite postepeno","Koristite zastavice, kanarinac ili fazno uvođenje gde posledice to opravdavaju.",{"label":1174,"description":1175},"9. Evaluirajte produkcijske tragove","Izmerite stvarno ponašanje zadatka i identifikujte ponavljajuće se neuspehe.",{"label":1177,"description":1178},"10. Vratite neuspehe nazad u skupove podataka za evaluaciju","Pretvorite incidente i ispravke u trajnu pokrivenost regresije.",{"label":1180,"description":1181},"11. Nadgledajte životne cikluse provajdera i podataka","Pratite ukidanja, svežinu indeksa, promene izvora i dostupnost runtime-a.",{"label":1183,"description":1184},"12. Uklonite zastarele verzije čisto","Uklonite stare uputstva\u002Fmodele\u002Findekse\u002Fakreditive nakon migracije i odluka o zadržavanju dokaza.","Upravljajte kompletnim sistemom koji proizvodi ponašanje",{},{"id":1188,"data":1189,"type":42,"tunes":1191},"h-checklist",{"text":1190,"level":253},"Kontrolna lista arhitekture LLMOps-a",{},{"id":1193,"data":1194,"type":411,"tunes":1241},"checklist-table",{"content":1195,"stretched":43,"withHeadings":14},[1196,1199,1202,1205,1208,1211,1214,1217,1220,1223,1226,1229,1232,1235,1238],[1197,1198],"Pitanje","Očekivani dokaz",[1200,1201],"Koji model\u002Fprovajder\u002Fverzija je poslužio zahtev?","Identitet modela koji se može pratiti",[1203,1204],"Koje uputstvo\u002Finstrukcije su bile aktivne?","Verzionirani kod\u002Fkonfiguracija aplikacije",[1206,1207],"Koji kontekst je stigao do modela?","Trag konteksta\u002Fpretrage",[1209,1210],"Koja verzija korpusa\u002Findeksa je korišćena?","Loza pretrage",[1212,1213],"Koji alati su bili dostupni i pozvani?","Šema alata + trag trajektorije",[1215,1216],"Koje dozvole su primenjene?","Zapis o autorizaciji u runtime-u",[1218,1219],"Kako se meri kvalitet?","Verzionirani skup podataka za evaluaciju + ocenjivači",[1221,1222],"Kako se testiraju nadogradnje modela?","Bihevioralni paket regresionih testova",[1224,1225],"Kako se uzorkuje kvalitet u produkciji?","Proces evaluacije tragova\u002Fpovratnih informacija",[1227,1228],"Može li se jedan neuspeh približno reprodukovati?","Loza modela\u002Fkonteksta\u002Fprovajdera\u002Faplikacije",[1230,1231],"Gde se troše troškovi?","Atribucija modela\u002Falata\u002Fpretrage po tragu",[1233,1234],"Šta pokreće vraćanje na prethodno stanje?","Definisani prag kvaliteta\u002Fbezbednosti\u002Ftroškova\u002Fdostupnosti",[1236,1237],"Kako se rukuje ukidanjem provajdera?","Proces migracije\u002Frezervnog rešenja",[1239,1240],"Kako se upravlja lokalnim modelima?","Kontrole zdravlja, resursa, učitavanja\u002Fistovara i verzija",{},{"id":1243,"data":1244,"type":42,"tunes":1246},"h-edge",{"text":1245,"level":253},"Rubni slučajevi i ograničenja",{},{"id":1248,"data":1249,"type":218,"tunes":1251},"p-edge-1",{"text":1250},"Jednostavna aplikacija koja poziva jedan fiksni hostovani model bez pretrage ili alata može zahtevati samo lagani LLMOps: verzionisani kod uputstava, evaluacije, fiksiranje modela, osnovno praćenje i nadgledanje provajdera.",{},{"id":1253,"data":1254,"type":218,"tunes":1256},"p-edge-2",{"text":1255},"Samostalno hostovani fino podešeni model može zahtevati skoro pun klasični MLOps stek plus LLM-specifičnu evaluaciju aplikacije, čineći granicu između MLOps-a i LLMOps-a namerno zamućenom.",{},{"id":1258,"data":1259,"type":218,"tunes":1261},"p-edge-3",{"text":1260},"Platforma agenata može imati minimalne operacije obuke modela, ali značajne runtime operacije jer se neuspesi javljaju u izboru alata, stanju i orkestraciji.",{},{"id":1263,"data":1264,"type":218,"tunes":1266},"p-edge-4",{"text":1265},"Sistem sa puno RAG-a može biti operativno dominiran unosom dokumenata i kvalitetom pretrage, a ne posluživanjem modela.",{},{"id":1268,"data":1269,"type":218,"tunes":1271},"p-edge-5",{"text":1270},"Terminologija će nastaviti da se razvija. Trajno arhitektonsko pitanje nije koja oznaka „Ops“ pobeđuje, već koji artefakti proizvode ponašanje i stoga moraju biti verzionisani, evaluirani, posmatrani i upravljani.",{},{"id":1273,"data":1274,"type":42,"tunes":1276},"h-change",{"text":1275,"level":253},"Šta bi promenilo ovaj odgovor?",{},{"id":1278,"data":1279,"type":218,"tunes":1281},"p-change-1",{"text":1280},"Ako provajderi osnovnih modela standardizuju savršeno stabilno ponašanje modela i dugoročnu podršku verzijama, upravljanje provajderima\u002Fsnimcima moglo bi postati operativno manje značajno.",{},{"id":1283,"data":1284,"type":218,"tunes":1286},"p-change-2",{"text":1285},"Ako aplikacije sve više preuzimaju fino podešavanje ili obuku, klasične MLOps brige ponovo postaju centralne.",{},{"id":1288,"data":1289,"type":218,"tunes":1291},"p-change-3",{"text":1290},"Operativni princip bi ostao: svaka komponenta koja može materijalno da promeni produkcijsko ponašanje pripada lozi, testiranju, observabilnosti i kontroli promena.",{},{"id":1293,"data":1294,"type":42,"tunes":1296},"h-related",{"text":1295,"level":253},"Povezano kanonsko znanje",{},{"id":1298,"data":1299,"type":218,"tunes":1301},"p-related-1",{"text":1300},"LLMOps se nalazi ispod AI upravljanja i arhitekture preduzeća AI: upravljanje definiše koje promene zahtevaju dokaz i odobrenje, dok LLMOps pruža operativnu mašineriju za verzionisanje, evaluaciju, raspoređivanje i posmatranje tih promena.",{},{"id":1303,"data":1304,"type":218,"tunes":1306},"p-related-2",{"text":1305},"Inženjering konteksta i RAG su operativni poddomeni unutar mnogih LLM aplikacija jer kontekst i pretraga mogu promeniti ponašanje nezavisno od modela.",{},{"id":1308,"data":1309,"type":218,"tunes":1311},"p-related-3",{"text":1310},"Agentni AI proširuje LLMOps dalje u operacije trajektorije, dozvola i runtime-a alata.",{},{"id":1313,"data":1314,"type":603,"tunes":1319},"ref-memory",{"url":1315,"title":1316,"excerpt":1317,"ctaLabel":1318},"https:\u002F\u002Fstajic.de\u002Fsr\u002Fblog\u002Fai-agent-memory-is-not-rag-how-to-separate-memory-retrieval-state-and-context","Memorija AI agenata nije RAG: Kako razdvojiti memoriju, pretragu, stanje i kontekst","Operativna pouzdanost se poboljšava kada memorija, pretraga, stanje aplikacije i kontekst modela ostanu odvojeni objekti životnog ciklusa.","Pročitajte članak o arhitekturi",{},{"id":1321,"data":1322,"type":603,"tunes":1327},"ref-avb",{"url":1323,"title":1324,"excerpt":1325,"ctaLabel":1326},"https:\u002F\u002Fstajic.de\u002Fsr\u002Fblog\u002Fthe-answer-validity-boundary-the-missing-layer-between-relevance-and-reliable-ai-answers","Granica validnosti odgovora: Nedostajući sloj između relevantnosti i pouzdanih AI odgovora","LLMOps evaluacija treba da očuva verziju, obim i uslove dokaza pod kojima odgovor ostaje potkrepljen.","Pročitajte o granici validnosti odgovora",{},{"id":1329,"data":1330,"type":42,"tunes":1332},"h-faq",{"text":1331,"level":253},"Često postavljana pitanja",{},{"id":1334,"data":1335,"type":1334,"tunes":1374},"faq",{"items":1336,"title":1373},[1337,1341,1345,1349,1353,1357,1361,1365,1369],{"id":1338,"answer":1339,"question":1340},"faq1","MLOps upravlja sistemima mašinskog učenja kroz podatke, obuku, implementaciju i nadzor. LLMOps proširuje te prakse na LLM aplikacije gde promptovi, kontekst, pretraga, provajderi, alati i evaluacije takođe materijalno utiču na ponašanje.","Koja je razlika između MLOps i LLMOps?",{"id":1342,"answer":1343,"question":1344},"faq2","Ne. LLMOps ponovo koristi MLOps discipline kao što su CI\u002FCD, poreklo, evaluacija, implementacija i nadzor i dodaje operativne brige specifične za LLM.","Da li LLMOps zamenjuje MLOps?",{"id":1346,"answer":1347,"question":1348},"faq3","Ne nužno. Mnoge koriste eksterne temeljne modele i umesto toga se oslanjaju na kontinuiranu evaluaciju promptova, modela, pretrage i ponašanja aplikacije. Fino podešeni ili samostalno obučeni sistemi i dalje mogu zahtevati pipeline za obuku.","Da li LLM aplikacije zahtevaju kontinuiranu obuku?",{"id":1350,"answer":1351,"question":1352},"faq4","Generativni izlazi su otvoreni i ponašanje modela može se promeniti kroz promptove, snimke i kontekst. Evaluacije pružaju ponovljive dokaze da izdanje i dalje zadovoljava definisane kriterijume kvaliteta i bezbednosti.","Zašto su evaluacije toliko važne u LLMOps?",{"id":1354,"answer":1355,"question":1356},"faq5","Najmanje: kod aplikacije, model\u002Fprovajder\u002Fverziju, promptove, skupove podataka za evaluaciju\u002Focenjivače, konfiguraciju pretrage\u002Findekse, šeme alata, pravila konteksta i relevantnu konfiguraciju bezbednosti\u002Fdozvola.","Šta treba verzionisati u LLMOps?",{"id":1358,"answer":1359,"question":1360},"faq6","Ne. Isti prompt može se ponašati drugačije sa drugim modelom, skupom pretrage, redosledom konteksta, površinom alata ili provajderom.","Da li je verzionisanje promptova dovoljno?",{"id":1362,"answer":1363,"question":1364},"faq7","GenAIOps je još jedan industrijski termin za upravljanje generativnim AI aplikacijama. Neki prodavci ga koriste naizmenično ili kao širu oznaku od LLMOps.","Šta je GenAIOps?",{"id":1366,"answer":1367,"question":1368},"faq8","Nadzirite tragove od početka do kraja uključujući pozive modela, promptove\u002Fkontekst, pretragu, alate, latenciju, tokene\u002Ftroškove, uzorke kvaliteta, bezbednost i konačne ishode zadataka.","Kako nadzirati LLM aplikaciju?",{"id":1370,"answer":1371,"question":1372},"faq9","Da. Lokalni modeli dodaju sopstvene operativne brige kao što su datoteke modela, hardver\u002FVRAM, učitavanje\u002Fistovar, zdravlje okruženja, kvantizacija i upravljanje nadogradnjama.","Mogu li lokalni LLM-ovi koristiti LLMOps prakse?","MLOps vs LLMOps FAQ",{},{"id":1376,"data":1377,"type":42,"tunes":1379},"h-glossary",{"text":1378,"level":253},"Pojmovnik",{},{"id":1381,"data":1382,"type":1381,"tunes":1428},"glossary",{"title":1383,"entries":1384},"Ključni MLOps i LLMOps pojmovi",[1385,1387,1389,1393,1397,1401,1405,1409,1413,1416,1420,1424],{"term":415,"anchor":414,"definition":1386},"Inženjerske prakse za izgradnju, implementaciju, nadzor i održavanje sistema mašinskog učenja i njihovog životnog ciklusa podataka\u002Fmodela.",{"term":418,"anchor":417,"definition":1388},"Operativne prakse za produkcijske aplikacije čije ponašanje materijalno zavisi od velikih jezičkih modela i okolnih promptova, konteksta, pretrage, alata i okruženja.",{"term":1390,"anchor":1391,"definition":1392},"GenAIOps","genaiops","Operativna disciplina za generativne AI aplikacije; često se koristi kao šira ili alternativna oznaka za LLMOps.",{"term":1394,"anchor":1395,"definition":1396},"Kontinuirana obuka","continuous-training","Automatizovano ili ponovljeno ponovno obučavanje i serviranje ML modela kako se podaci ili implementacije menjaju.",{"term":1398,"anchor":1399,"definition":1400},"Kontinuirana evaluacija","continuous-evaluation","Ponovljena evaluacija ponašanja kandidata i produkcijskih AI sistema prema verzionisanim skupovima podataka i kriterijumima.",{"term":1402,"anchor":1403,"definition":1404},"Snimak modela","model-snapshot","Konkretna verzija hostovanog ili pakovanog modela čije ponašanje može biti testirano i referencirano.",{"term":1406,"anchor":1407,"definition":1408},"Poreklo aplikacije","application-lineage","Sledljiv odnos između koda, modela\u002Fprovajdera, promptova, podataka\u002Fpretrage, alata, okruženja i konfiguracije izdanja.",{"term":1410,"anchor":1411,"definition":1412},"Trag","trace","Strukturirani zapis jednog izvršavanja aplikacije koji sadrži segmente kao što su pozivi modela, pretrage i operacije alata.",{"term":469,"anchor":1414,"definition":1415},"eval-dataset","Verzionisani skup reprezentativnih ulaza, očekivanja i opciono tragova\u002Fizlaza koji se koristi za merenje ponašanja.",{"term":1417,"anchor":1418,"definition":1419},"LLM sudija","llm-judge","Jezički model koji se koristi kao evaluator za kvalitativne ili semantičke kriterijume; i sam je verzionisana zavisnost evaluacije.",{"term":1421,"anchor":1422,"definition":1423},"Regresija ponašanja","behavioral-regression","Degradacija izlaza ili putanje aplikacije uprkos tome što interfejsi i kod nastavljaju uspešno da se izvršavaju.",{"term":1425,"anchor":1426,"definition":1427},"Usmeravanje provajdera","provider-routing","Politika za izbor između dostupnih provajdera\u002Fendpointa modela prema sposobnosti, trošku, latenciji, privatnosti ili dostupnosti.",{},{"id":1430,"data":1431,"type":42,"tunes":1433},"h-conclusion",{"text":1432,"level":253},"Zaključak",{},{"id":1435,"data":1436,"type":218,"tunes":1438},"p-conclusion-1",{"text":1437},"MLOps i LLMOps dele isti inženjerski cilj: učiniti AI sisteme dovoljno reproduktivnim, dovoljno testabilnim i dovoljno opservabilnim da pouzdano rade u produkciji.",{},{"id":1440,"data":1441,"type":218,"tunes":1443},"p-conclusion-2",{"text":1442},"Razlika je u obliku sistema. Klasični MLOps se često fokusira na obuku i serviranje artefakata modela; LLMOps mora upravljati bihevioralnim stekom u kojem se snimci modela, promptovi, kontekst, pretraga, alati, dozvole i provajderi mogu menjati nezavisno.",{},{"id":1445,"data":1446,"type":218,"tunes":1448},"p-conclusion-3",{"text":1447},"Najkraće korisno pravilo je: verzionisati, evaluirati i posmatrati sve što može materijalno promeniti ponašanje LLM aplikacije — ne samo model.",{},{"id":1450,"data":1451,"type":42,"tunes":1453},"h-sources",{"text":1452,"level":253},"Primarni izvori i trenutna dokumentacija",{},{"id":1455,"data":1456,"type":218,"tunes":1458},"p-sources-note",{"text":1457},"Izvori u nastavku postavljaju MLOps osnovu i trenutne operativne obrasce za LLM i agentske aplikacije. Projektni odeljci su originalni dokazi implementacije i namerno su uži od tvrdnji o kompletnoj LLMOps platformi.",{},{"id":1460,"data":1461,"type":1467,"tunes":1468},"src-google-mlops",{"link":1462,"meta":1463},"https:\u002F\u002Fdocs.cloud.google.com\u002Farchitecture\u002Fmlops-continuous-delivery-and-automation-pipelines-in-machine-learning",{"image":1464,"title":1465,"description":1466},{"url":381},"Google Cloud — MLOps: Pipeline za kontinuiranu isporuku i automatizaciju","Referentna arhitektura koja opisuje CI, CD, kontinuiranu obuku, registar modela, metapodatke, serviranje i nadzor za ML sisteme.","linkTool",{},{"id":1470,"data":1471,"type":1467,"tunes":1477},"src-aws-lineage",{"link":1472,"meta":1473},"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops02-bp04.html",{"image":1474,"title":1475,"description":1476},{"url":381},"AWS Machine Learning Lens — Poreklo modela","Trenutne smernice za praćenje koda, podataka, modela, okruženja i infrastrukture kroz ML izdanja.",{},{"id":1479,"data":1480,"type":1467,"tunes":1486},"src-aws-monitor",{"link":1481,"meta":1482},"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops06-bp02.html",{"image":1483,"title":1484,"description":1485},{"url":381},"AWS Machine Learning Lens — Opservabilnost i praćenje modela","Trenutne smernice za nadzor produkcijskih modela, drift, zdravlje endpointa i poreklo.",{},{"id":1488,"data":1489,"type":1467,"tunes":1495},"src-azure-llmops",{"link":1490,"meta":1491},"https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fmachine-learning\u002Fprompt-flow\u002Fhow-to-end-to-end-llmops-with-prompt-flow",{"image":1492,"title":1493,"description":1494},{"url":381},"Microsoft Azure — GenAIOps \u002F LLMOps životni ciklus","Zvanične smernice koje opisuju GenAIOps, ponekad nazvan LLMOps, kroz inicijalizaciju, eksperimentisanje, evaluaciju\u002Frafiniranje i implementaciju.",{},{"id":1497,"data":1498,"type":1467,"tunes":1504},"src-mlflow-genai",{"link":1499,"meta":1500},"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002F",{"image":1501,"title":1502,"description":1503},{"url":381},"MLflow — Agenti i LLM aplikacije","Trenutna GenAI operativna dokumentacija koja pokriva praćenje, evaluaciju, promptove i produkcijsku opservabilnost za LLM aplikacije i agente.",{},{"id":1506,"data":1507,"type":1467,"tunes":1513},"src-mlflow-traces",{"link":1508,"meta":1509},"https:\u002F\u002Fwww.mlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Feval-monitor\u002Frunning-evaluation\u002Ftraces\u002F",{"image":1510,"title":1511,"description":1512},{"url":381},"MLflow — Evaluacija produkcijskih tragova","Trenutne smernice za evaluaciju kompletnih LLM\u002Fagentskih tragova, uključujući pretragu i putanje poziva alata.",{},{"id":1515,"data":1516,"type":1467,"tunes":1522},"src-mlflow-prompt-eval",{"link":1517,"meta":1518},"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Fprompt-registry\u002Fevaluate-prompts\u002F",{"image":1519,"title":1520,"description":1521},{"url":381},"MLflow — Evaluacija promptova","Trenutni tok rada za evaluaciju promptova\u002Fmodela koristeći verzionisane promptove, skupove podataka, ocenjivače i tragove.",{},{"id":1524,"data":1525,"type":1467,"tunes":1531},"src-openai-api",{"link":1526,"meta":1527},"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Freference\u002Foverview",{"image":1528,"title":1529,"description":1530},{"url":381},"OpenAI API — Verzionisanje i snimci modela","Trenutne smernice API-ja koje preporučuju fiksirane verzije modela i evaluacije jer se ponašanje promptovanja može promeniti između snimaka.",{},{"id":1533,"data":1534,"type":1467,"tunes":1540},"src-openai-prompting",{"link":1535,"meta":1536},"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fguides\u002Fprompting",{"image":1537,"title":1538,"description":1539},{"url":381},"OpenAI — Promptovanje","Trenutne smernice da se produkcijski promptovi tretiraju kao aplikacijski kod, verzioniraju kroz kontrolu izvornog koda i da se promene pokriju testovima i proverama evaluacije.",{},{"id":1542,"data":1543,"type":1467,"tunes":1549},"src-openai-deprecations",{"link":1544,"meta":1545},"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fdeprecations",{"image":1546,"title":1547,"description":1548},{"url":381},"OpenAI — Ukidanja","Trenutni dokazi o životnom ciklusu provajdera koji pokazuju ukidanje modela i platformskih površina kao operativnu zavisnost.",{},{"id":1551,"data":1552,"type":1467,"tunes":1558},"src-openai-promptfoo",{"link":1553,"meta":1554},"https:\u002F\u002Fdevelopers.openai.com\u002Fcookbook\u002Fexamples\u002Fevaluation\u002Fmoving-from-openai-evals-to-promptfoo",{"image":1555,"title":1556,"description":1557},{"url":381},"OpenAI — Premeštanje tokova evaluacije na Promptfoo","Trenutne smernice za migraciju iz 2026. koje ilustruju zašto sredstva za evaluaciju treba da ostanu prenosiva kada se alat provajdera menja.",{},"2.31","MLOps upravlja sistemima mašinskog učenja; LLMOps proširuje te prakse na promptove, kontekst, pretragu, provajdere, alate, evaluacije i ponašanje u vreme izvršavanja oko velikih jezičkih modela.","\u002Fuploads\u002F2026\u002F10\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm-1791487319869-2v7hxo.webp","mlops-vs-llmops-what-changes-when-the-model-is-an-llm-1791487319869-2v7hxo","PUBLISHED","2026-10-08T15:20:00.000Z","2026-10-08T19:20:01.249Z","2026-10-08T19:31:17.955Z",{"en":1568,"de":1569,"sr":1570,"es":1571,"fr":1572,"it":1573,"ru":1574,"zh":1575},"\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fde\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fsr\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fes\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Ffr\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fit\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fru\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm","\u002Fzh\u002Fblog\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm",[1577,1581,1585,1589],{"id":1578,"name":1579,"slug":1580},88,"Verzionisanje (prompt, modeli)","versioning",{"id":1582,"name":1583,"slug":1584},91,"Monitoring (kvalitet, drift)","monitoring",{"id":1586,"name":1587,"slug":1588},89,"Eval harness","evaluation-harness",{"id":1590,"name":1591,"slug":1592},58,"Evaluacija i gate-ovi kvaliteta","evaluation",{"id":1594,"login":1595,"email":1596,"displayName":1597},"20","rooth8233","aleksandar@stajic.de","Aleksandar Stajić",[1599,2722],{"lang":1600,"title":1601,"content":1602,"contentJson":1603,"excerpt":2721},"en","MLOps vs LLMOps: What Changes When the Model Is an LLM","{\"time\":1791487321430,\"blocks\":[{\"id\":\"intro\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLOps is the engineering discipline for reliably developing, deploying, versioning and operating machine-learning systems; LLMOps extends that discipline to applications built around large language models, where production behavior depends not only on a model artifact but also on prompts, context, retrieval, provider\u002Fmodel versions, tool calls, safety controls and evaluation pipelines. LLMOps does not replace MLOps. It changes the operational unit from “a model plus serving pipeline” toward “an evolving LLM application whose behavior emerges from several independently changing components.”\"},\"tunes\":{}},{\"id\":\"direct\",\"type\":\"callout\",\"data\":{\"variant\":\"info\",\"title\":\"Direct answer\",\"body\":\"\u003Cstrong>MLOps operates ML systems. LLMOps operates LLM applications.\u003C\u002Fstrong>\u003Cbr>\u003Cbr>Classical MLOps commonly centers on data pipelines, training, validation, model registry, deployment, drift and retraining. LLMOps keeps those disciplines where relevant, but often adds prompt\u002Fcontext versioning, model\u002Fprovider abstraction, RAG indexes, agent\u002Ftool traces, semantic evaluations, safety tests, token\u002Fcost monitoring and regression testing across rapidly changing model snapshots.\"},\"tunes\":{}},{\"id\":\"boundary\",\"type\":\"callout\",\"data\":{\"variant\":\"warning\",\"title\":\"LLMOps is not just prompt management\",\"body\":\"A production LLM application can fail even when the prompt is unchanged: the provider can change a model snapshot, a RAG corpus can become stale, a reranker can regress, tool permissions can change, context assembly can drop evidence, or an agent can take a wrong trajectory. LLMOps therefore has to observe and version the system around the model, not only prompt text.\"},\"tunes\":{}},{\"id\":\"term-note\",\"type\":\"callout\",\"data\":{\"variant\":\"note\",\"title\":\"Terminology boundary\",\"body\":\"\u003Cstrong>LLMOps\u003C\u002Fstrong>, \u003Cstrong>GenAIOps\u003C\u002Fstrong> and related terms are widely used engineering labels, but they are not one universal formal standard with a single canonical lifecycle. Microsoft currently describes GenAIOps as “sometimes called LLMOps,” while MLflow groups operational tooling around agents and LLM applications. This article uses LLMOps as a practical architecture term for operating production systems whose behavior materially depends on LLMs.\"},\"tunes\":{}},{\"id\":\"current\",\"type\":\"callout\",\"data\":{\"variant\":\"note\",\"title\":\"Current-source note — 8 October 2026\",\"body\":\"The operational surface is changing quickly. OpenAI currently recommends pinning model snapshots and running evals because prompting behavior can change between snapshots, and several older platform-specific prompt\u002Feval surfaces are being retired in 2026. The stable architectural lesson is to keep prompts, tests and evals portable and versioned with the application rather than depend on one provider's dashboard object model.\"},\"tunes\":{}},{\"id\":\"toc\",\"type\":\"tableOfContents\",\"data\":{\"title\":\"Contents\",\"minLevel\":2,\"maxLevel\":3},\"tunes\":{}},{\"id\":\"h-meaning\",\"type\":\"header\",\"data\":{\"text\":\"What MLOps really means\",\"level\":2},\"tunes\":{}},{\"id\":\"p-mlops-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLOps applies software-engineering and operational discipline to machine-learning systems. The production challenge is broader than training a model: data collection, data validation, experimentation, reproducibility, model evaluation, deployment, infrastructure and monitoring all have to work together.\"},\"tunes\":{}},{\"id\":\"p-mlops-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Google's MLOps architecture guidance frames the discipline around continuous integration, continuous delivery and continuous training. CI validates not only code but also data, schemas and models; CD deploys ML pipelines and prediction services; CT can retrain and redeploy models as data or implementations change.\"},\"tunes\":{}},{\"id\":\"p-mlops-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"AWS guidance adds the same operational concerns from another angle: model lineage, model\u002Fversion traceability, drift monitoring and production-quality monitoring are core parts of keeping ML systems reliable after deployment.\"},\"tunes\":{}},{\"id\":\"h-llmops\",\"type\":\"header\",\"data\":{\"text\":\"What changes when the model is an LLM\",\"level\":2},\"tunes\":{}},{\"id\":\"p-llmops-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Large language models change the production problem because the application often does not own the complete model-training lifecycle. A team may call a hosted model API, run an open model locally, switch between providers or use several models for different tasks.\"},\"tunes\":{}},{\"id\":\"p-llmops-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The model is therefore only one versioned dependency inside a larger behavioral system. Prompts, retrieval results, context order, tools, model snapshot, temperature\u002Freasoning settings, safety filters and runtime orchestration can all change the output.\"},\"tunes\":{}},{\"id\":\"p-llmops-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"This creates a broader operational question: which combination of model, context, data, prompt, tools and runtime produced this behavior? LLMOps exists to make that question answerable and the answer reproducible enough for engineering work.\"},\"tunes\":{}},{\"id\":\"h-simple\",\"type\":\"header\",\"data\":{\"text\":\"The simplest example\",\"level\":2},\"tunes\":{}},{\"id\":\"p-simple-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Suppose an application answers internal policy questions.\"},\"tunes\":{}},{\"id\":\"p-simple-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"In a classical ML framing, you might version a trained classifier, deploy it and monitor prediction quality. In an LLM application, the answer might depend on a hosted model snapshot, a system prompt, an embedding model, a vector index, retrieval filters, a reranker and the final selected context.\"},\"tunes\":{}},{\"id\":\"p-simple-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Changing any one of those components can change the final answer even though the application endpoint and user question stay identical.\"},\"tunes\":{}},{\"id\":\"simple-flow\",\"type\":\"processFlow\",\"data\":{\"title\":\"A typical LLMOps release path\",\"orientation\":\"auto\",\"steps\":[{\"label\":\"1. Change one component\",\"description\":\"Prompt, model, provider, retrieval setting, tool schema or application code changes.\"},{\"label\":\"2. Run deterministic tests\",\"description\":\"Validate schemas, permissions, tool contracts, retrieval filters and application behavior.\"},{\"label\":\"3. Run behavioral evals\",\"description\":\"Compare representative outputs, retrieval quality and agent\u002Ftool trajectories against acceptance criteria.\"},{\"label\":\"4. Compare cost and latency\",\"description\":\"Measure token use, model calls, retrieval\u002Ftool overhead and response latency.\"},{\"label\":\"5. Deploy controlled version\",\"description\":\"Ship the concrete application configuration with model\u002Fprovider versions recorded.\"},{\"label\":\"6. Trace production behavior\",\"description\":\"Capture relevant model, retrieval, tool and runtime spans.\"},{\"label\":\"7. Evaluate production traces\",\"description\":\"Sample real executions for quality, grounding, safety and task success.\"},{\"label\":\"8. Roll back or iterate\",\"description\":\"Use regression evidence and operational signals to decide the next release.\"}]},\"tunes\":{}},{\"id\":\"h-stops\",\"type\":\"header\",\"data\":{\"text\":\"Where the simple example stops\",\"level\":2},\"tunes\":{}},{\"id\":\"p-stops-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Some LLM systems still train or fine-tune their own models, so traditional MLOps practices such as training pipelines, model registry and data lineage remain directly relevant.\"},\"tunes\":{}},{\"id\":\"p-stops-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Other systems use only external foundation-model APIs and never run continuous training. Their main operational workload is application evaluation, model\u002Fprovider change management, prompt\u002Fcontext versioning, retrieval quality and observability.\"},\"tunes\":{}},{\"id\":\"p-stops-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"There is therefore no single universal “LLMOps pipeline.” The exact lifecycle depends on whether you train, fine-tune, self-host, retrieve external knowledge, run agents or depend on managed model APIs.\"},\"tunes\":{}},{\"id\":\"h-compare\",\"type\":\"header\",\"data\":{\"text\":\"MLOps vs LLMOps\",\"level\":2},\"tunes\":{}},{\"id\":\"main-comparison\",\"type\":\"comparison\",\"data\":{\"title\":\"What stays the same and what expands\",\"layout\":\"table\",\"columns\":[{\"id\":\"mlops\",\"label\":\"MLOps\"},{\"id\":\"llmops\",\"label\":\"LLMOps\"}],\"rows\":[{\"id\":\"unit\",\"label\":\"Primary operational unit\",\"values\":[\"\",\"\"]},{\"id\":\"model\",\"label\":\"Model ownership\",\"values\":[\"\",\"\"]},{\"id\":\"change\",\"label\":\"Typical change\",\"values\":[\"\",\"\"]},{\"id\":\"eval\",\"label\":\"Evaluation\",\"values\":[\"\",\"\"]},{\"id\":\"monitor\",\"label\":\"Production monitoring\",\"values\":[\"\",\"\"]},{\"id\":\"training\",\"label\":\"Continuous training\",\"values\":[\"\",\"\"]},{\"id\":\"registry\",\"label\":\"Versioned artifacts\",\"values\":[\"\",\"\"]},{\"id\":\"rollback\",\"label\":\"Rollback target\",\"values\":[\"\",\"\"]}]},\"tunes\":{}},{\"id\":\"h-extension\",\"type\":\"header\",\"data\":{\"text\":\"LLMOps extends MLOps rather than replacing it\",\"level\":2},\"tunes\":{}},{\"id\":\"p-extension-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"The core operational principles do not disappear: source control, CI\u002FCD, reproducibility, lineage, deployment controls, monitoring, rollback and measurable acceptance criteria remain essential.\"},\"tunes\":{}},{\"id\":\"p-extension-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The extension is that more behavior-defining artifacts now sit outside the model weights. A managed foundation model can change behavior through snapshot upgrades, while application output can change through prompt or retrieval changes without any model retraining.\"},\"tunes\":{}},{\"id\":\"p-extension-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"This is why the useful hierarchy is usually DevOps → MLOps → LLMOps\u002FGenAIOps as increasingly specialized operational concerns, not three mutually exclusive practices.\"},\"tunes\":{}},{\"id\":\"h-artifacts\",\"type\":\"header\",\"data\":{\"text\":\"What has to be versioned in LLMOps?\",\"level\":2},\"tunes\":{}},{\"id\":\"artifact-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Artifact\",\"Why it matters\"],[\"Application code\",\"Defines orchestration, validation, retries and business behavior\"],[\"Model family + snapshot\u002Fversion\",\"Different snapshots can produce different behavior\"],[\"Provider \u002F endpoint\",\"Changes data flow, latency, limits, pricing and availability\"],[\"Prompt\u002Finstruction code\",\"Changes model behavior even with same model\"],[\"Generation\u002Freasoning parameters\",\"Can alter determinism, latency, depth and cost\"],[\"Eval dataset\",\"Defines what “good enough” is tested against\"],[\"Scorers \u002F graders\",\"Define how quality is measured\"],[\"Embedding model\",\"Changes vector representation and retrieval behavior\"],[\"Chunking\u002Findex configuration\",\"Changes what can be retrieved\"],[\"Reranker \u002F retrieval fusion\",\"Changes result ordering\"],[\"Tool schemas\",\"Change what the model can request and how\"],[\"Permission profile\",\"Changes what tool actions may actually execute\"],[\"Context assembly rules\",\"Change what evidence and state reach the model\"],[\"Safety\u002Fguardrail configuration\",\"Changes allowed or blocked behavior\"]]},\"tunes\":{}},{\"id\":\"h-model-version\",\"type\":\"header\",\"data\":{\"text\":\"Model snapshots become release dependencies\",\"level\":2},\"tunes\":{}},{\"id\":\"p-model-version-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"With hosted LLMs, the team may not control model training, but it still controls which model or snapshot the application calls.\"},\"tunes\":{}},{\"id\":\"p-model-version-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"OpenAI's current API guidance explicitly warns that prompting behavior can change between model snapshots and recommends pinning production applications to specific snapshots where consistency matters, then running evals when upgrading.\"},\"tunes\":{}},{\"id\":\"p-model-version-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The operational consequence is straightforward: model upgrades should be treated as application releases, not invisible infrastructure maintenance.\"},\"tunes\":{}},{\"id\":\"h-provider\",\"type\":\"header\",\"data\":{\"text\":\"Provider lifecycle becomes part of operations\",\"level\":2},\"tunes\":{}},{\"id\":\"p-provider-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLM applications often depend on provider rate limits, deprecation schedules, API semantics, context limits, data-handling rules and pricing.\"},\"tunes\":{}},{\"id\":\"p-provider-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"A provider can deprecate a model while your application code remains unchanged. OpenAI's current deprecation schedule, for example, includes 2026 retirement dates for older model snapshots and platform surfaces.\"},\"tunes\":{}},{\"id\":\"p-provider-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps therefore needs provider lifecycle tracking, migration testing and fallback decisions in addition to model-quality monitoring.\"},\"tunes\":{}},{\"id\":\"h-prompt\",\"type\":\"header\",\"data\":{\"text\":\"Prompts behave like production code\",\"level\":2},\"tunes\":{}},{\"id\":\"p-prompt-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Prompts are executable behavioral configuration. Small changes can alter output quality, tool selection and policy interpretation.\"},\"tunes\":{}},{\"id\":\"p-prompt-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"OpenAI's current guidance recommends storing production prompts in application code, reviewing prompt changes through pull requests, using typed inputs and covering changes with tests and evaluation checks.\"},\"tunes\":{}},{\"id\":\"p-prompt-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"That makes prompt versioning less like editing marketing copy and more like changing a function whose output is probabilistic and model-dependent.\"},\"tunes\":{}},{\"id\":\"h-context\",\"type\":\"header\",\"data\":{\"text\":\"Context engineering becomes an operational concern\",\"level\":2},\"tunes\":{}},{\"id\":\"p-context-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"The production model rarely receives only a static prompt. It may receive conversation history, retrieved documents, tool outputs, memory, current application state and policy instructions.\"},\"tunes\":{}},{\"id\":\"p-context-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps must therefore observe context assembly: which evidence was selected, which state version was current, whether truncation occurred and whether important instructions survived compaction.\"},\"tunes\":{}},{\"id\":\"p-context-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"A model regression and a context regression can look identical at the final answer. Tracing the actual context path is what lets the team separate them.\"},\"tunes\":{}},{\"id\":\"h-rag\",\"type\":\"header\",\"data\":{\"text\":\"RAG creates its own operational lifecycle\",\"level\":2},\"tunes\":{}},{\"id\":\"p-rag-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"A RAG system introduces a second production pipeline beside model inference: ingestion, extraction, chunking, metadata, embeddings, indexes, retrieval, reranking and context selection.\"},\"tunes\":{}},{\"id\":\"p-rag-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The knowledge corpus can change every day even when the model and prompt do not. A stale index or broken metadata filter can therefore degrade answer quality without any model drift.\"},\"tunes\":{}},{\"id\":\"p-rag-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps for RAG should track corpus\u002Findex version, embedding model, chunking policy, retrieval configuration, source freshness and retrieval metrics separately from generation quality.\"},\"tunes\":{}},{\"id\":\"ref-rag-diagnostic\",\"type\":\"referralArticle\",\"data\":{\"url\":\"https:\u002F\u002Fstajic.de\u002Fblog\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method\",\"title\":\"RAG Failed — But Which Layer Actually Failed? A Diagnostic Method\",\"excerpt\":\"A production LLM pipeline needs separate observability for source coverage, retrieval, ranking, context assembly and generation.\",\"ctaLabel\":\"Read the RAG diagnostic method\"},\"tunes\":{}},{\"id\":\"h-evals\",\"type\":\"header\",\"data\":{\"text\":\"Evals replace “looks good to me” with release evidence\",\"level\":2},\"tunes\":{}},{\"id\":\"p-eval-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Generative outputs are often open-ended, so exact-match tests are insufficient for many tasks. LLMOps adds evaluation datasets and scorers that can measure task success, correctness, safety, groundedness, style or domain-specific acceptance criteria.\"},\"tunes\":{}},{\"id\":\"p-eval-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLflow's current GenAI evaluation stack supports versioned evaluation datasets, prompt\u002Fmodel comparisons, custom scorers and evaluation over complete traces.\"},\"tunes\":{}},{\"id\":\"p-eval-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The strongest practice is evaluation-driven development: define representative cases and acceptance criteria before or alongside changes, then compare releases against the same evidence.\"},\"tunes\":{}},{\"id\":\"eval-rule\",\"type\":\"callout\",\"data\":{\"variant\":\"success\",\"title\":\"Behavioral changes need behavioral tests\",\"body\":\"A deployment should not be considered equivalent merely because the API contract still works. If the prompt, model, retrieval or tools changed, the behavioral regression suite should run again.\"},\"tunes\":{}},{\"id\":\"h-judges\",\"type\":\"header\",\"data\":{\"text\":\"LLM-as-a-judge is useful but not ground truth\",\"level\":2},\"tunes\":{}},{\"id\":\"p-judge-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLM judges can scale evaluation for qualities that are expensive to encode as deterministic assertions, such as relevance, tone or groundedness.\"},\"tunes\":{}},{\"id\":\"p-judge-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"However, the judge is another model with its own bias, version and prompt. Judge configuration should therefore be versioned and calibrated against human or deterministic reference cases where consequence matters.\"},\"tunes\":{}},{\"id\":\"p-judge-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"A production eval can mix deterministic checks, reference-based metrics, model judges and human review rather than asking one metric to represent every quality dimension.\"},\"tunes\":{}},{\"id\":\"h-tracing\",\"type\":\"header\",\"data\":{\"text\":\"Tracing becomes more important than endpoint logs\",\"level\":2},\"tunes\":{}},{\"id\":\"p-trace-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Traditional API logs can tell you that a request took two seconds and returned HTTP 200. They cannot tell you which retrieved chunks were selected, which tool the agent called or which model span consumed most tokens.\"},\"tunes\":{}},{\"id\":\"p-trace-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLflow's current GenAI tracing captures prompts, retrievals, tool calls and application spans, and its production evaluation flow can score intermediate trajectory information rather than only final text.\"},\"tunes\":{}},{\"id\":\"p-trace-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"This is a major LLMOps shift: observability follows the behavioral graph of the application, not only the serving endpoint.\"},\"tunes\":{}},{\"id\":\"h-agent\",\"type\":\"header\",\"data\":{\"text\":\"Agents expand LLMOps into runtime operations\",\"level\":2},\"tunes\":{}},{\"id\":\"p-agent-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"An agentic application can perform several model calls, tool invocations and state transitions before producing a result.\"},\"tunes\":{}},{\"id\":\"p-agent-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Operating agents therefore requires step counts, tool-call traces, permission denials, retries, loop detection, human approvals and verified final state in addition to ordinary model latency and token metrics.\"},\"tunes\":{}},{\"id\":\"p-agent-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"A correct final answer can hide a bad trajectory, so agent evaluation must inspect the path as well as the result.\"},\"tunes\":{}},{\"id\":\"ref-agent-reliability\",\"type\":\"referralArticle\",\"data\":{\"url\":\"https:\u002F\u002Fstajic.de\u002Fblog\u002Fai-agent-reliability-why-the-final-answer-is-not-enough\",\"title\":\"AI Agent Reliability: Why the Final Answer Is Not Enough\",\"excerpt\":\"Why agent production evaluation must include tool calls, state transitions, approvals and recoverability.\",\"ctaLabel\":\"Read the agent reliability article\"},\"tunes\":{}},{\"id\":\"h-cost\",\"type\":\"header\",\"data\":{\"text\":\"Tokens, model calls and context become cost variables\",\"level\":2},\"tunes\":{}},{\"id\":\"p-cost-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Classical ML inference cost is often dominated by serving infrastructure or per-prediction compute. LLM applications can add provider token pricing, repeated agent calls, embedding calls, reranking and tool\u002Fruntime overhead.\"},\"tunes\":{}},{\"id\":\"p-cost-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Cost therefore has to be attributed to task or trace, not only to one endpoint. A workflow that makes eight hidden model calls can be functionally correct but operationally unacceptable.\"},\"tunes\":{}},{\"id\":\"p-cost-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Latency behaves the same way: model latency, retrieval, reranking and external tools compose into end-to-end user latency.\"},\"tunes\":{}},{\"id\":\"h-cache\",\"type\":\"header\",\"data\":{\"text\":\"Caching becomes semantic, not only technical\",\"level\":2},\"tunes\":{}},{\"id\":\"p-cache-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLM systems can cache prompts, embeddings, retrieval results or full responses, but the cache key must reflect the semantics that can change the result.\"},\"tunes\":{}},{\"id\":\"p-cache-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"A response cache that ignores model version, tenant, permissions or source freshness can return a technically valid but semantically invalid answer.\"},\"tunes\":{}},{\"id\":\"p-cache-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps therefore treats cache invalidation as part of model\u002Fcontext\u002Fdata versioning rather than only infrastructure optimization.\"},\"tunes\":{}},{\"id\":\"h-safety\",\"type\":\"header\",\"data\":{\"text\":\"Safety and permissions become release criteria\",\"level\":2},\"tunes\":{}},{\"id\":\"p-safety-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Generative systems can produce unbounded text and agents can trigger external actions. Safety testing therefore sits closer to ordinary CI\u002FCD than in many classical predictive ML systems.\"},\"tunes\":{}},{\"id\":\"p-safety-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Permission checks, prompt-injection tests, tenant-isolation tests and side-effect approvals should be reproducible regression tests where those risks exist.\"},\"tunes\":{}},{\"id\":\"p-safety-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The model may suggest an operation, but the runtime still has to enforce authorization. LLMOps owns the evidence that those controls continue to work after model, prompt or tool changes.\"},\"tunes\":{}},{\"id\":\"h-ci\",\"type\":\"header\",\"data\":{\"text\":\"What CI looks like in LLMOps\",\"level\":2},\"tunes\":{}},{\"id\":\"ci-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"CI layer\",\"Example checks\"],[\"Code\",\"Unit tests, type checks, schema validation\"],[\"Prompts\",\"Template rendering, required variables, policy text, snapshot review\"],[\"Models\u002Fproviders\",\"Compatibility, output schema, capability and regression tests\"],[\"RAG\",\"Chunking fixtures, filter tests, Recall@k, reranker regression\"],[\"Tools\",\"Input\u002Foutput schema tests, permission tests, idempotency tests\"],[\"Agents\",\"Trajectory fixtures, loop limits, handoff\u002Ftool-selection tests\"],[\"Security\",\"Prompt injection, unauthorized tools, cross-tenant negative tests\"],[\"Behavioral evals\",\"Task success, correctness, grounding, safety, domain criteria\"],[\"Operational\",\"Latency, token\u002Fcost budgets, timeout\u002Ffallback behavior\"]]},\"tunes\":{}},{\"id\":\"h-cd\",\"type\":\"header\",\"data\":{\"text\":\"What CD looks like in LLMOps\",\"level\":2},\"tunes\":{}},{\"id\":\"p-cd-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"A production release may deploy no new model artifact at all. It may simply ship a new prompt, retrieval configuration, tool set or provider mapping.\"},\"tunes\":{}},{\"id\":\"p-cd-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The release bundle should therefore identify the complete behavior-defining configuration rather than only the application container image.\"},\"tunes\":{}},{\"id\":\"p-cd-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Feature flags, staged rollout, shadow evaluation, canary traffic and rollback are useful because LLM behavior can regress in ways that static contract tests do not detect.\"},\"tunes\":{}},{\"id\":\"h-ct\",\"type\":\"header\",\"data\":{\"text\":\"Continuous training becomes optional; continuous evaluation becomes central\",\"level\":2},\"tunes\":{}},{\"id\":\"p-ct-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Traditional MLOps often emphasizes continuous training when new data or drift justifies retraining.\"},\"tunes\":{}},{\"id\":\"p-ct-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Many LLM applications never train the foundation model. Their equivalent continuous loop is continuous evaluation: collect failures and representative production cases, add them to evaluation datasets, test candidate prompt\u002Fmodel\u002Fretrieval changes and redeploy only when evidence improves.\"},\"tunes\":{}},{\"id\":\"p-ct-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Fine-tuning can reintroduce a training lifecycle, but it should sit inside the same broader evaluation and release process.\"},\"tunes\":{}},{\"id\":\"h-monitor\",\"type\":\"header\",\"data\":{\"text\":\"What should be monitored in production?\",\"level\":2},\"tunes\":{}},{\"id\":\"monitor-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Signal class\",\"Examples\"],[\"System health\",\"Errors, timeouts, endpoint availability\"],[\"Model\u002Fprovider\",\"Model ID, snapshot, rate limits, provider errors\"],[\"Latency\",\"End-to-end, model, retrieval, tool and reranker spans\"],[\"Cost\",\"Input\u002Foutput tokens, embeddings, tool\u002FAPI spend\"],[\"Quality\",\"Sampled task success, correctness, relevance, groundedness\"],[\"RAG\",\"Retrieval recall proxies, empty retrieval, stale sources, citation coverage\"],[\"Agents\",\"Tool selection, retries, loops, handoffs, approval frequency\"],[\"Security\",\"Denied actions, prompt-injection indicators, tenant-boundary failures\"],[\"User feedback\",\"Corrections, abandonment, escalation, explicit ratings\"],[\"Change drift\",\"Provider\u002Fmodel\u002Fconfig changes relative to approved release\"]]},\"tunes\":{}},{\"id\":\"h-prod-eval\",\"type\":\"header\",\"data\":{\"text\":\"Production traces can become evaluation data\",\"level\":2},\"tunes\":{}},{\"id\":\"p-prod-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"One of the most useful modern LLMOps patterns is to turn sampled production traces into evaluation records.\"},\"tunes\":{}},{\"id\":\"p-prod-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLflow currently supports retrieving production traces and scoring not only outputs but intermediate spans such as retrieval or tool-call trajectories.\"},\"tunes\":{}},{\"id\":\"p-prod-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"This closes the loop between observability and development: real failures can become regression cases in the next release rather than disappear inside logs.\"},\"tunes\":{}},{\"id\":\"h-repro\",\"type\":\"header\",\"data\":{\"text\":\"Reproducibility becomes conditional rather than exact\",\"level\":2},\"tunes\":{}},{\"id\":\"p-repro-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Classical ML reproducibility often aims to recreate a model from versioned code, data, environment and training parameters.\"},\"tunes\":{}},{\"id\":\"p-repro-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Hosted LLM applications cannot always reproduce identical output token-for-token because generation is probabilistic and providers may control infrastructure.\"},\"tunes\":{}},{\"id\":\"p-repro-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps therefore aims for behavioral reproducibility: record enough model\u002Fprovider\u002Fversion, prompt, context inputs, retrieval state and runtime configuration to reproduce the conditions and validate behavior within expected tolerances.\"},\"tunes\":{}},{\"id\":\"h-lineage\",\"type\":\"header\",\"data\":{\"text\":\"Lineage expands from model lineage to application lineage\",\"level\":2},\"tunes\":{}},{\"id\":\"p-lineage-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"AWS's MLOps guidance treats model lineage as the history of code, data, model and infrastructure artifacts needed for diagnosis and reproducibility.\"},\"tunes\":{}},{\"id\":\"p-lineage-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"For LLM applications, lineage should additionally connect prompts, eval datasets, retrieval\u002Findex versions, tool schemas, agent\u002Fruntime configuration and provider\u002Fmodel snapshots.\"},\"tunes\":{}},{\"id\":\"p-lineage-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The target question becomes: Which exact application configuration produced this trace?\"},\"tunes\":{}},{\"id\":\"h-routing\",\"type\":\"header\",\"data\":{\"text\":\"Multi-provider and model routing create operational policy\",\"level\":2},\"tunes\":{}},{\"id\":\"p-route-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Once an application can use several providers or local models, routing becomes an operational policy rather than a simple model string.\"},\"tunes\":{}},{\"id\":\"p-route-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Routing may depend on capability, latency, cost, privacy, context length, availability, tool support or locality. A fallback can preserve uptime while changing answer quality or data-processing assumptions.\"},\"tunes\":{}},{\"id\":\"p-route-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps should therefore log which route was actually selected and evaluate routes independently rather than treat every compatible endpoint as behaviorally interchangeable.\"},\"tunes\":{}},{\"id\":\"h-implementation\",\"type\":\"header\",\"data\":{\"text\":\"Original implementation evidence\",\"level\":2},\"tunes\":{}},{\"id\":\"h-client\",\"type\":\"header\",\"data\":{\"text\":\"Aaasaasa AI Client: provider, model and runtime are separate operational objects\",\"level\":3},\"tunes\":{}},{\"id\":\"p-client-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"Aaasaasa AI Client separates agent\u002Fclient, provider, model, runtime location and permissions. Its AI Hub supports Ollama, LM Studio\u002FOpenAI-compatible endpoints and other provider protocols rather than treating “the model” as one global setting.\"},\"tunes\":{}},{\"id\":\"p-client-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The implementation includes dynamic local model discovery, streaming, thinking output and explicit Ollama warm\u002Fload and unload controls. That is operational evidence that local LLM serving introduces resource lifecycle concerns beyond an API model name.\"},\"tunes\":{}},{\"id\":\"p-client-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Provider status is queried through provider adapters, and connection types distinguish local, cloud API, account-backed, remote-agent and web-client paths. These are concrete operational dimensions an LLM-aware platform has to surface.\"},\"tunes\":{}},{\"id\":\"p-client-4\",\"type\":\"paragraph\",\"data\":{\"text\":\"The repository also preserves an important boundary: a local runtime is not automatically local inference. Provider\u002Fmodel\u002Fruntime location are versioned or configurable concerns that affect privacy, latency, cost and availability.\"},\"tunes\":{}},{\"id\":\"h-sot\",\"type\":\"header\",\"data\":{\"text\":\"Source of Truth Research Engine: LLM application state extends beyond the model\",\"level\":3},\"tunes\":{}},{\"id\":\"p-sot-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"The Source of Truth Research Engine combines lexical search, optional embeddings, source snapshots, SHA-256 identity, claims, provenance and contradiction tracking around local model-assisted research.\"},\"tunes\":{}},{\"id\":\"p-sot-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"This is useful LLMOps evidence because changing the model alone does not define the research system. Retrieval, source acquisition, evidence classification and persistent provenance are independent operational artifacts.\"},\"tunes\":{}},{\"id\":\"p-sot-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The implementation deliberately treats semantic similarity as discovery rather than evidence, showing why LLMOps observability should distinguish retrieval behavior from claim validity.\"},\"tunes\":{}},{\"id\":\"impl-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Observed implementation\",\"LLMOps lesson\"],[\"Multiple provider protocols\",\"Provider identity is an operational dependency\"],[\"Dynamic model discovery\",\"Available models can change independently of application code\"],[\"Ollama load\u002Funload controls\",\"Local models have memory\u002Fresource lifecycle\"],[\"Provider health\u002Fstatus adapters\",\"Model availability needs runtime observability\"],[\"Separate runtime and inference location\",\"Deployment topology is not one boolean “local\u002Fcloud”\"],[\"Central permissions\",\"Model capability and tool authority must remain separate\"],[\"Lexical + semantic retrieval pipeline\",\"Retrieval configuration is part of application behavior\"],[\"Source\u002Fprovenance persistence\",\"Operational state and evidence live outside model weights\"]]},\"tunes\":{}},{\"id\":\"impl-boundary\",\"type\":\"callout\",\"data\":{\"variant\":\"note\",\"title\":\"Evidence boundary\",\"body\":\"These projects demonstrate multi-provider\u002Flocal-model operations, permission separation, retrieval infrastructure and evidence persistence. They are not presented as a complete commercial LLMOps platform or proof of large-scale production traffic.\"},\"tunes\":{}},{\"id\":\"h-failures\",\"type\":\"header\",\"data\":{\"text\":\"Common LLMOps failure modes\",\"level\":2},\"tunes\":{}},{\"id\":\"failure-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Failure mode\",\"What actually went wrong\"],[\"Model alias upgraded silently\",\"Behavior changed without controlled release\"],[\"Prompt changed without evals\",\"Behavioral regression passed normal unit tests\"],[\"RAG index stale\",\"Generation model was blamed for retrieval\u002Fdata failure\"],[\"Only final answer is logged\",\"Root cause in retrieval\u002Ftool\u002Fcontext trajectory is invisible\"],[\"Provider fallback is silent\",\"Different model\u002Fdata path changes behavior without attribution\"],[\"Token cost tracked globally\",\"Expensive workflows cannot be localized\"],[\"Judge model changed\",\"Evaluation scores drift without application change\"],[\"Production traces never become tests\",\"Known failures repeatedly return\"],[\"Local model stays loaded indefinitely\",\"VRAM\u002Fresource pressure becomes operational instability\"],[\"Permissions encoded only in prompt\",\"Model behavior is mistaken for authorization\"],[\"One eval score gates everything\",\"Different quality dimensions are collapsed into a misleading number\"],[\"Model registry exists but prompt\u002Findex versions do not\",\"Application lineage remains incomplete\"]]},\"tunes\":{}},{\"id\":\"h-misconceptions\",\"type\":\"header\",\"data\":{\"text\":\"Common misconceptions\",\"level\":2},\"tunes\":{}},{\"id\":\"misconceptions-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Misconception\",\"Correction\"],[\"“LLMOps replaces MLOps.”\",\"LLMOps extends MLOps principles to LLM-specific application behavior.\"],[\"“LLMOps is prompt engineering.”\",\"Prompts are one artifact among models, providers, context, retrieval, tools, evals and runtime.\"],[\"“Hosted APIs remove operations work.”\",\"They remove some model-serving\u002Ftraining work but add provider lifecycle, version and dependency management.\"],[\"“If the API is stable, the app is stable.”\",\"Model behavior and provider\u002Fmodel snapshots can change independently of API schema.\"],[\"“RAG is just data preprocessing.”\",\"In production it has its own ingestion, index, retrieval and freshness lifecycle.\"],[\"“LLM outputs cannot be tested.”\",\"They can be evaluated with deterministic, reference, judge and human criteria.\"],[\"“LLM judges are objective ground truth.”\",\"They are model-based evaluators that also require calibration and version control.\"],[\"“A local model eliminates LLMOps.”\",\"Local serving adds model files, VRAM, load\u002Funload, runtime health and upgrade concerns.\"],[\"“Observability means token counts.”\",\"Useful observability follows prompts, retrievals, tools, model spans and outcomes.\"],[\"“Continuous training is mandatory.”\",\"Many LLM apps use continuous evaluation without training the foundation model.\"]]},\"tunes\":{}},{\"id\":\"h-design\",\"type\":\"header\",\"data\":{\"text\":\"A practical LLMOps design sequence\",\"level\":2},\"tunes\":{}},{\"id\":\"design-flow\",\"type\":\"processFlow\",\"data\":{\"title\":\"Operate the complete behavior-producing system\",\"orientation\":\"auto\",\"steps\":[{\"label\":\"1. Define the behavior unit\",\"description\":\"List every component that can materially change output: model, prompt, retrieval, tools, context and policy.\"},{\"label\":\"2. Establish application lineage\",\"description\":\"Version code, model\u002Fprovider, prompts, eval datasets, retrieval configuration and tool contracts.\"},{\"label\":\"3. Build representative eval datasets\",\"description\":\"Use expected success\u002Ffailure cases from design and production.\"},{\"label\":\"4. Separate deterministic and behavioral tests\",\"description\":\"Keep schema\u002Fsecurity assertions distinct from semantic output evaluation.\"},{\"label\":\"5. Trace end-to-end execution\",\"description\":\"Instrument model, retrieval, reranking, tools and agent\u002Fruntime spans.\"},{\"label\":\"6. Define release gates\",\"description\":\"Set quality, safety, latency and cost thresholds.\"},{\"label\":\"7. Pin or explicitly record model versions\",\"description\":\"Treat model\u002Fprovider changes as release events.\"},{\"label\":\"8. Deploy progressively\",\"description\":\"Use flags, canaries or staged rollout where consequence warrants it.\"},{\"label\":\"9. Evaluate production traces\",\"description\":\"Measure real task behavior and identify recurrent failures.\"},{\"label\":\"10. Feed failures back into eval datasets\",\"description\":\"Turn incidents and corrections into permanent regression coverage.\"},{\"label\":\"11. Monitor provider and data lifecycles\",\"description\":\"Track deprecations, index freshness, source changes and runtime availability.\"},{\"label\":\"12. Retire obsolete versions cleanly\",\"description\":\"Remove old prompts\u002Fmodels\u002Findexes\u002Fcredentials after migration and evidence retention decisions.\"}]},\"tunes\":{}},{\"id\":\"h-checklist\",\"type\":\"header\",\"data\":{\"text\":\"LLMOps architecture checklist\",\"level\":2},\"tunes\":{}},{\"id\":\"checklist-table\",\"type\":\"table\",\"data\":{\"withHeadings\":true,\"stretched\":false,\"content\":[[\"Question\",\"Expected evidence\"],[\"Which model\u002Fprovider\u002Fversion served the request?\",\"Traceable model identity\"],[\"Which prompt\u002Finstructions were active?\",\"Versioned application code\u002Fconfig\"],[\"Which context reached the model?\",\"Context\u002Fretrieval trace\"],[\"Which corpus\u002Findex version was used?\",\"Retrieval lineage\"],[\"Which tools were available and called?\",\"Tool schema + trajectory trace\"],[\"Which permissions applied?\",\"Runtime authorization record\"],[\"How is quality measured?\",\"Versioned eval dataset + scorers\"],[\"How are model upgrades tested?\",\"Behavioral regression suite\"],[\"How is production quality sampled?\",\"Trace evaluation\u002Ffeedback process\"],[\"Can one failure be reproduced approximately?\",\"Model\u002Fcontext\u002Fprovider\u002Fapplication lineage\"],[\"Where is cost spent?\",\"Per-trace model\u002Ftool\u002Fretrieval attribution\"],[\"What triggers rollback?\",\"Defined quality\u002Fsafety\u002Fcost\u002Favailability threshold\"],[\"How are provider deprecations handled?\",\"Migration\u002Ffallback process\"],[\"How are local models operated?\",\"Health, resource, load\u002Funload and version controls\"]]},\"tunes\":{}},{\"id\":\"h-edge\",\"type\":\"header\",\"data\":{\"text\":\"Edge cases and limitations\",\"level\":2},\"tunes\":{}},{\"id\":\"p-edge-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"A simple application that calls one fixed hosted model with no retrieval or tools may need only lightweight LLMOps: versioned prompt code, evals, model pinning, basic tracing and provider monitoring.\"},\"tunes\":{}},{\"id\":\"p-edge-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"A self-hosted fine-tuned model may require nearly the full classical MLOps stack plus LLM-specific application evaluation, making the boundary between MLOps and LLMOps intentionally blurry.\"},\"tunes\":{}},{\"id\":\"p-edge-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"An agent platform can have minimal model-training operations but substantial runtime operations because failures occur in tool selection, state and orchestration.\"},\"tunes\":{}},{\"id\":\"p-edge-4\",\"type\":\"paragraph\",\"data\":{\"text\":\"A RAG-heavy system can be operationally dominated by document ingestion and retrieval quality rather than model serving.\"},\"tunes\":{}},{\"id\":\"p-edge-5\",\"type\":\"paragraph\",\"data\":{\"text\":\"Terminology will continue to evolve. The durable architecture question is not which “Ops” label wins, but which artifacts produce behavior and therefore must be versioned, evaluated, observed and governed.\"},\"tunes\":{}},{\"id\":\"h-change\",\"type\":\"header\",\"data\":{\"text\":\"What would change this answer?\",\"level\":2},\"tunes\":{}},{\"id\":\"p-change-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"If foundation-model providers standardize perfectly stable model behavior and long-term version support, provider\u002Fsnapshot management could become less operationally significant.\"},\"tunes\":{}},{\"id\":\"p-change-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"If applications increasingly own fine-tuning or training, classical MLOps concerns become more central again.\"},\"tunes\":{}},{\"id\":\"p-change-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The operational principle would remain: every component that can materially change production behavior belongs in lineage, testing, observability and change control.\"},\"tunes\":{}},{\"id\":\"h-related\",\"type\":\"header\",\"data\":{\"text\":\"Related canonical knowledge\",\"level\":2},\"tunes\":{}},{\"id\":\"p-related-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"LLMOps sits below AI Governance and Enterprise AI Architecture: governance defines which changes require evidence and approval, while LLMOps provides the operational machinery to version, evaluate, deploy and observe those changes.\"},\"tunes\":{}},{\"id\":\"p-related-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"Context Engineering and RAG are operational subdomains inside many LLM applications because context and retrieval can change behavior independently of the model.\"},\"tunes\":{}},{\"id\":\"p-related-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"Agentic AI extends LLMOps further into trajectory, permissions and tool-runtime operations.\"},\"tunes\":{}},{\"id\":\"ref-memory\",\"type\":\"referralArticle\",\"data\":{\"url\":\"https:\u002F\u002Fstajic.de\u002Fblog\u002Fai-agent-memory-is-not-rag-how-to-separate-memory-retrieval-state-and-context\",\"title\":\"AI Agent Memory Is Not RAG: How to Separate Memory, Retrieval, State and Context\",\"excerpt\":\"Operational reliability improves when memory, retrieval, application state and model context remain separate lifecycle objects.\",\"ctaLabel\":\"Read the architecture article\"},\"tunes\":{}},{\"id\":\"ref-avb\",\"type\":\"referralArticle\",\"data\":{\"url\":\"https:\u002F\u002Fstajic.de\u002Fblog\u002Fthe-answer-validity-boundary-the-missing-layer-between-relevance-and-reliable-ai-answers\",\"title\":\"The Answer Validity Boundary: The Missing Layer Between Relevance and Reliable AI Answers\",\"excerpt\":\"LLMOps evaluation should preserve the version, scope and evidence conditions under which an answer remains supported.\",\"ctaLabel\":\"Read the Answer Validity Boundary\"},\"tunes\":{}},{\"id\":\"h-faq\",\"type\":\"header\",\"data\":{\"text\":\"Frequently asked questions\",\"level\":2},\"tunes\":{}},{\"id\":\"faq\",\"type\":\"faq\",\"data\":{\"title\":\"MLOps vs LLMOps FAQ\",\"items\":[{\"id\":\"faq1\",\"question\":\"What is the difference between MLOps and LLMOps?\",\"answer\":\"MLOps operates machine-learning systems across data, training, deployment and monitoring. LLMOps extends those practices to LLM applications where prompts, context, retrieval, providers, tools and evaluations also materially affect behavior.\"},{\"id\":\"faq2\",\"question\":\"Does LLMOps replace MLOps?\",\"answer\":\"No. LLMOps reuses MLOps disciplines such as CI\u002FCD, lineage, evaluation, deployment and monitoring and adds LLM-specific operational concerns.\"},{\"id\":\"faq3\",\"question\":\"Do LLM applications need continuous training?\",\"answer\":\"Not necessarily. Many use external foundation models and instead rely on continuous evaluation of prompts, models, retrieval and application behavior. Fine-tuned or self-trained systems can still require training pipelines.\"},{\"id\":\"faq4\",\"question\":\"Why are evals so important in LLMOps?\",\"answer\":\"Generative outputs are open-ended and model behavior can change across prompts, snapshots and context. Evals provide repeatable evidence that a release still meets defined quality and safety criteria.\"},{\"id\":\"faq5\",\"question\":\"What should be versioned in LLMOps?\",\"answer\":\"At minimum: application code, model\u002Fprovider\u002Fversion, prompts, eval datasets\u002Fscorers, retrieval configuration\u002Findexes, tool schemas, context rules and relevant safety\u002Fpermission configuration.\"},{\"id\":\"faq6\",\"question\":\"Is prompt versioning enough?\",\"answer\":\"No. The same prompt can behave differently with another model, retrieval set, context order, tool surface or provider.\"},{\"id\":\"faq7\",\"question\":\"What is GenAIOps?\",\"answer\":\"GenAIOps is another industry term for operating generative-AI applications. Some vendors use it interchangeably or as a broader label than LLMOps.\"},{\"id\":\"faq8\",\"question\":\"How do you monitor an LLM application?\",\"answer\":\"Monitor end-to-end traces including model calls, prompts\u002Fcontext, retrieval, tools, latency, token\u002Fcost, quality samples, safety and final task outcomes.\"},{\"id\":\"faq9\",\"question\":\"Can local LLMs use LLMOps practices?\",\"answer\":\"Yes. Local models add their own operational concerns such as model files, hardware\u002FVRAM, load\u002Funload, runtime health, quantization and upgrade management.\"}]},\"tunes\":{}},{\"id\":\"h-glossary\",\"type\":\"header\",\"data\":{\"text\":\"Glossary\",\"level\":2},\"tunes\":{}},{\"id\":\"glossary\",\"type\":\"glossary\",\"data\":{\"title\":\"Key MLOps and LLMOps terms\",\"entries\":[{\"term\":\"MLOps\",\"definition\":\"Engineering practices for building, deploying, monitoring and maintaining machine-learning systems and their data\u002Fmodel lifecycle.\",\"anchor\":\"mlops\"},{\"term\":\"LLMOps\",\"definition\":\"Operational practices for production applications whose behavior materially depends on large language models and surrounding prompts, context, retrieval, tools and runtime.\",\"anchor\":\"llmops\"},{\"term\":\"GenAIOps\",\"definition\":\"Operational discipline for generative-AI applications; often used as a broader or alternate label for LLMOps.\",\"anchor\":\"genaiops\"},{\"term\":\"Continuous training\",\"definition\":\"Automated or repeated retraining and serving of ML models as data or implementations change.\",\"anchor\":\"continuous-training\"},{\"term\":\"Continuous evaluation\",\"definition\":\"Repeated evaluation of candidate and production AI behavior against versioned datasets and criteria.\",\"anchor\":\"continuous-evaluation\"},{\"term\":\"Model snapshot\",\"definition\":\"A concrete version of a hosted or packaged model whose behavior can be tested and referenced.\",\"anchor\":\"model-snapshot\"},{\"term\":\"Application lineage\",\"definition\":\"Traceable relationship among code, model\u002Fprovider, prompts, data\u002Fretrieval, tools, runtime and release configuration.\",\"anchor\":\"application-lineage\"},{\"term\":\"Trace\",\"definition\":\"Structured record of one application execution containing spans such as model calls, retrievals and tool operations.\",\"anchor\":\"trace\"},{\"term\":\"Eval dataset\",\"definition\":\"Versioned set of representative inputs, expectations and optionally traces\u002Foutputs used to measure behavior.\",\"anchor\":\"eval-dataset\"},{\"term\":\"LLM judge\",\"definition\":\"A language model used as an evaluator for qualitative or semantic criteria; it is itself a versioned evaluation dependency.\",\"anchor\":\"llm-judge\"},{\"term\":\"Behavioral regression\",\"definition\":\"A degradation in application output or trajectory despite interfaces and code continuing to execute successfully.\",\"anchor\":\"behavioral-regression\"},{\"term\":\"Provider routing\",\"definition\":\"Policy for selecting among available model providers\u002Fendpoints according to capability, cost, latency, privacy or availability.\",\"anchor\":\"provider-routing\"}]},\"tunes\":{}},{\"id\":\"h-conclusion\",\"type\":\"header\",\"data\":{\"text\":\"Conclusion\",\"level\":2},\"tunes\":{}},{\"id\":\"p-conclusion-1\",\"type\":\"paragraph\",\"data\":{\"text\":\"MLOps and LLMOps share the same engineering objective: make AI systems reproducible enough, testable enough and observable enough to operate reliably in production.\"},\"tunes\":{}},{\"id\":\"p-conclusion-2\",\"type\":\"paragraph\",\"data\":{\"text\":\"The difference is the shape of the system. Classical MLOps often centers on training and serving model artifacts; LLMOps must operate a behavioral stack in which model snapshots, prompts, context, retrieval, tools, permissions and providers can change independently.\"},\"tunes\":{}},{\"id\":\"p-conclusion-3\",\"type\":\"paragraph\",\"data\":{\"text\":\"The shortest useful rule is: version, evaluate and observe everything that can materially change the LLM application's behavior — not only the model.\"},\"tunes\":{}},{\"id\":\"h-sources\",\"type\":\"header\",\"data\":{\"text\":\"Primary sources and current documentation\",\"level\":2},\"tunes\":{}},{\"id\":\"p-sources-note\",\"type\":\"paragraph\",\"data\":{\"text\":\"The sources below ground the MLOps baseline and the current operational patterns for LLM and agent applications. Project sections are original implementation evidence and are intentionally narrower than claims about a complete LLMOps platform.\"},\"tunes\":{}},{\"id\":\"src-google-mlops\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdocs.cloud.google.com\u002Farchitecture\u002Fmlops-continuous-delivery-and-automation-pipelines-in-machine-learning\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"Google Cloud — MLOps: Continuous delivery and automation pipelines\",\"description\":\"Reference architecture describing CI, CD, continuous training, model registry, metadata, serving and monitoring for ML systems.\"}},\"tunes\":{}},{\"id\":\"src-aws-lineage\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops02-bp04.html\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"AWS Machine Learning Lens — Model lineage\",\"description\":\"Current guidance for tracking code, data, models, environments and infrastructure across ML releases.\"}},\"tunes\":{}},{\"id\":\"src-aws-monitor\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdocs.aws.amazon.com\u002Fwellarchitected\u002Flatest\u002Fmachine-learning-lens\u002Fmlops06-bp02.html\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"AWS Machine Learning Lens — Model observability and tracking\",\"description\":\"Current guidance for production model monitoring, drift, endpoint health and lineage.\"}},\"tunes\":{}},{\"id\":\"src-azure-llmops\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fmachine-learning\u002Fprompt-flow\u002Fhow-to-end-to-end-llmops-with-prompt-flow\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"Microsoft Azure — GenAIOps \u002F LLMOps lifecycle\",\"description\":\"Official guidance describing GenAIOps, sometimes called LLMOps, across initialization, experimentation, evaluation\u002Frefinement and deployment.\"}},\"tunes\":{}},{\"id\":\"src-mlflow-genai\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002F\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"MLflow — Agents and LLM applications\",\"description\":\"Current GenAI operations documentation covering tracing, evaluation, prompts and production observability for LLM applications and agents.\"}},\"tunes\":{}},{\"id\":\"src-mlflow-traces\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fwww.mlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Feval-monitor\u002Frunning-evaluation\u002Ftraces\u002F\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"MLflow — Evaluating production traces\",\"description\":\"Current guidance for evaluating complete LLM\u002Fagent traces, including retrieval and tool-call trajectories.\"}},\"tunes\":{}},{\"id\":\"src-mlflow-prompt-eval\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fmlflow.org\u002Fdocs\u002Flatest\u002Fgenai\u002Fprompt-registry\u002Fevaluate-prompts\u002F\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"MLflow — Evaluating prompts\",\"description\":\"Current prompt\u002Fmodel evaluation workflow using versioned prompts, datasets, scorers and traces.\"}},\"tunes\":{}},{\"id\":\"src-openai-api\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Freference\u002Foverview\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"OpenAI API — Versioning and model snapshots\",\"description\":\"Current API guidance recommending pinned model versions and evals because prompting behavior can change between snapshots.\"}},\"tunes\":{}},{\"id\":\"src-openai-prompting\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fguides\u002Fprompting\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"OpenAI — Prompting\",\"description\":\"Current guidance to treat production prompts as application code, version them through source control and cover changes with tests and evaluation checks.\"}},\"tunes\":{}},{\"id\":\"src-openai-deprecations\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fdeprecations\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"OpenAI — Deprecations\",\"description\":\"Current provider lifecycle evidence showing model and platform-surface retirement as an operational dependency.\"}},\"tunes\":{}},{\"id\":\"src-openai-promptfoo\",\"type\":\"linkTool\",\"data\":{\"link\":\"https:\u002F\u002Fdevelopers.openai.com\u002Fcookbook\u002Fexamples\u002Fevaluation\u002Fmoving-from-openai-evals-to-promptfoo\",\"meta\":{\"image\":{\"url\":\"\"},\"title\":\"OpenAI — Moving evaluation workflows to Promptfoo\",\"description\":\"Current 2026 migration guidance illustrating why evaluation assets should remain portable as provider tooling changes.\"}},\"tunes\":{}}],\"version\":\"2.31.6\"}",{"time":1604,"blocks":1605,"version":2720},1791487321430,[1606,1610,1615,1620,1625,1630,1634,1638,1642,1646,1650,1654,1658,1662,1666,1670,1674,1678,1682,1711,1715,1719,1723,1727,1730,1762,1766,1770,1774,1778,1782,1831,1835,1839,1843,1847,1851,1855,1859,1863,1867,1871,1875,1879,1883,1887,1891,1895,1899,1903,1907,1911,1918,1922,1926,1930,1934,1939,1943,1947,1951,1955,1959,1963,1967,1971,1975,1979,1983,1987,1994,1998,2002,2006,2010,2014,2018,2022,2026,2030,2034,2038,2042,2046,2079,2083,2087,2091,2095,2099,2103,2107,2111,2115,2148,2152,2156,2160,2164,2168,2172,2176,2180,2184,2188,2192,2196,2200,2204,2208,2212,2216,2220,2224,2228,2232,2236,2240,2244,2248,2252,2283,2288,2292,2335,2339,2376,2380,2421,2425,2474,2478,2482,2486,2490,2494,2498,2502,2506,2510,2514,2518,2522,2526,2530,2537,2544,2548,2579,2583,2619,2623,2627,2631,2635,2639,2643,2650,2657,2664,2671,2678,2685,2692,2699,2706,2713],{"id":215,"data":1607,"type":218,"tunes":1609},{"text":1608},"MLOps is the engineering discipline for reliably developing, deploying, versioning and operating machine-learning systems; LLMOps extends that discipline to applications built around large language models, where production behavior depends not only on a model artifact but also on prompts, context, retrieval, provider\u002Fmodel versions, tool calls, safety controls and evaluation pipelines. LLMOps does not replace MLOps. It changes the operational unit from “a model plus serving pipeline” toward “an evolving LLM application whose behavior emerges from several independently changing components.”",{},{"id":221,"data":1611,"type":226,"tunes":1614},{"body":1612,"title":1613,"variant":225},"\u003Cstrong>MLOps operates ML systems. LLMOps operates LLM applications.\u003C\u002Fstrong>\u003Cbr>\u003Cbr>Classical MLOps commonly centers on data pipelines, training, validation, model registry, deployment, drift and retraining. LLMOps keeps those disciplines where relevant, but often adds prompt\u002Fcontext versioning, model\u002Fprovider abstraction, RAG indexes, agent\u002Ftool traces, semantic evaluations, safety tests, token\u002Fcost monitoring and regression testing across rapidly changing model snapshots.","Direct answer",{},{"id":229,"data":1616,"type":226,"tunes":1619},{"body":1617,"title":1618,"variant":233},"A production LLM application can fail even when the prompt is unchanged: the provider can change a model snapshot, a RAG corpus can become stale, a reranker can regress, tool permissions can change, context assembly can drop evidence, or an agent can take a wrong trajectory. LLMOps therefore has to observe and version the system around the model, not only prompt text.","LLMOps is not just prompt management",{},{"id":236,"data":1621,"type":226,"tunes":1624},{"body":1622,"title":1623,"variant":240},"\u003Cstrong>LLMOps\u003C\u002Fstrong>, \u003Cstrong>GenAIOps\u003C\u002Fstrong> and related terms are widely used engineering labels, but they are not one universal formal standard with a single canonical lifecycle. Microsoft currently describes GenAIOps as “sometimes called LLMOps,” while MLflow groups operational tooling around agents and LLM applications. This article uses LLMOps as a practical architecture term for operating production systems whose behavior materially depends on LLMs.","Terminology boundary",{},{"id":243,"data":1626,"type":226,"tunes":1629},{"body":1627,"title":1628,"variant":240},"The operational surface is changing quickly. OpenAI currently recommends pinning model snapshots and running evals because prompting behavior can change between snapshots, and several older platform-specific prompt\u002Feval surfaces are being retired in 2026. The stable architectural lesson is to keep prompts, tests and evals portable and versioned with the application rather than depend on one provider's dashboard object model.","Current-source note — 8 October 2026",{},{"id":249,"data":1631,"type":254,"tunes":1633},{"title":1632,"maxLevel":252,"minLevel":253},"Contents",{},{"id":257,"data":1635,"type":42,"tunes":1637},{"text":1636,"level":253},"What MLOps really means",{},{"id":262,"data":1639,"type":218,"tunes":1641},{"text":1640},"MLOps applies software-engineering and operational discipline to machine-learning systems. The production challenge is broader than training a model: data collection, data validation, experimentation, reproducibility, model evaluation, deployment, infrastructure and monitoring all have to work together.",{},{"id":267,"data":1643,"type":218,"tunes":1645},{"text":1644},"Google's MLOps architecture guidance frames the discipline around continuous integration, continuous delivery and continuous training. CI validates not only code but also data, schemas and models; CD deploys ML pipelines and prediction services; CT can retrain and redeploy models as data or implementations change.",{},{"id":272,"data":1647,"type":218,"tunes":1649},{"text":1648},"AWS guidance adds the same operational concerns from another angle: model lineage, model\u002Fversion traceability, drift monitoring and production-quality monitoring are core parts of keeping ML systems reliable after deployment.",{},{"id":277,"data":1651,"type":42,"tunes":1653},{"text":1652,"level":253},"What changes when the model is an LLM",{},{"id":282,"data":1655,"type":218,"tunes":1657},{"text":1656},"Large language models change the production problem because the application often does not own the complete model-training lifecycle. A team may call a hosted model API, run an open model locally, switch between providers or use several models for different tasks.",{},{"id":287,"data":1659,"type":218,"tunes":1661},{"text":1660},"The model is therefore only one versioned dependency inside a larger behavioral system. Prompts, retrieval results, context order, tools, model snapshot, temperature\u002Freasoning settings, safety filters and runtime orchestration can all change the output.",{},{"id":292,"data":1663,"type":218,"tunes":1665},{"text":1664},"This creates a broader operational question: which combination of model, context, data, prompt, tools and runtime produced this behavior? LLMOps exists to make that question answerable and the answer reproducible enough for engineering work.",{},{"id":297,"data":1667,"type":42,"tunes":1669},{"text":1668,"level":253},"The simplest example",{},{"id":302,"data":1671,"type":218,"tunes":1673},{"text":1672},"Suppose an application answers internal policy questions.",{},{"id":307,"data":1675,"type":218,"tunes":1677},{"text":1676},"In a classical ML framing, you might version a trained classifier, deploy it and monitor prediction quality. In an LLM application, the answer might depend on a hosted model snapshot, a system prompt, an embedding model, a vector index, retrieval filters, a reranker and the final selected context.",{},{"id":312,"data":1679,"type":218,"tunes":1681},{"text":1680},"Changing any one of those components can change the final answer even though the application endpoint and user question stay identical.",{},{"id":317,"data":1683,"type":346,"tunes":1710},{"steps":1684,"title":1709,"orientation":345},[1685,1688,1691,1694,1697,1700,1703,1706],{"label":1686,"description":1687},"1. Change one component","Prompt, model, provider, retrieval setting, tool schema or application code changes.",{"label":1689,"description":1690},"2. Run deterministic tests","Validate schemas, permissions, tool contracts, retrieval filters and application behavior.",{"label":1692,"description":1693},"3. Run behavioral evals","Compare representative outputs, retrieval quality and agent\u002Ftool trajectories against acceptance criteria.",{"label":1695,"description":1696},"4. Compare cost and latency","Measure token use, model calls, retrieval\u002Ftool overhead and response latency.",{"label":1698,"description":1699},"5. Deploy controlled version","Ship the concrete application configuration with model\u002Fprovider versions recorded.",{"label":1701,"description":1702},"6. Trace production behavior","Capture relevant model, retrieval, tool and runtime spans.",{"label":1704,"description":1705},"7. Evaluate production traces","Sample real executions for quality, grounding, safety and task success.",{"label":1707,"description":1708},"8. Roll back or iterate","Use regression evidence and operational signals to decide the next release.","A typical LLMOps release path",{},{"id":349,"data":1712,"type":42,"tunes":1714},{"text":1713,"level":253},"Where the simple example stops",{},{"id":354,"data":1716,"type":218,"tunes":1718},{"text":1717},"Some LLM systems still train or fine-tune their own models, so traditional MLOps practices such as training pipelines, model registry and data lineage remain directly relevant.",{},{"id":359,"data":1720,"type":218,"tunes":1722},{"text":1721},"Other systems use only external foundation-model APIs and never run continuous training. Their main operational workload is application evaluation, model\u002Fprovider change management, prompt\u002Fcontext versioning, retrieval quality and observability.",{},{"id":364,"data":1724,"type":218,"tunes":1726},{"text":1725},"There is therefore no single universal “LLMOps pipeline.” The exact lifecycle depends on whether you train, fine-tune, self-host, retrieve external knowledge, run agents or depend on managed model APIs.",{},{"id":369,"data":1728,"type":42,"tunes":1729},{"text":371,"level":253},{},{"id":374,"data":1731,"type":419,"tunes":1761},{"rows":1732,"title":1757,"layout":411,"columns":1758},[1733,1736,1739,1742,1745,1748,1751,1754],{"id":378,"label":1734,"values":1735},"Primary operational unit",[381,381],{"id":383,"label":1737,"values":1738},"Model ownership",[381,381],{"id":387,"label":1740,"values":1741},"Typical change",[381,381],{"id":391,"label":1743,"values":1744},"Evaluation",[381,381],{"id":395,"label":1746,"values":1747},"Production monitoring",[381,381],{"id":399,"label":1749,"values":1750},"Continuous training",[381,381],{"id":403,"label":1752,"values":1753},"Versioned artifacts",[381,381],{"id":407,"label":1755,"values":1756},"Rollback target",[381,381],"What stays the same and what expands",[1759,1760],{"id":414,"label":415},{"id":417,"label":418},{},{"id":422,"data":1763,"type":42,"tunes":1765},{"text":1764,"level":253},"LLMOps extends MLOps rather than replacing it",{},{"id":427,"data":1767,"type":218,"tunes":1769},{"text":1768},"The core operational principles do not disappear: source control, CI\u002FCD, reproducibility, lineage, deployment controls, monitoring, rollback and measurable acceptance criteria remain essential.",{},{"id":432,"data":1771,"type":218,"tunes":1773},{"text":1772},"The extension is that more behavior-defining artifacts now sit outside the model weights. A managed foundation model can change behavior through snapshot upgrades, while application output can change through prompt or retrieval changes without any model retraining.",{},{"id":437,"data":1775,"type":218,"tunes":1777},{"text":1776},"This is why the useful hierarchy is usually DevOps → MLOps → LLMOps\u002FGenAIOps as increasingly specialized operational concerns, not three mutually exclusive practices.",{},{"id":442,"data":1779,"type":42,"tunes":1781},{"text":1780,"level":253},"What has to be versioned in LLMOps?",{},{"id":447,"data":1783,"type":411,"tunes":1830},{"content":1784,"stretched":43,"withHeadings":14},[1785,1788,1791,1794,1797,1800,1803,1806,1809,1812,1815,1818,1821,1824,1827],[1786,1787],"Artifact","Why it matters",[1789,1790],"Application code","Defines orchestration, validation, retries and business behavior",[1792,1793],"Model family + snapshot\u002Fversion","Different snapshots can produce different behavior",[1795,1796],"Provider \u002F endpoint","Changes data flow, latency, limits, pricing and availability",[1798,1799],"Prompt\u002Finstruction code","Changes model behavior even with same model",[1801,1802],"Generation\u002Freasoning parameters","Can alter determinism, latency, depth and cost",[1804,1805],"Eval dataset","Defines what “good enough” is tested against",[1807,1808],"Scorers \u002F graders","Define how quality is measured",[1810,1811],"Embedding model","Changes vector representation and retrieval behavior",[1813,1814],"Chunking\u002Findex configuration","Changes what can be retrieved",[1816,1817],"Reranker \u002F retrieval fusion","Changes result ordering",[1819,1820],"Tool schemas","Change what the model can request and how",[1822,1823],"Permission profile","Changes what tool actions may actually execute",[1825,1826],"Context assembly rules","Change what evidence and state reach the model",[1828,1829],"Safety\u002Fguardrail configuration","Changes allowed or blocked behavior",{},{"id":497,"data":1832,"type":42,"tunes":1834},{"text":1833,"level":253},"Model snapshots become release dependencies",{},{"id":502,"data":1836,"type":218,"tunes":1838},{"text":1837},"With hosted LLMs, the team may not control model training, but it still controls which model or snapshot the application calls.",{},{"id":507,"data":1840,"type":218,"tunes":1842},{"text":1841},"OpenAI's current API guidance explicitly warns that prompting behavior can change between model snapshots and recommends pinning production applications to specific snapshots where consistency matters, then running evals when upgrading.",{},{"id":512,"data":1844,"type":218,"tunes":1846},{"text":1845},"The operational consequence is straightforward: model upgrades should be treated as application releases, not invisible infrastructure maintenance.",{},{"id":517,"data":1848,"type":42,"tunes":1850},{"text":1849,"level":253},"Provider lifecycle becomes part of operations",{},{"id":522,"data":1852,"type":218,"tunes":1854},{"text":1853},"LLM applications often depend on provider rate limits, deprecation schedules, API semantics, context limits, data-handling rules and pricing.",{},{"id":527,"data":1856,"type":218,"tunes":1858},{"text":1857},"A provider can deprecate a model while your application code remains unchanged. OpenAI's current deprecation schedule, for example, includes 2026 retirement dates for older model snapshots and platform surfaces.",{},{"id":532,"data":1860,"type":218,"tunes":1862},{"text":1861},"LLMOps therefore needs provider lifecycle tracking, migration testing and fallback decisions in addition to model-quality monitoring.",{},{"id":537,"data":1864,"type":42,"tunes":1866},{"text":1865,"level":253},"Prompts behave like production code",{},{"id":542,"data":1868,"type":218,"tunes":1870},{"text":1869},"Prompts are executable behavioral configuration. Small changes can alter output quality, tool selection and policy interpretation.",{},{"id":547,"data":1872,"type":218,"tunes":1874},{"text":1873},"OpenAI's current guidance recommends storing production prompts in application code, reviewing prompt changes through pull requests, using typed inputs and covering changes with tests and evaluation checks.",{},{"id":552,"data":1876,"type":218,"tunes":1878},{"text":1877},"That makes prompt versioning less like editing marketing copy and more like changing a function whose output is probabilistic and model-dependent.",{},{"id":557,"data":1880,"type":42,"tunes":1882},{"text":1881,"level":253},"Context engineering becomes an operational concern",{},{"id":562,"data":1884,"type":218,"tunes":1886},{"text":1885},"The production model rarely receives only a static prompt. It may receive conversation history, retrieved documents, tool outputs, memory, current application state and policy instructions.",{},{"id":567,"data":1888,"type":218,"tunes":1890},{"text":1889},"LLMOps must therefore observe context assembly: which evidence was selected, which state version was current, whether truncation occurred and whether important instructions survived compaction.",{},{"id":572,"data":1892,"type":218,"tunes":1894},{"text":1893},"A model regression and a context regression can look identical at the final answer. Tracing the actual context path is what lets the team separate them.",{},{"id":577,"data":1896,"type":42,"tunes":1898},{"text":1897,"level":253},"RAG creates its own operational lifecycle",{},{"id":582,"data":1900,"type":218,"tunes":1902},{"text":1901},"A RAG system introduces a second production pipeline beside model inference: ingestion, extraction, chunking, metadata, embeddings, indexes, retrieval, reranking and context selection.",{},{"id":587,"data":1904,"type":218,"tunes":1906},{"text":1905},"The knowledge corpus can change every day even when the model and prompt do not. A stale index or broken metadata filter can therefore degrade answer quality without any model drift.",{},{"id":592,"data":1908,"type":218,"tunes":1910},{"text":1909},"LLMOps for RAG should track corpus\u002Findex version, embedding model, chunking policy, retrieval configuration, source freshness and retrieval metrics separately from generation quality.",{},{"id":597,"data":1912,"type":603,"tunes":1917},{"url":1913,"title":1914,"excerpt":1915,"ctaLabel":1916},"https:\u002F\u002Fstajic.de\u002Fblog\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method","RAG Failed — But Which Layer Actually Failed? A Diagnostic Method","A production LLM pipeline needs separate observability for source coverage, retrieval, ranking, context assembly and generation.","Read the RAG diagnostic method",{},{"id":606,"data":1919,"type":42,"tunes":1921},{"text":1920,"level":253},"Evals replace “looks good to me” with release evidence",{},{"id":611,"data":1923,"type":218,"tunes":1925},{"text":1924},"Generative outputs are often open-ended, so exact-match tests are insufficient for many tasks. LLMOps adds evaluation datasets and scorers that can measure task success, correctness, safety, groundedness, style or domain-specific acceptance criteria.",{},{"id":616,"data":1927,"type":218,"tunes":1929},{"text":1928},"MLflow's current GenAI evaluation stack supports versioned evaluation datasets, prompt\u002Fmodel comparisons, custom scorers and evaluation over complete traces.",{},{"id":621,"data":1931,"type":218,"tunes":1933},{"text":1932},"The strongest practice is evaluation-driven development: define representative cases and acceptance criteria before or alongside changes, then compare releases against the same evidence.",{},{"id":626,"data":1935,"type":226,"tunes":1938},{"body":1936,"title":1937,"variant":630},"A deployment should not be considered equivalent merely because the API contract still works. If the prompt, model, retrieval or tools changed, the behavioral regression suite should run again.","Behavioral changes need behavioral tests",{},{"id":633,"data":1940,"type":42,"tunes":1942},{"text":1941,"level":253},"LLM-as-a-judge is useful but not ground truth",{},{"id":638,"data":1944,"type":218,"tunes":1946},{"text":1945},"LLM judges can scale evaluation for qualities that are expensive to encode as deterministic assertions, such as relevance, tone or groundedness.",{},{"id":643,"data":1948,"type":218,"tunes":1950},{"text":1949},"However, the judge is another model with its own bias, version and prompt. Judge configuration should therefore be versioned and calibrated against human or deterministic reference cases where consequence matters.",{},{"id":648,"data":1952,"type":218,"tunes":1954},{"text":1953},"A production eval can mix deterministic checks, reference-based metrics, model judges and human review rather than asking one metric to represent every quality dimension.",{},{"id":653,"data":1956,"type":42,"tunes":1958},{"text":1957,"level":253},"Tracing becomes more important than endpoint logs",{},{"id":658,"data":1960,"type":218,"tunes":1962},{"text":1961},"Traditional API logs can tell you that a request took two seconds and returned HTTP 200. They cannot tell you which retrieved chunks were selected, which tool the agent called or which model span consumed most tokens.",{},{"id":663,"data":1964,"type":218,"tunes":1966},{"text":1965},"MLflow's current GenAI tracing captures prompts, retrievals, tool calls and application spans, and its production evaluation flow can score intermediate trajectory information rather than only final text.",{},{"id":668,"data":1968,"type":218,"tunes":1970},{"text":1969},"This is a major LLMOps shift: observability follows the behavioral graph of the application, not only the serving endpoint.",{},{"id":673,"data":1972,"type":42,"tunes":1974},{"text":1973,"level":253},"Agents expand LLMOps into runtime operations",{},{"id":678,"data":1976,"type":218,"tunes":1978},{"text":1977},"An agentic application can perform several model calls, tool invocations and state transitions before producing a result.",{},{"id":683,"data":1980,"type":218,"tunes":1982},{"text":1981},"Operating agents therefore requires step counts, tool-call traces, permission denials, retries, loop detection, human approvals and verified final state in addition to ordinary model latency and token metrics.",{},{"id":688,"data":1984,"type":218,"tunes":1986},{"text":1985},"A correct final answer can hide a bad trajectory, so agent evaluation must inspect the path as well as the result.",{},{"id":693,"data":1988,"type":603,"tunes":1993},{"url":1989,"title":1990,"excerpt":1991,"ctaLabel":1992},"https:\u002F\u002Fstajic.de\u002Fblog\u002Fai-agent-reliability-why-the-final-answer-is-not-enough","AI Agent Reliability: Why the Final Answer Is Not Enough","Why agent production evaluation must include tool calls, state transitions, approvals and recoverability.","Read the agent reliability article",{},{"id":701,"data":1995,"type":42,"tunes":1997},{"text":1996,"level":253},"Tokens, model calls and context become cost variables",{},{"id":706,"data":1999,"type":218,"tunes":2001},{"text":2000},"Classical ML inference cost is often dominated by serving infrastructure or per-prediction compute. LLM applications can add provider token pricing, repeated agent calls, embedding calls, reranking and tool\u002Fruntime overhead.",{},{"id":711,"data":2003,"type":218,"tunes":2005},{"text":2004},"Cost therefore has to be attributed to task or trace, not only to one endpoint. A workflow that makes eight hidden model calls can be functionally correct but operationally unacceptable.",{},{"id":716,"data":2007,"type":218,"tunes":2009},{"text":2008},"Latency behaves the same way: model latency, retrieval, reranking and external tools compose into end-to-end user latency.",{},{"id":721,"data":2011,"type":42,"tunes":2013},{"text":2012,"level":253},"Caching becomes semantic, not only technical",{},{"id":726,"data":2015,"type":218,"tunes":2017},{"text":2016},"LLM systems can cache prompts, embeddings, retrieval results or full responses, but the cache key must reflect the semantics that can change the result.",{},{"id":731,"data":2019,"type":218,"tunes":2021},{"text":2020},"A response cache that ignores model version, tenant, permissions or source freshness can return a technically valid but semantically invalid answer.",{},{"id":736,"data":2023,"type":218,"tunes":2025},{"text":2024},"LLMOps therefore treats cache invalidation as part of model\u002Fcontext\u002Fdata versioning rather than only infrastructure optimization.",{},{"id":741,"data":2027,"type":42,"tunes":2029},{"text":2028,"level":253},"Safety and permissions become release criteria",{},{"id":746,"data":2031,"type":218,"tunes":2033},{"text":2032},"Generative systems can produce unbounded text and agents can trigger external actions. Safety testing therefore sits closer to ordinary CI\u002FCD than in many classical predictive ML systems.",{},{"id":751,"data":2035,"type":218,"tunes":2037},{"text":2036},"Permission checks, prompt-injection tests, tenant-isolation tests and side-effect approvals should be reproducible regression tests where those risks exist.",{},{"id":756,"data":2039,"type":218,"tunes":2041},{"text":2040},"The model may suggest an operation, but the runtime still has to enforce authorization. LLMOps owns the evidence that those controls continue to work after model, prompt or tool changes.",{},{"id":761,"data":2043,"type":42,"tunes":2045},{"text":2044,"level":253},"What CI looks like in LLMOps",{},{"id":766,"data":2047,"type":411,"tunes":2078},{"content":2048,"stretched":43,"withHeadings":14},[2049,2052,2055,2058,2061,2063,2066,2069,2072,2075],[2050,2051],"CI layer","Example checks",[2053,2054],"Code","Unit tests, type checks, schema validation",[2056,2057],"Prompts","Template rendering, required variables, policy text, snapshot review",[2059,2060],"Models\u002Fproviders","Compatibility, output schema, capability and regression tests",[782,2062],"Chunking fixtures, filter tests, Recall@k, reranker regression",[2064,2065],"Tools","Input\u002Foutput schema tests, permission tests, idempotency tests",[2067,2068],"Agents","Trajectory fixtures, loop limits, handoff\u002Ftool-selection tests",[2070,2071],"Security","Prompt injection, unauthorized tools, cross-tenant negative tests",[2073,2074],"Behavioral evals","Task success, correctness, grounding, safety, domain criteria",[2076,2077],"Operational","Latency, token\u002Fcost budgets, timeout\u002Ffallback behavior",{},{"id":801,"data":2080,"type":42,"tunes":2082},{"text":2081,"level":253},"What CD looks like in LLMOps",{},{"id":806,"data":2084,"type":218,"tunes":2086},{"text":2085},"A production release may deploy no new model artifact at all. It may simply ship a new prompt, retrieval configuration, tool set or provider mapping.",{},{"id":811,"data":2088,"type":218,"tunes":2090},{"text":2089},"The release bundle should therefore identify the complete behavior-defining configuration rather than only the application container image.",{},{"id":816,"data":2092,"type":218,"tunes":2094},{"text":2093},"Feature flags, staged rollout, shadow evaluation, canary traffic and rollback are useful because LLM behavior can regress in ways that static contract tests do not detect.",{},{"id":821,"data":2096,"type":42,"tunes":2098},{"text":2097,"level":253},"Continuous training becomes optional; continuous evaluation becomes central",{},{"id":826,"data":2100,"type":218,"tunes":2102},{"text":2101},"Traditional MLOps often emphasizes continuous training when new data or drift justifies retraining.",{},{"id":831,"data":2104,"type":218,"tunes":2106},{"text":2105},"Many LLM applications never train the foundation model. Their equivalent continuous loop is continuous evaluation: collect failures and representative production cases, add them to evaluation datasets, test candidate prompt\u002Fmodel\u002Fretrieval changes and redeploy only when evidence improves.",{},{"id":836,"data":2108,"type":218,"tunes":2110},{"text":2109},"Fine-tuning can reintroduce a training lifecycle, but it should sit inside the same broader evaluation and release process.",{},{"id":841,"data":2112,"type":42,"tunes":2114},{"text":2113,"level":253},"What should be monitored in production?",{},{"id":846,"data":2116,"type":411,"tunes":2147},{"content":2117,"stretched":43,"withHeadings":14},[2118,2121,2124,2126,2129,2132,2135,2137,2139,2141,2144],[2119,2120],"Signal class","Examples",[2122,2123],"System health","Errors, timeouts, endpoint availability",[856,2125],"Model ID, snapshot, rate limits, provider errors",[2127,2128],"Latency","End-to-end, model, retrieval, tool and reranker spans",[2130,2131],"Cost","Input\u002Foutput tokens, embeddings, tool\u002FAPI spend",[2133,2134],"Quality","Sampled task success, correctness, relevance, groundedness",[782,2136],"Retrieval recall proxies, empty retrieval, stale sources, citation coverage",[2067,2138],"Tool selection, retries, loops, handoffs, approval frequency",[2070,2140],"Denied actions, prompt-injection indicators, tenant-boundary failures",[2142,2143],"User feedback","Corrections, abandonment, escalation, explicit ratings",[2145,2146],"Change drift","Provider\u002Fmodel\u002Fconfig changes relative to approved release",{},{"id":881,"data":2149,"type":42,"tunes":2151},{"text":2150,"level":253},"Production traces can become evaluation data",{},{"id":886,"data":2153,"type":218,"tunes":2155},{"text":2154},"One of the most useful modern LLMOps patterns is to turn sampled production traces into evaluation records.",{},{"id":891,"data":2157,"type":218,"tunes":2159},{"text":2158},"MLflow currently supports retrieving production traces and scoring not only outputs but intermediate spans such as retrieval or tool-call trajectories.",{},{"id":896,"data":2161,"type":218,"tunes":2163},{"text":2162},"This closes the loop between observability and development: real failures can become regression cases in the next release rather than disappear inside logs.",{},{"id":901,"data":2165,"type":42,"tunes":2167},{"text":2166,"level":253},"Reproducibility becomes conditional rather than exact",{},{"id":906,"data":2169,"type":218,"tunes":2171},{"text":2170},"Classical ML reproducibility often aims to recreate a model from versioned code, data, environment and training parameters.",{},{"id":911,"data":2173,"type":218,"tunes":2175},{"text":2174},"Hosted LLM applications cannot always reproduce identical output token-for-token because generation is probabilistic and providers may control infrastructure.",{},{"id":916,"data":2177,"type":218,"tunes":2179},{"text":2178},"LLMOps therefore aims for behavioral reproducibility: record enough model\u002Fprovider\u002Fversion, prompt, context inputs, retrieval state and runtime configuration to reproduce the conditions and validate behavior within expected tolerances.",{},{"id":921,"data":2181,"type":42,"tunes":2183},{"text":2182,"level":253},"Lineage expands from model lineage to application lineage",{},{"id":926,"data":2185,"type":218,"tunes":2187},{"text":2186},"AWS's MLOps guidance treats model lineage as the history of code, data, model and infrastructure artifacts needed for diagnosis and reproducibility.",{},{"id":931,"data":2189,"type":218,"tunes":2191},{"text":2190},"For LLM applications, lineage should additionally connect prompts, eval datasets, retrieval\u002Findex versions, tool schemas, agent\u002Fruntime configuration and provider\u002Fmodel snapshots.",{},{"id":936,"data":2193,"type":218,"tunes":2195},{"text":2194},"The target question becomes: Which exact application configuration produced this trace?",{},{"id":941,"data":2197,"type":42,"tunes":2199},{"text":2198,"level":253},"Multi-provider and model routing create operational policy",{},{"id":946,"data":2201,"type":218,"tunes":2203},{"text":2202},"Once an application can use several providers or local models, routing becomes an operational policy rather than a simple model string.",{},{"id":951,"data":2205,"type":218,"tunes":2207},{"text":2206},"Routing may depend on capability, latency, cost, privacy, context length, availability, tool support or locality. A fallback can preserve uptime while changing answer quality or data-processing assumptions.",{},{"id":956,"data":2209,"type":218,"tunes":2211},{"text":2210},"LLMOps should therefore log which route was actually selected and evaluate routes independently rather than treat every compatible endpoint as behaviorally interchangeable.",{},{"id":961,"data":2213,"type":42,"tunes":2215},{"text":2214,"level":253},"Original implementation evidence",{},{"id":966,"data":2217,"type":42,"tunes":2219},{"text":2218,"level":252},"Aaasaasa AI Client: provider, model and runtime are separate operational objects",{},{"id":971,"data":2221,"type":218,"tunes":2223},{"text":2222},"Aaasaasa AI Client separates agent\u002Fclient, provider, model, runtime location and permissions. Its AI Hub supports Ollama, LM Studio\u002FOpenAI-compatible endpoints and other provider protocols rather than treating “the model” as one global setting.",{},{"id":976,"data":2225,"type":218,"tunes":2227},{"text":2226},"The implementation includes dynamic local model discovery, streaming, thinking output and explicit Ollama warm\u002Fload and unload controls. That is operational evidence that local LLM serving introduces resource lifecycle concerns beyond an API model name.",{},{"id":981,"data":2229,"type":218,"tunes":2231},{"text":2230},"Provider status is queried through provider adapters, and connection types distinguish local, cloud API, account-backed, remote-agent and web-client paths. These are concrete operational dimensions an LLM-aware platform has to surface.",{},{"id":986,"data":2233,"type":218,"tunes":2235},{"text":2234},"The repository also preserves an important boundary: a local runtime is not automatically local inference. Provider\u002Fmodel\u002Fruntime location are versioned or configurable concerns that affect privacy, latency, cost and availability.",{},{"id":991,"data":2237,"type":42,"tunes":2239},{"text":2238,"level":252},"Source of Truth Research Engine: LLM application state extends beyond the model",{},{"id":996,"data":2241,"type":218,"tunes":2243},{"text":2242},"The Source of Truth Research Engine combines lexical search, optional embeddings, source snapshots, SHA-256 identity, claims, provenance and contradiction tracking around local model-assisted research.",{},{"id":1001,"data":2245,"type":218,"tunes":2247},{"text":2246},"This is useful LLMOps evidence because changing the model alone does not define the research system. Retrieval, source acquisition, evidence classification and persistent provenance are independent operational artifacts.",{},{"id":1006,"data":2249,"type":218,"tunes":2251},{"text":2250},"The implementation deliberately treats semantic similarity as discovery rather than evidence, showing why LLMOps observability should distinguish retrieval behavior from claim validity.",{},{"id":1011,"data":2253,"type":411,"tunes":2282},{"content":2254,"stretched":43,"withHeadings":14},[2255,2258,2261,2264,2267,2270,2273,2276,2279],[2256,2257],"Observed implementation","LLMOps lesson",[2259,2260],"Multiple provider protocols","Provider identity is an operational dependency",[2262,2263],"Dynamic model discovery","Available models can change independently of application code",[2265,2266],"Ollama load\u002Funload controls","Local models have memory\u002Fresource lifecycle",[2268,2269],"Provider health\u002Fstatus adapters","Model availability needs runtime observability",[2271,2272],"Separate runtime and inference location","Deployment topology is not one boolean “local\u002Fcloud”",[2274,2275],"Central permissions","Model capability and tool authority must remain separate",[2277,2278],"Lexical + semantic retrieval pipeline","Retrieval configuration is part of application behavior",[2280,2281],"Source\u002Fprovenance persistence","Operational state and evidence live outside model weights",{},{"id":1043,"data":2284,"type":226,"tunes":2287},{"body":2285,"title":2286,"variant":240},"These projects demonstrate multi-provider\u002Flocal-model operations, permission separation, retrieval infrastructure and evidence persistence. They are not presented as a complete commercial LLMOps platform or proof of large-scale production traffic.","Evidence boundary",{},{"id":1049,"data":2289,"type":42,"tunes":2291},{"text":2290,"level":253},"Common LLMOps failure modes",{},{"id":1054,"data":2293,"type":411,"tunes":2334},{"content":2294,"stretched":43,"withHeadings":14},[2295,2298,2301,2304,2307,2310,2313,2316,2319,2322,2325,2328,2331],[2296,2297],"Failure mode","What actually went wrong",[2299,2300],"Model alias upgraded silently","Behavior changed without controlled release",[2302,2303],"Prompt changed without evals","Behavioral regression passed normal unit tests",[2305,2306],"RAG index stale","Generation model was blamed for retrieval\u002Fdata failure",[2308,2309],"Only final answer is logged","Root cause in retrieval\u002Ftool\u002Fcontext trajectory is invisible",[2311,2312],"Provider fallback is silent","Different model\u002Fdata path changes behavior without attribution",[2314,2315],"Token cost tracked globally","Expensive workflows cannot be localized",[2317,2318],"Judge model changed","Evaluation scores drift without application change",[2320,2321],"Production traces never become tests","Known failures repeatedly return",[2323,2324],"Local model stays loaded indefinitely","VRAM\u002Fresource pressure becomes operational instability",[2326,2327],"Permissions encoded only in prompt","Model behavior is mistaken for authorization",[2329,2330],"One eval score gates everything","Different quality dimensions are collapsed into a misleading number",[2332,2333],"Model registry exists but prompt\u002Findex versions do not","Application lineage remains incomplete",{},{"id":1098,"data":2336,"type":42,"tunes":2338},{"text":2337,"level":253},"Common misconceptions",{},{"id":1103,"data":2340,"type":411,"tunes":2375},{"content":2341,"stretched":43,"withHeadings":14},[2342,2345,2348,2351,2354,2357,2360,2363,2366,2369,2372],[2343,2344],"Misconception","Correction",[2346,2347],"“LLMOps replaces MLOps.”","LLMOps extends MLOps principles to LLM-specific application behavior.",[2349,2350],"“LLMOps is prompt engineering.”","Prompts are one artifact among models, providers, context, retrieval, tools, evals and runtime.",[2352,2353],"“Hosted APIs remove operations work.”","They remove some model-serving\u002Ftraining work but add provider lifecycle, version and dependency management.",[2355,2356],"“If the API is stable, the app is stable.”","Model behavior and provider\u002Fmodel snapshots can change independently of API schema.",[2358,2359],"“RAG is just data preprocessing.”","In production it has its own ingestion, index, retrieval and freshness lifecycle.",[2361,2362],"“LLM outputs cannot be tested.”","They can be evaluated with deterministic, reference, judge and human criteria.",[2364,2365],"“LLM judges are objective ground truth.”","They are model-based evaluators that also require calibration and version control.",[2367,2368],"“A local model eliminates LLMOps.”","Local serving adds model files, VRAM, load\u002Funload, runtime health and upgrade concerns.",[2370,2371],"“Observability means token counts.”","Useful observability follows prompts, retrievals, tools, model spans and outcomes.",[2373,2374],"“Continuous training is mandatory.”","Many LLM apps use continuous evaluation without training the foundation model.",{},{"id":1141,"data":2377,"type":42,"tunes":2379},{"text":2378,"level":253},"A practical LLMOps design sequence",{},{"id":1146,"data":2381,"type":346,"tunes":2420},{"steps":2382,"title":2419,"orientation":345},[2383,2386,2389,2392,2395,2398,2401,2404,2407,2410,2413,2416],{"label":2384,"description":2385},"1. Define the behavior unit","List every component that can materially change output: model, prompt, retrieval, tools, context and policy.",{"label":2387,"description":2388},"2. Establish application lineage","Version code, model\u002Fprovider, prompts, eval datasets, retrieval configuration and tool contracts.",{"label":2390,"description":2391},"3. Build representative eval datasets","Use expected success\u002Ffailure cases from design and production.",{"label":2393,"description":2394},"4. Separate deterministic and behavioral tests","Keep schema\u002Fsecurity assertions distinct from semantic output evaluation.",{"label":2396,"description":2397},"5. Trace end-to-end execution","Instrument model, retrieval, reranking, tools and agent\u002Fruntime spans.",{"label":2399,"description":2400},"6. Define release gates","Set quality, safety, latency and cost thresholds.",{"label":2402,"description":2403},"7. Pin or explicitly record model versions","Treat model\u002Fprovider changes as release events.",{"label":2405,"description":2406},"8. Deploy progressively","Use flags, canaries or staged rollout where consequence warrants it.",{"label":2408,"description":2409},"9. Evaluate production traces","Measure real task behavior and identify recurrent failures.",{"label":2411,"description":2412},"10. Feed failures back into eval datasets","Turn incidents and corrections into permanent regression coverage.",{"label":2414,"description":2415},"11. Monitor provider and data lifecycles","Track deprecations, index freshness, source changes and runtime availability.",{"label":2417,"description":2418},"12. Retire obsolete versions cleanly","Remove old prompts\u002Fmodels\u002Findexes\u002Fcredentials after migration and evidence retention decisions.","Operate the complete behavior-producing system",{},{"id":1188,"data":2422,"type":42,"tunes":2424},{"text":2423,"level":253},"LLMOps architecture checklist",{},{"id":1193,"data":2426,"type":411,"tunes":2473},{"content":2427,"stretched":43,"withHeadings":14},[2428,2431,2434,2437,2440,2443,2446,2449,2452,2455,2458,2461,2464,2467,2470],[2429,2430],"Question","Expected evidence",[2432,2433],"Which model\u002Fprovider\u002Fversion served the request?","Traceable model identity",[2435,2436],"Which prompt\u002Finstructions were active?","Versioned application code\u002Fconfig",[2438,2439],"Which context reached the model?","Context\u002Fretrieval trace",[2441,2442],"Which corpus\u002Findex version was used?","Retrieval lineage",[2444,2445],"Which tools were available and called?","Tool schema + trajectory trace",[2447,2448],"Which permissions applied?","Runtime authorization record",[2450,2451],"How is quality measured?","Versioned eval dataset + scorers",[2453,2454],"How are model upgrades tested?","Behavioral regression suite",[2456,2457],"How is production quality sampled?","Trace evaluation\u002Ffeedback process",[2459,2460],"Can one failure be reproduced approximately?","Model\u002Fcontext\u002Fprovider\u002Fapplication lineage",[2462,2463],"Where is cost spent?","Per-trace model\u002Ftool\u002Fretrieval attribution",[2465,2466],"What triggers rollback?","Defined quality\u002Fsafety\u002Fcost\u002Favailability threshold",[2468,2469],"How are provider deprecations handled?","Migration\u002Ffallback process",[2471,2472],"How are local models operated?","Health, resource, load\u002Funload and version controls",{},{"id":1243,"data":2475,"type":42,"tunes":2477},{"text":2476,"level":253},"Edge cases and limitations",{},{"id":1248,"data":2479,"type":218,"tunes":2481},{"text":2480},"A simple application that calls one fixed hosted model with no retrieval or tools may need only lightweight LLMOps: versioned prompt code, evals, model pinning, basic tracing and provider monitoring.",{},{"id":1253,"data":2483,"type":218,"tunes":2485},{"text":2484},"A self-hosted fine-tuned model may require nearly the full classical MLOps stack plus LLM-specific application evaluation, making the boundary between MLOps and LLMOps intentionally blurry.",{},{"id":1258,"data":2487,"type":218,"tunes":2489},{"text":2488},"An agent platform can have minimal model-training operations but substantial runtime operations because failures occur in tool selection, state and orchestration.",{},{"id":1263,"data":2491,"type":218,"tunes":2493},{"text":2492},"A RAG-heavy system can be operationally dominated by document ingestion and retrieval quality rather than model serving.",{},{"id":1268,"data":2495,"type":218,"tunes":2497},{"text":2496},"Terminology will continue to evolve. The durable architecture question is not which “Ops” label wins, but which artifacts produce behavior and therefore must be versioned, evaluated, observed and governed.",{},{"id":1273,"data":2499,"type":42,"tunes":2501},{"text":2500,"level":253},"What would change this answer?",{},{"id":1278,"data":2503,"type":218,"tunes":2505},{"text":2504},"If foundation-model providers standardize perfectly stable model behavior and long-term version support, provider\u002Fsnapshot management could become less operationally significant.",{},{"id":1283,"data":2507,"type":218,"tunes":2509},{"text":2508},"If applications increasingly own fine-tuning or training, classical MLOps concerns become more central again.",{},{"id":1288,"data":2511,"type":218,"tunes":2513},{"text":2512},"The operational principle would remain: every component that can materially change production behavior belongs in lineage, testing, observability and change control.",{},{"id":1293,"data":2515,"type":42,"tunes":2517},{"text":2516,"level":253},"Related canonical knowledge",{},{"id":1298,"data":2519,"type":218,"tunes":2521},{"text":2520},"LLMOps sits below AI Governance and Enterprise AI Architecture: governance defines which changes require evidence and approval, while LLMOps provides the operational machinery to version, evaluate, deploy and observe those changes.",{},{"id":1303,"data":2523,"type":218,"tunes":2525},{"text":2524},"Context Engineering and RAG are operational subdomains inside many LLM applications because context and retrieval can change behavior independently of the model.",{},{"id":1308,"data":2527,"type":218,"tunes":2529},{"text":2528},"Agentic AI extends LLMOps further into trajectory, permissions and tool-runtime operations.",{},{"id":1313,"data":2531,"type":603,"tunes":2536},{"url":2532,"title":2533,"excerpt":2534,"ctaLabel":2535},"https:\u002F\u002Fstajic.de\u002Fblog\u002Fai-agent-memory-is-not-rag-how-to-separate-memory-retrieval-state-and-context","AI Agent Memory Is Not RAG: How to Separate Memory, Retrieval, State and Context","Operational reliability improves when memory, retrieval, application state and model context remain separate lifecycle objects.","Read the architecture article",{},{"id":1321,"data":2538,"type":603,"tunes":2543},{"url":2539,"title":2540,"excerpt":2541,"ctaLabel":2542},"https:\u002F\u002Fstajic.de\u002Fblog\u002Fthe-answer-validity-boundary-the-missing-layer-between-relevance-and-reliable-ai-answers","The Answer Validity Boundary: The Missing Layer Between Relevance and Reliable AI Answers","LLMOps evaluation should preserve the version, scope and evidence conditions under which an answer remains supported.","Read the Answer Validity Boundary",{},{"id":1329,"data":2545,"type":42,"tunes":2547},{"text":2546,"level":253},"Frequently asked questions",{},{"id":1334,"data":2549,"type":1334,"tunes":2578},{"items":2550,"title":1373},[2551,2554,2557,2560,2563,2566,2569,2572,2575],{"id":1338,"answer":2552,"question":2553},"MLOps operates machine-learning systems across data, training, deployment and monitoring. LLMOps extends those practices to LLM applications where prompts, context, retrieval, providers, tools and evaluations also materially affect behavior.","What is the difference between MLOps and LLMOps?",{"id":1342,"answer":2555,"question":2556},"No. LLMOps reuses MLOps disciplines such as CI\u002FCD, lineage, evaluation, deployment and monitoring and adds LLM-specific operational concerns.","Does LLMOps replace MLOps?",{"id":1346,"answer":2558,"question":2559},"Not necessarily. Many use external foundation models and instead rely on continuous evaluation of prompts, models, retrieval and application behavior. Fine-tuned or self-trained systems can still require training pipelines.","Do LLM applications need continuous training?",{"id":1350,"answer":2561,"question":2562},"Generative outputs are open-ended and model behavior can change across prompts, snapshots and context. Evals provide repeatable evidence that a release still meets defined quality and safety criteria.","Why are evals so important in LLMOps?",{"id":1354,"answer":2564,"question":2565},"At minimum: application code, model\u002Fprovider\u002Fversion, prompts, eval datasets\u002Fscorers, retrieval configuration\u002Findexes, tool schemas, context rules and relevant safety\u002Fpermission configuration.","What should be versioned in LLMOps?",{"id":1358,"answer":2567,"question":2568},"No. The same prompt can behave differently with another model, retrieval set, context order, tool surface or provider.","Is prompt versioning enough?",{"id":1362,"answer":2570,"question":2571},"GenAIOps is another industry term for operating generative-AI applications. Some vendors use it interchangeably or as a broader label than LLMOps.","What is GenAIOps?",{"id":1366,"answer":2573,"question":2574},"Monitor end-to-end traces including model calls, prompts\u002Fcontext, retrieval, tools, latency, token\u002Fcost, quality samples, safety and final task outcomes.","How do you monitor an LLM application?",{"id":1370,"answer":2576,"question":2577},"Yes. Local models add their own operational concerns such as model files, hardware\u002FVRAM, load\u002Funload, runtime health, quantization and upgrade management.","Can local LLMs use LLMOps practices?",{},{"id":1376,"data":2580,"type":42,"tunes":2582},{"text":2581,"level":253},"Glossary",{},{"id":1381,"data":2584,"type":1381,"tunes":2618},{"title":2585,"entries":2586},"Key MLOps and LLMOps terms",[2587,2589,2591,2593,2595,2598,2601,2604,2607,2609,2612,2615],{"term":415,"anchor":414,"definition":2588},"Engineering practices for building, deploying, monitoring and maintaining machine-learning systems and their data\u002Fmodel lifecycle.",{"term":418,"anchor":417,"definition":2590},"Operational practices for production applications whose behavior materially depends on large language models and surrounding prompts, context, retrieval, tools and runtime.",{"term":1390,"anchor":1391,"definition":2592},"Operational discipline for generative-AI applications; often used as a broader or alternate label for LLMOps.",{"term":1749,"anchor":1395,"definition":2594},"Automated or repeated retraining and serving of ML models as data or implementations change.",{"term":2596,"anchor":1399,"definition":2597},"Continuous evaluation","Repeated evaluation of candidate and production AI behavior against versioned datasets and criteria.",{"term":2599,"anchor":1403,"definition":2600},"Model snapshot","A concrete version of a hosted or packaged model whose behavior can be tested and referenced.",{"term":2602,"anchor":1407,"definition":2603},"Application lineage","Traceable relationship among code, model\u002Fprovider, prompts, data\u002Fretrieval, tools, runtime and release configuration.",{"term":2605,"anchor":1411,"definition":2606},"Trace","Structured record of one application execution containing spans such as model calls, retrievals and tool operations.",{"term":1804,"anchor":1414,"definition":2608},"Versioned set of representative inputs, expectations and optionally traces\u002Foutputs used to measure behavior.",{"term":2610,"anchor":1418,"definition":2611},"LLM judge","A language model used as an evaluator for qualitative or semantic criteria; it is itself a versioned evaluation dependency.",{"term":2613,"anchor":1422,"definition":2614},"Behavioral regression","A degradation in application output or trajectory despite interfaces and code continuing to execute successfully.",{"term":2616,"anchor":1426,"definition":2617},"Provider routing","Policy for selecting among available model providers\u002Fendpoints according to capability, cost, latency, privacy or availability.",{},{"id":1430,"data":2620,"type":42,"tunes":2622},{"text":2621,"level":253},"Conclusion",{},{"id":1435,"data":2624,"type":218,"tunes":2626},{"text":2625},"MLOps and LLMOps share the same engineering objective: make AI systems reproducible enough, testable enough and observable enough to operate reliably in production.",{},{"id":1440,"data":2628,"type":218,"tunes":2630},{"text":2629},"The difference is the shape of the system. Classical MLOps often centers on training and serving model artifacts; LLMOps must operate a behavioral stack in which model snapshots, prompts, context, retrieval, tools, permissions and providers can change independently.",{},{"id":1445,"data":2632,"type":218,"tunes":2634},{"text":2633},"The shortest useful rule is: version, evaluate and observe everything that can materially change the LLM application's behavior — not only the model.",{},{"id":1450,"data":2636,"type":42,"tunes":2638},{"text":2637,"level":253},"Primary sources and current documentation",{},{"id":1455,"data":2640,"type":218,"tunes":2642},{"text":2641},"The sources below ground the MLOps baseline and the current operational patterns for LLM and agent applications. Project sections are original implementation evidence and are intentionally narrower than claims about a complete LLMOps platform.",{},{"id":1460,"data":2644,"type":1467,"tunes":2649},{"link":1462,"meta":2645},{"image":2646,"title":2647,"description":2648},{"url":381},"Google Cloud — MLOps: Continuous delivery and automation pipelines","Reference architecture describing CI, CD, continuous training, model registry, metadata, serving and monitoring for ML systems.",{},{"id":1470,"data":2651,"type":1467,"tunes":2656},{"link":1472,"meta":2652},{"image":2653,"title":2654,"description":2655},{"url":381},"AWS Machine Learning Lens — Model lineage","Current guidance for tracking code, data, models, environments and infrastructure across ML releases.",{},{"id":1479,"data":2658,"type":1467,"tunes":2663},{"link":1481,"meta":2659},{"image":2660,"title":2661,"description":2662},{"url":381},"AWS Machine Learning Lens — Model observability and tracking","Current guidance for production model monitoring, drift, endpoint health and lineage.",{},{"id":1488,"data":2665,"type":1467,"tunes":2670},{"link":1490,"meta":2666},{"image":2667,"title":2668,"description":2669},{"url":381},"Microsoft Azure — GenAIOps \u002F LLMOps lifecycle","Official guidance describing GenAIOps, sometimes called LLMOps, across initialization, experimentation, evaluation\u002Frefinement and deployment.",{},{"id":1497,"data":2672,"type":1467,"tunes":2677},{"link":1499,"meta":2673},{"image":2674,"title":2675,"description":2676},{"url":381},"MLflow — Agents and LLM applications","Current GenAI operations documentation covering tracing, evaluation, prompts and production observability for LLM applications and agents.",{},{"id":1506,"data":2679,"type":1467,"tunes":2684},{"link":1508,"meta":2680},{"image":2681,"title":2682,"description":2683},{"url":381},"MLflow — Evaluating production traces","Current guidance for evaluating complete LLM\u002Fagent traces, including retrieval and tool-call trajectories.",{},{"id":1515,"data":2686,"type":1467,"tunes":2691},{"link":1517,"meta":2687},{"image":2688,"title":2689,"description":2690},{"url":381},"MLflow — Evaluating prompts","Current prompt\u002Fmodel evaluation workflow using versioned prompts, datasets, scorers and traces.",{},{"id":1524,"data":2693,"type":1467,"tunes":2698},{"link":1526,"meta":2694},{"image":2695,"title":2696,"description":2697},{"url":381},"OpenAI API — Versioning and model snapshots","Current API guidance recommending pinned model versions and evals because prompting behavior can change between snapshots.",{},{"id":1533,"data":2700,"type":1467,"tunes":2705},{"link":1535,"meta":2701},{"image":2702,"title":2703,"description":2704},{"url":381},"OpenAI — Prompting","Current guidance to treat production prompts as application code, version them through source control and cover changes with tests and evaluation checks.",{},{"id":1542,"data":2707,"type":1467,"tunes":2712},{"link":1544,"meta":2708},{"image":2709,"title":2710,"description":2711},{"url":381},"OpenAI — Deprecations","Current provider lifecycle evidence showing model and platform-surface retirement as an operational dependency.",{},{"id":1551,"data":2714,"type":1467,"tunes":2719},{"link":1553,"meta":2715},{"image":2716,"title":2717,"description":2718},{"url":381},"OpenAI — Moving evaluation workflows to Promptfoo","Current 2026 migration guidance illustrating why evaluation assets should remain portable as provider tooling changes.",{},"2.31.6","MLOps operates machine-learning systems; LLMOps extends those practices to prompts, context, retrieval, providers, tools, evaluations and runtime behavior around large language models.",{"lang":7,"title":208,"content":210,"contentJson":2723,"excerpt":1560},{"time":212,"blocks":2724,"version":1559},[2725,2728,2731,2734,2737,2740,2743,2746,2749,2752,2755,2758,2761,2764,2767,2770,2773,2776,2779,2791,2794,2797,2800,2803,2806,2829,2832,2835,2838,2841,2844,2863,2866,2869,2872,2875,2878,2881,2884,2887,2890,2893,2896,2899,2902,2905,2908,2911,2914,2917,2920,2923,2926,2929,2932,2935,2938,2941,2944,2947,2950,2953,2956,2959,2962,2965,2968,2971,2974,2977,2980,2983,2986,2989,2992,2995,2998,3001,3004,3007,3010,3013,3016,3019,3033,3036,3039,3042,3045,3048,3051,3054,3057,3060,3075,3078,3081,3084,3087,3090,3093,3096,3099,3102,3105,3108,3111,3114,3117,3120,3123,3126,3129,3132,3135,3138,3141,3144,3147,3150,3153,3166,3169,3172,3189,3192,3207,3210,3226,3229,3248,3251,3254,3257,3260,3263,3266,3269,3272,3275,3278,3281,3284,3287,3290,3293,3296,3299,3312,3315,3331,3334,3337,3340,3343,3346,3349,3354,3359,3364,3369,3374,3379,3384,3389,3394,3399],{"id":215,"data":2726,"type":218,"tunes":2727},{"text":217},{},{"id":221,"data":2729,"type":226,"tunes":2730},{"body":223,"title":224,"variant":225},{},{"id":229,"data":2732,"type":226,"tunes":2733},{"body":231,"title":232,"variant":233},{},{"id":236,"data":2735,"type":226,"tunes":2736},{"body":238,"title":239,"variant":240},{},{"id":243,"data":2738,"type":226,"tunes":2739},{"body":245,"title":246,"variant":240},{},{"id":249,"data":2741,"type":254,"tunes":2742},{"title":251,"maxLevel":252,"minLevel":253},{},{"id":257,"data":2744,"type":42,"tunes":2745},{"text":259,"level":253},{},{"id":262,"data":2747,"type":218,"tunes":2748},{"text":264},{},{"id":267,"data":2750,"type":218,"tunes":2751},{"text":269},{},{"id":272,"data":2753,"type":218,"tunes":2754},{"text":274},{},{"id":277,"data":2756,"type":42,"tunes":2757},{"text":279,"level":253},{},{"id":282,"data":2759,"type":218,"tunes":2760},{"text":284},{},{"id":287,"data":2762,"type":218,"tunes":2763},{"text":289},{},{"id":292,"data":2765,"type":218,"tunes":2766},{"text":294},{},{"id":297,"data":2768,"type":42,"tunes":2769},{"text":299,"level":253},{},{"id":302,"data":2771,"type":218,"tunes":2772},{"text":304},{},{"id":307,"data":2774,"type":218,"tunes":2775},{"text":309},{},{"id":312,"data":2777,"type":218,"tunes":2778},{"text":314},{},{"id":317,"data":2780,"type":346,"tunes":2790},{"steps":2781,"title":344,"orientation":345},[2782,2783,2784,2785,2786,2787,2788,2789],{"label":321,"description":322},{"label":324,"description":325},{"label":327,"description":328},{"label":330,"description":331},{"label":333,"description":334},{"label":336,"description":337},{"label":339,"description":340},{"label":342,"description":343},{},{"id":349,"data":2792,"type":42,"tunes":2793},{"text":351,"level":253},{},{"id":354,"data":2795,"type":218,"tunes":2796},{"text":356},{},{"id":359,"data":2798,"type":218,"tunes":2799},{"text":361},{},{"id":364,"data":2801,"type":218,"tunes":2802},{"text":366},{},{"id":369,"data":2804,"type":42,"tunes":2805},{"text":371,"level":253},{},{"id":374,"data":2807,"type":419,"tunes":2828},{"rows":2808,"title":410,"layout":411,"columns":2825},[2809,2811,2813,2815,2817,2819,2821,2823],{"id":378,"label":379,"values":2810},[381,381],{"id":383,"label":384,"values":2812},[381,381],{"id":387,"label":388,"values":2814},[381,381],{"id":391,"label":392,"values":2816},[381,381],{"id":395,"label":396,"values":2818},[381,381],{"id":399,"label":400,"values":2820},[381,381],{"id":403,"label":404,"values":2822},[381,381],{"id":407,"label":408,"values":2824},[381,381],[2826,2827],{"id":414,"label":415},{"id":417,"label":418},{},{"id":422,"data":2830,"type":42,"tunes":2831},{"text":424,"level":253},{},{"id":427,"data":2833,"type":218,"tunes":2834},{"text":429},{},{"id":432,"data":2836,"type":218,"tunes":2837},{"text":434},{},{"id":437,"data":2839,"type":218,"tunes":2840},{"text":439},{},{"id":442,"data":2842,"type":42,"tunes":2843},{"text":444,"level":253},{},{"id":447,"data":2845,"type":411,"tunes":2862},{"content":2846,"stretched":43,"withHeadings":14},[2847,2848,2849,2850,2851,2852,2853,2854,2855,2856,2857,2858,2859,2860,2861],[451,452],[454,455],[457,458],[460,461],[463,464],[466,467],[469,470],[472,473],[475,476],[478,479],[481,482],[484,485],[487,488],[490,491],[493,494],{},{"id":497,"data":2864,"type":42,"tunes":2865},{"text":499,"level":253},{},{"id":502,"data":2867,"type":218,"tunes":2868},{"text":504},{},{"id":507,"data":2870,"type":218,"tunes":2871},{"text":509},{},{"id":512,"data":2873,"type":218,"tunes":2874},{"text":514},{},{"id":517,"data":2876,"type":42,"tunes":2877},{"text":519,"level":253},{},{"id":522,"data":2879,"type":218,"tunes":2880},{"text":524},{},{"id":527,"data":2882,"type":218,"tunes":2883},{"text":529},{},{"id":532,"data":2885,"type":218,"tunes":2886},{"text":534},{},{"id":537,"data":2888,"type":42,"tunes":2889},{"text":539,"level":253},{},{"id":542,"data":2891,"type":218,"tunes":2892},{"text":544},{},{"id":547,"data":2894,"type":218,"tunes":2895},{"text":549},{},{"id":552,"data":2897,"type":218,"tunes":2898},{"text":554},{},{"id":557,"data":2900,"type":42,"tunes":2901},{"text":559,"level":253},{},{"id":562,"data":2903,"type":218,"tunes":2904},{"text":564},{},{"id":567,"data":2906,"type":218,"tunes":2907},{"text":569},{},{"id":572,"data":2909,"type":218,"tunes":2910},{"text":574},{},{"id":577,"data":2912,"type":42,"tunes":2913},{"text":579,"level":253},{},{"id":582,"data":2915,"type":218,"tunes":2916},{"text":584},{},{"id":587,"data":2918,"type":218,"tunes":2919},{"text":589},{},{"id":592,"data":2921,"type":218,"tunes":2922},{"text":594},{},{"id":597,"data":2924,"type":603,"tunes":2925},{"url":599,"title":600,"excerpt":601,"ctaLabel":602},{},{"id":606,"data":2927,"type":42,"tunes":2928},{"text":608,"level":253},{},{"id":611,"data":2930,"type":218,"tunes":2931},{"text":613},{},{"id":616,"data":2933,"type":218,"tunes":2934},{"text":618},{},{"id":621,"data":2936,"type":218,"tunes":2937},{"text":623},{},{"id":626,"data":2939,"type":226,"tunes":2940},{"body":628,"title":629,"variant":630},{},{"id":633,"data":2942,"type":42,"tunes":2943},{"text":635,"level":253},{},{"id":638,"data":2945,"type":218,"tunes":2946},{"text":640},{},{"id":643,"data":2948,"type":218,"tunes":2949},{"text":645},{},{"id":648,"data":2951,"type":218,"tunes":2952},{"text":650},{},{"id":653,"data":2954,"type":42,"tunes":2955},{"text":655,"level":253},{},{"id":658,"data":2957,"type":218,"tunes":2958},{"text":660},{},{"id":663,"data":2960,"type":218,"tunes":2961},{"text":665},{},{"id":668,"data":2963,"type":218,"tunes":2964},{"text":670},{},{"id":673,"data":2966,"type":42,"tunes":2967},{"text":675,"level":253},{},{"id":678,"data":2969,"type":218,"tunes":2970},{"text":680},{},{"id":683,"data":2972,"type":218,"tunes":2973},{"text":685},{},{"id":688,"data":2975,"type":218,"tunes":2976},{"text":690},{},{"id":693,"data":2978,"type":603,"tunes":2979},{"url":695,"title":696,"excerpt":697,"ctaLabel":698},{},{"id":701,"data":2981,"type":42,"tunes":2982},{"text":703,"level":253},{},{"id":706,"data":2984,"type":218,"tunes":2985},{"text":708},{},{"id":711,"data":2987,"type":218,"tunes":2988},{"text":713},{},{"id":716,"data":2990,"type":218,"tunes":2991},{"text":718},{},{"id":721,"data":2993,"type":42,"tunes":2994},{"text":723,"level":253},{},{"id":726,"data":2996,"type":218,"tunes":2997},{"text":728},{},{"id":731,"data":2999,"type":218,"tunes":3000},{"text":733},{},{"id":736,"data":3002,"type":218,"tunes":3003},{"text":738},{},{"id":741,"data":3005,"type":42,"tunes":3006},{"text":743,"level":253},{},{"id":746,"data":3008,"type":218,"tunes":3009},{"text":748},{},{"id":751,"data":3011,"type":218,"tunes":3012},{"text":753},{},{"id":756,"data":3014,"type":218,"tunes":3015},{"text":758},{},{"id":761,"data":3017,"type":42,"tunes":3018},{"text":763,"level":253},{},{"id":766,"data":3020,"type":411,"tunes":3032},{"content":3021,"stretched":43,"withHeadings":14},[3022,3023,3024,3025,3026,3027,3028,3029,3030,3031],[770,771],[773,774],[776,777],[779,780],[782,783],[785,786],[788,789],[791,792],[794,795],[797,798],{},{"id":801,"data":3034,"type":42,"tunes":3035},{"text":803,"level":253},{},{"id":806,"data":3037,"type":218,"tunes":3038},{"text":808},{},{"id":811,"data":3040,"type":218,"tunes":3041},{"text":813},{},{"id":816,"data":3043,"type":218,"tunes":3044},{"text":818},{},{"id":821,"data":3046,"type":42,"tunes":3047},{"text":823,"level":253},{},{"id":826,"data":3049,"type":218,"tunes":3050},{"text":828},{},{"id":831,"data":3052,"type":218,"tunes":3053},{"text":833},{},{"id":836,"data":3055,"type":218,"tunes":3056},{"text":838},{},{"id":841,"data":3058,"type":42,"tunes":3059},{"text":843,"level":253},{},{"id":846,"data":3061,"type":411,"tunes":3074},{"content":3062,"stretched":43,"withHeadings":14},[3063,3064,3065,3066,3067,3068,3069,3070,3071,3072,3073],[850,851],[853,854],[856,857],[859,860],[862,863],[865,866],[782,868],[788,870],[791,872],[874,875],[877,878],{},{"id":881,"data":3076,"type":42,"tunes":3077},{"text":883,"level":253},{},{"id":886,"data":3079,"type":218,"tunes":3080},{"text":888},{},{"id":891,"data":3082,"type":218,"tunes":3083},{"text":893},{},{"id":896,"data":3085,"type":218,"tunes":3086},{"text":898},{},{"id":901,"data":3088,"type":42,"tunes":3089},{"text":903,"level":253},{},{"id":906,"data":3091,"type":218,"tunes":3092},{"text":908},{},{"id":911,"data":3094,"type":218,"tunes":3095},{"text":913},{},{"id":916,"data":3097,"type":218,"tunes":3098},{"text":918},{},{"id":921,"data":3100,"type":42,"tunes":3101},{"text":923,"level":253},{},{"id":926,"data":3103,"type":218,"tunes":3104},{"text":928},{},{"id":931,"data":3106,"type":218,"tunes":3107},{"text":933},{},{"id":936,"data":3109,"type":218,"tunes":3110},{"text":938},{},{"id":941,"data":3112,"type":42,"tunes":3113},{"text":943,"level":253},{},{"id":946,"data":3115,"type":218,"tunes":3116},{"text":948},{},{"id":951,"data":3118,"type":218,"tunes":3119},{"text":953},{},{"id":956,"data":3121,"type":218,"tunes":3122},{"text":958},{},{"id":961,"data":3124,"type":42,"tunes":3125},{"text":963,"level":253},{},{"id":966,"data":3127,"type":42,"tunes":3128},{"text":968,"level":252},{},{"id":971,"data":3130,"type":218,"tunes":3131},{"text":973},{},{"id":976,"data":3133,"type":218,"tunes":3134},{"text":978},{},{"id":981,"data":3136,"type":218,"tunes":3137},{"text":983},{},{"id":986,"data":3139,"type":218,"tunes":3140},{"text":988},{},{"id":991,"data":3142,"type":42,"tunes":3143},{"text":993,"level":252},{},{"id":996,"data":3145,"type":218,"tunes":3146},{"text":998},{},{"id":1001,"data":3148,"type":218,"tunes":3149},{"text":1003},{},{"id":1006,"data":3151,"type":218,"tunes":3152},{"text":1008},{},{"id":1011,"data":3154,"type":411,"tunes":3165},{"content":3155,"stretched":43,"withHeadings":14},[3156,3157,3158,3159,3160,3161,3162,3163,3164],[1015,1016],[1018,1019],[1021,1022],[1024,1025],[1027,1028],[1030,1031],[1033,1034],[1036,1037],[1039,1040],{},{"id":1043,"data":3167,"type":226,"tunes":3168},{"body":1045,"title":1046,"variant":240},{},{"id":1049,"data":3170,"type":42,"tunes":3171},{"text":1051,"level":253},{},{"id":1054,"data":3173,"type":411,"tunes":3188},{"content":3174,"stretched":43,"withHeadings":14},[3175,3176,3177,3178,3179,3180,3181,3182,3183,3184,3185,3186,3187],[1058,1059],[1061,1062],[1064,1065],[1067,1068],[1070,1071],[1073,1074],[1076,1077],[1079,1080],[1082,1083],[1085,1086],[1088,1089],[1091,1092],[1094,1095],{},{"id":1098,"data":3190,"type":42,"tunes":3191},{"text":1100,"level":253},{},{"id":1103,"data":3193,"type":411,"tunes":3206},{"content":3194,"stretched":43,"withHeadings":14},[3195,3196,3197,3198,3199,3200,3201,3202,3203,3204,3205],[1107,1108],[1110,1111],[1113,1114],[1116,1117],[1119,1120],[1122,1123],[1125,1126],[1128,1129],[1131,1132],[1134,1135],[1137,1138],{},{"id":1141,"data":3208,"type":42,"tunes":3209},{"text":1143,"level":253},{},{"id":1146,"data":3211,"type":346,"tunes":3225},{"steps":3212,"title":1185,"orientation":345},[3213,3214,3215,3216,3217,3218,3219,3220,3221,3222,3223,3224],{"label":1150,"description":1151},{"label":1153,"description":1154},{"label":1156,"description":1157},{"label":1159,"description":1160},{"label":1162,"description":1163},{"label":1165,"description":1166},{"label":1168,"description":1169},{"label":1171,"description":1172},{"label":1174,"description":1175},{"label":1177,"description":1178},{"label":1180,"description":1181},{"label":1183,"description":1184},{},{"id":1188,"data":3227,"type":42,"tunes":3228},{"text":1190,"level":253},{},{"id":1193,"data":3230,"type":411,"tunes":3247},{"content":3231,"stretched":43,"withHeadings":14},[3232,3233,3234,3235,3236,3237,3238,3239,3240,3241,3242,3243,3244,3245,3246],[1197,1198],[1200,1201],[1203,1204],[1206,1207],[1209,1210],[1212,1213],[1215,1216],[1218,1219],[1221,1222],[1224,1225],[1227,1228],[1230,1231],[1233,1234],[1236,1237],[1239,1240],{},{"id":1243,"data":3249,"type":42,"tunes":3250},{"text":1245,"level":253},{},{"id":1248,"data":3252,"type":218,"tunes":3253},{"text":1250},{},{"id":1253,"data":3255,"type":218,"tunes":3256},{"text":1255},{},{"id":1258,"data":3258,"type":218,"tunes":3259},{"text":1260},{},{"id":1263,"data":3261,"type":218,"tunes":3262},{"text":1265},{},{"id":1268,"data":3264,"type":218,"tunes":3265},{"text":1270},{},{"id":1273,"data":3267,"type":42,"tunes":3268},{"text":1275,"level":253},{},{"id":1278,"data":3270,"type":218,"tunes":3271},{"text":1280},{},{"id":1283,"data":3273,"type":218,"tunes":3274},{"text":1285},{},{"id":1288,"data":3276,"type":218,"tunes":3277},{"text":1290},{},{"id":1293,"data":3279,"type":42,"tunes":3280},{"text":1295,"level":253},{},{"id":1298,"data":3282,"type":218,"tunes":3283},{"text":1300},{},{"id":1303,"data":3285,"type":218,"tunes":3286},{"text":1305},{},{"id":1308,"data":3288,"type":218,"tunes":3289},{"text":1310},{},{"id":1313,"data":3291,"type":603,"tunes":3292},{"url":1315,"title":1316,"excerpt":1317,"ctaLabel":1318},{},{"id":1321,"data":3294,"type":603,"tunes":3295},{"url":1323,"title":1324,"excerpt":1325,"ctaLabel":1326},{},{"id":1329,"data":3297,"type":42,"tunes":3298},{"text":1331,"level":253},{},{"id":1334,"data":3300,"type":1334,"tunes":3311},{"items":3301,"title":1373},[3302,3303,3304,3305,3306,3307,3308,3309,3310],{"id":1338,"answer":1339,"question":1340},{"id":1342,"answer":1343,"question":1344},{"id":1346,"answer":1347,"question":1348},{"id":1350,"answer":1351,"question":1352},{"id":1354,"answer":1355,"question":1356},{"id":1358,"answer":1359,"question":1360},{"id":1362,"answer":1363,"question":1364},{"id":1366,"answer":1367,"question":1368},{"id":1370,"answer":1371,"question":1372},{},{"id":1376,"data":3313,"type":42,"tunes":3314},{"text":1378,"level":253},{},{"id":1381,"data":3316,"type":1381,"tunes":3330},{"title":1383,"entries":3317},[3318,3319,3320,3321,3322,3323,3324,3325,3326,3327,3328,3329],{"term":415,"anchor":414,"definition":1386},{"term":418,"anchor":417,"definition":1388},{"term":1390,"anchor":1391,"definition":1392},{"term":1394,"anchor":1395,"definition":1396},{"term":1398,"anchor":1399,"definition":1400},{"term":1402,"anchor":1403,"definition":1404},{"term":1406,"anchor":1407,"definition":1408},{"term":1410,"anchor":1411,"definition":1412},{"term":469,"anchor":1414,"definition":1415},{"term":1417,"anchor":1418,"definition":1419},{"term":1421,"anchor":1422,"definition":1423},{"term":1425,"anchor":1426,"definition":1427},{},{"id":1430,"data":3332,"type":42,"tunes":3333},{"text":1432,"level":253},{},{"id":1435,"data":3335,"type":218,"tunes":3336},{"text":1437},{},{"id":1440,"data":3338,"type":218,"tunes":3339},{"text":1442},{},{"id":1445,"data":3341,"type":218,"tunes":3342},{"text":1447},{},{"id":1450,"data":3344,"type":42,"tunes":3345},{"text":1452,"level":253},{},{"id":1455,"data":3347,"type":218,"tunes":3348},{"text":1457},{},{"id":1460,"data":3350,"type":1467,"tunes":3353},{"link":1462,"meta":3351},{"image":3352,"title":1465,"description":1466},{"url":381},{},{"id":1470,"data":3355,"type":1467,"tunes":3358},{"link":1472,"meta":3356},{"image":3357,"title":1475,"description":1476},{"url":381},{},{"id":1479,"data":3360,"type":1467,"tunes":3363},{"link":1481,"meta":3361},{"image":3362,"title":1484,"description":1485},{"url":381},{},{"id":1488,"data":3365,"type":1467,"tunes":3368},{"link":1490,"meta":3366},{"image":3367,"title":1493,"description":1494},{"url":381},{},{"id":1497,"data":3370,"type":1467,"tunes":3373},{"link":1499,"meta":3371},{"image":3372,"title":1502,"description":1503},{"url":381},{},{"id":1506,"data":3375,"type":1467,"tunes":3378},{"link":1508,"meta":3376},{"image":3377,"title":1511,"description":1512},{"url":381},{},{"id":1515,"data":3380,"type":1467,"tunes":3383},{"link":1517,"meta":3381},{"image":3382,"title":1520,"description":1521},{"url":381},{},{"id":1524,"data":3385,"type":1467,"tunes":3388},{"link":1526,"meta":3386},{"image":3387,"title":1529,"description":1530},{"url":381},{},{"id":1533,"data":3390,"type":1467,"tunes":3393},{"link":1535,"meta":3391},{"image":3392,"title":1538,"description":1539},{"url":381},{},{"id":1542,"data":3395,"type":1467,"tunes":3398},{"link":1544,"meta":3396},{"image":3397,"title":1547,"description":1548},{"url":381},{},{"id":1551,"data":3400,"type":1467,"tunes":3403},{"link":1553,"meta":3401},{"image":3402,"title":1556,"description":1557},{"url":381},{},"Post erfolgreich abgerufen",{"items":3406,"source":3487,"manualIds":3488,"manualMatchedIds":3489},[3407,3414,3421,3428,3434,3441,3448,3455,3462,3468,3475,3482],{"id":3408,"slug":3409,"title":3410,"excerpt":3411,"featuredImage":3412,"publishedAt":3413},"445","qwen-3-6-in-production-release-runbook-ai-rollback-and-llmops-versioning","Qwen 3.6 u produkciji: Runbook za izdavanje, AI rollback i LLMOps verziranje","Qwen 3.6 nije samo još jedna nadogradnja modela. To je istovremeno događaj objavljivanja, scenario povratka na prethodnu verziju i problem verziranja. Ovaj članak objašnjava kako Qwen 3.6 treba tretirati u produkciji kroz LLMOps disciplinu, sledljivost promptova i modela, kontrolisano uvođenje i spremnost za povratak na prethodnu verziju zasnovanu na dokazima.","\u002Fuploads\u002F2026\u002F02\u002Fnew-qwen-3-5-plus-1771515512741-dcbi9p.webp","2026-05-04T02:49:00.000Z",{"id":3415,"slug":3416,"title":3417,"excerpt":3418,"featuredImage":3419,"publishedAt":3420},"477","computer-use-agents-why-a-successful-demo-can-still-be-an-unreliable-system","Agenti za korišćenje računara: Zašto uspešan demo i dalje može biti nepouzdan sistem","Agenti za korišćenje računara sada mogu da završe impresivne radne tokove u pregledaču i na radnoj površini, ali jedno uspešno izvršavanje dokazuje sposobnost—ne pouzdanost. Ovaj članak pokazuje kako testirati ponovljivost, robusnost u odnosu na okruženje, kontrolu dugog horizonta, svest o stanju, verifikaciju ishoda i bezbedno upravljanje ciljevima.","\u002Fuploads\u002F2026\u002F09\u002Fcomputer-use-agents-why-a-successful-demo-can-still-be-an-unreliable-system-1790352854690-75qnrg.webp","2026-09-25T12:13:00.000Z",{"id":3422,"slug":3423,"title":3424,"excerpt":3425,"featuredImage":3426,"publishedAt":3427},"478","what-is-rag-the-simplest-explanation-of-how-it-works","Šta je RAG? Najjednostavnije objašnjenje kako funkcioniše","RAG zvuči komplikovano, ali ideja je jednostavna: pre nego što AI odgovori, prvo potraži korisne informacije iz izvora znanja i daje te informacije jezičkom modelu. Ovaj vodič objašnjava RAG, LLM-ove, stanje, memoriju i alate koristeći jedan jednostavan mentalni model.","\u002Fuploads\u002F2026\u002F09\u002Fwhat-is-rag-the-simplest-explanation-of-how-it-works-1790377492124-khjagt.webp","2026-09-25T19:03:00.000Z",{"id":3429,"slug":3430,"title":3431,"excerpt":3432,"featuredImage":3412,"publishedAt":3433},"384","new-qwen-3-5-plus","Novi Qwen 3.5-Plus: AI otvorenog koda je upravo postao ozbiljan.","Otkrijte revolucionarne funkcije i prednosti Alibabinog Qwen 3.5-Plus modela, AI otvorenog koda koji menja pravila igre za programere.","2026-02-19T10:23:00.000Z",{"id":3435,"slug":3436,"title":3437,"excerpt":3438,"featuredImage":3439,"publishedAt":3440},"488","what-is-context-engineering-what-the-model-receives-before-it-answers","Šta je kontekstualno inženjerstvo? Šta model prima pre nego što odgovori","Inženjering konteksta osmišljava koje informacije AI model prima pre inferencije, uključujući promptove, pretragu, memoriju, stanje aplikacije, rezultate alata i istoriju konverzacije.","\u002Fuploads\u002F2026\u002F10\u002Fwhat-is-context-engineering-what-the-model-receives-before-it-answers-1791480653258-018kcv.webp","2026-10-08T13:29:00.000Z",{"id":3442,"slug":3443,"title":3444,"excerpt":3445,"featuredImage":3446,"publishedAt":3447},"459","ollama-is-not-the-product-building-production-ready-open-llm-applications","Ollama nije proizvod: Izgradnja aplikacija spremnih za produkciju sa otvorenim LLM-ovima","Pokretanje lokalnog modela pomoću Ollama-e je jednostavno. Izgradnja Open-LLM aplikacije spremne za produkciju je teža: zahteva RAG, kontrolu pristupa, apstrakciju provajdera, evaluaciju, logovanje, disciplinu puštanja u rad i kontrolisani aplikativni sloj oko modela.","\u002Fuploads\u002F2026\u002F06\u002Follama-is-not-the-product-building-production-ready-open-llm-applications-1782679361640-h0usqf.webp","2026-06-28T16:39:00.000Z",{"id":3449,"slug":3450,"title":3451,"excerpt":3452,"featuredImage":3453,"publishedAt":3454},"475","managed-agent-harness-vs-self-hosted-agent-loop-what-you-gain-what-you-lose","Upravljani harness za agente naspram samostalno hostovane petlje agenta: Šta dobijate, šta gubite","“Samostalno hostovani agent” može značiti veoma različite arhitekture. Ovaj vodič razgraničava upravljani harness, samostalno hostovano okruženje za izvršavanje i potpuno samostalno upravljanu petlju agenta—i pokazuje koja je granica kontrole timovima zapravo potrebna.","\u002Fuploads\u002F2026\u002F09\u002Fmanaged-agent-harness-vs-self-hosted-agent-loop-what-you-gain-what-you-lose-1790352403475-kj10jh.webp","2026-09-25T12:05:00.000Z",{"id":3456,"slug":3457,"title":3458,"excerpt":3459,"featuredImage":3460,"publishedAt":3461},"455","zbt-z8102ax-dual-sim-failover-test","ZBT Z8102AX Dual-SIM failover: Šta radi, šta nedostaje i šta zahteva bolji firmver","ZBT Z8102AX je dual-SIM 5G OpenWrt ruter, ali sam dual-SIM hardver nije isto što i inteligentni failover. Ruter prepoznaje SIM karticu i uspešno se povezuje, ali automatsko prebacivanje, oporavak modema, odluke zasnovane na signalu i čista failover logika i dalje zahtevaju dublje testiranje.","\u002Fuploads\u002F2026\u002F06\u002Fopenwrt-router-review-dual-sim-03-1781620592829-7t77j7.webp","2026-06-16T10:40:00.000Z",{"id":3463,"slug":1588,"title":3464,"excerpt":3465,"featuredImage":3466,"publishedAt":3467},"434","Sveobuhvatan vodič za Evaluation Harness: Ovladavanje evaluacijom performansi LLM-ova","Ovaj vodič pruža detaljan pregled Evaluation Harness-a, ključnog okvira za rigoroznu procenu sposobnosti velikih jezičkih modela (LLM) u korporativnim LLMOps procesima. Naučite podešavanje, najbolje prakse i napredne tehnike kako biste osigurali pouzdano benčmarkovanje i optimizaciju modela.","\u002Fuploads\u002F2026\u002F04\u002Fevaluation-harness-1775466944495-4s0xv2.webp","2026-03-01T17:50:00.000Z",{"id":3469,"slug":3470,"title":3471,"excerpt":3472,"featuredImage":3473,"publishedAt":3474},"480","when-should-an-ai-stop-trusting-its-own-knowledge-the-retrieval-trigger","Kada bi AI trebalo da prestane da veruje sopstvenom znanju? — Okidač za pretragu","AI model ne zahteva pretragu za svako pitanje. Važan problem je znati kada njegovo interno znanje više nije dovoljno. Okidač za pretragu je praktična granica odlučivanja koja određuje kada AI sistem treba da prestane da se oslanja isključivo na znanje modela i pribavi spoljne dokaze pre odgovaranja.","\u002Fuploads\u002F2026\u002F09\u002Fwhen-should-an-ai-stop-trusting-its-own-knowledge-the-retrieval-trigger-1790574991244-f4rpyg.webp","2026-09-28T01:49:00.000Z",{"id":3476,"slug":3477,"title":3478,"excerpt":3479,"featuredImage":3480,"publishedAt":3481},"479","where-does-an-llm-get-its-data-rag-data-sources-in-python","Odakle LLM dobija svoje podatke? RAG izvori podataka u Python-u","LLM ne zna magično vaše fajlove, baze podataka ili API-je. Ovaj praktični nastavak RAG serije pokazuje, uz jednostavan Python, kako eksterni podaci postaju dokazi koji se mogu pronaći: od tekstualnih fajlova i SQL-a do pretrage punog teksta, embeddinga, sastavljanja konteksta i konačnog LLM poziva.","\u002Fuploads\u002F2026\u002F09\u002Fwhere-does-an-llm-get-its-data-rag-data-sources-in-python-1790517200521-nfsi5i.webp","2026-09-27T05:51:00.000Z",{"id":3483,"slug":3484,"title":3484,"excerpt":10,"featuredImage":3485,"publishedAt":3486},"369","git-with-automatic-upload-and-synchronization-to-a-production-server","\u002Fuploads\u002F2024\u002F05\u002Fstep-by-step-guide-illustration-showing-the-process-of-setting-up-Git-with-auto-upload-and-synchronization-to-a-production-server-large.webp","2024-05-28T22:48:00.000Z","fallback",[],[]]