[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"portal-settings:stajic:zh":3,"public-menus:all":38,"post:evaluation-harness:zh":221,"related:post:evaluation-harness:zh:1":619},{"statusCode":4,"data":5,"message":37},200,{"tenantId":6,"lang":7,"defaultLang":8,"siteUrl":9,"contactEmail":10,"brandName":11,"logoUrl":12,"siteName":11,"siteDescription":13,"ogImage":10,"robotsIndex":14,"socialLinks":10,"reservedSlugs":10,"seoPolicy":15},"stajic","zh","de","https:\u002F\u002Fstajic.de",null,"Stajic Platform","\u002FLogo_Planet.svg","Stajic Portal",true,{"branding":16,"relatedContent":17,"crossDomainLinks":18},{"logoUrl":12},{"enabled":14},[19,22,25,28,31,34],{"url":20,"label":21,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Ffigure.rocks","figure.rocks",{"url":23,"label":24,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Floving.rocks","loving.rocks",{"url":26,"label":27,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Fbazify.com","bazify.com",{"url":29,"label":30,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Fbazify.de","bazify.de",{"url":32,"label":33,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Fbazify.at","bazify.at",{"url":35,"label":36,"isActive":14,"showInFooter":14,"includeInSameAs":14},"https:\u002F\u002Fbazify.ba","bazify.ba","Portal settings resolved",[39,45],{"id":40,"name":41,"location":42,"isActive":14,"isDefault":43,"items":44},1,"main-navigation","header",false,[],{"id":46,"name":47,"location":48,"isActive":14,"isDefault":14,"items":49},4,"main-menu","sidebar",[50,66,79,93,103,118,133],{"id":51,"title":52,"url":60,"target":61,"icon":62,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":64,"portfolioId":10,"children":65},"item-18",{"de":53,"en":54,"es":55,"fr":56,"it":54,"ru":57,"sr":58,"zh":59},"Startseite","Home","Inicio","Accueil","Главная","Почетна","首页","\u002Ffull-stack-web-developer-munich-performance-seo-and-maintainable-builds","_self","i-lucide-home","page",111,[],{"id":67,"title":68,"url":75,"target":61,"icon":76,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":77,"portfolioId":10,"children":78},"item-22",{"de":69,"en":69,"es":70,"fr":69,"it":71,"ru":72,"sr":73,"zh":74},"Vision","Visión","Visione","Видение","Визија","想象","\u002Fueber-uns-webdesign-muenchen-webaplikation","i-lucide-eye",113,[],{"id":80,"title":81,"url":89,"target":61,"icon":90,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":91,"portfolioId":10,"children":92},"item-19",{"de":82,"en":83,"es":84,"fr":83,"it":85,"ru":86,"sr":87,"zh":88},"Leistungen","Services","Servicios","Servizi","Услуги","Услуге","服务","\u002Fservices-dienstleistungen-muenchen","i-lucide-wrench",116,[],{"id":94,"title":95,"url":99,"target":61,"icon":100,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":101,"portfolioId":10,"children":102},"item-23",{"de":96,"en":96,"es":96,"fr":96,"it":96,"ru":97,"sr":97,"zh":98},"Blog","Блог","博客","\u002Fblog","i-lucide-book-open",112,[],{"id":104,"title":105,"url":114,"target":61,"icon":115,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":116,"portfolioId":10,"children":117},"item-32",{"de":106,"en":107,"es":108,"fr":109,"it":110,"ru":111,"sr":112,"zh":113},"Neue Technologien","New Technologies","Nuevas tecnologías","Nouvelles technologies","Nuove tecnologie","Новые технологии","Нове технологије","新技术！","\u002Fneue-webtechnologien","i-lucide-sparkles",122,[],{"id":119,"title":120,"url":129,"target":61,"icon":130,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":131,"portfolioId":10,"children":132},"item-20",{"de":121,"en":122,"es":123,"fr":124,"it":125,"ru":126,"sr":127,"zh":128},"Kontakt","Contact us!","Contacto","Contact","Contatto","Контакт","Контактирајте нас","联系我们！","\u002Fcontact","i-lucide-mail",115,[],{"id":134,"title":135,"url":144,"target":61,"icon":145,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":146,"portfolioId":10,"children":147},"item-21",{"de":136,"en":137,"es":138,"fr":139,"it":140,"ru":141,"sr":142,"zh":143},"Unsere Arbeit","Our Work","Nuestro trabajo","Nos réalisations","I nostri lavori","Наши работы","Наши радови","文件夹","\u002Fportfolio","i-lucide-briefcase",114,[148,161,175,191,206],{"id":149,"title":150,"url":144,"target":61,"icon":159,"isActive":14,"type":63,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":146,"portfolioId":10,"children":160},"item-24",{"de":151,"en":152,"es":153,"fr":154,"it":155,"ru":156,"sr":157,"zh":158},"Alle Projekte","All Projects","Todos los proyectos","Tous les projets","Tutti i progetti","Все проекты","Сви пројекти","所有项目","i-lucide-grid-3x3",[],{"id":162,"title":163,"url":171,"target":61,"icon":172,"isActive":14,"type":173,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":10,"portfolioId":10,"children":174},"item-29",{"de":164,"en":165,"es":166,"fr":167,"it":168,"ru":169,"sr":170,"zh":143},"Local Roots, Global Reach","Local Roots - Global Reach","Empresa local ","Entreprise locale","Azienda locale","Местная компания","Локално предузеће глобално тржиште","\u002Fportfolio\u002Flocal-roots-global-reach-communication-media-systems-for-modern-business","i-lucide-globe-2","custom",[],{"id":176,"title":177,"url":186,"target":61,"icon":187,"isActive":14,"type":188,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":10,"portfolioId":189,"children":190},"menu-2916d060-8681-482a-af9a-c107a8bc30d8",{"de":178,"en":179,"es":180,"fr":181,"it":182,"ru":183,"sr":184,"zh":185},"Von Rohdaten zu verifizierbarem KI-Kontext: Eine praktische Architektur für agentenbereite Informationen","From Raw Data to Verifiable AI Context:","De datos sin procesar a contexto de IA verificable: una arquitectura práctica para información lista para agentes","Des données brutes au contexte d'IA vérifiable : une architecture pratique pour des informations prêtes pour les agents","Dai dati grezzi a un contesto IA verificabile: Un'architettura pratica per informazioni pronte per gli agenti","От необработанных данных к верифицируемому контексту ИИ: практическая архитектура информации, готовой для агентов","Od sirovih podataka do proverljivog AI konteksta: Praktična arhitektura za informacije spremne za agente","从原始数据到可验证的 AI 上下文：面向智能体就绪信息的实用架构","\u002Fportfolio\u002Ffrom-raw-data-to-verifiable-ai-context-a-practical-architecture-for-agent-ready-information","i-lucide-database","portfolio",22,[],{"id":192,"title":193,"url":202,"target":61,"icon":203,"isActive":14,"type":188,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":10,"portfolioId":204,"children":205},"menu-e2061cd7-ff53-49d5-a0e8-3d49090577c0",{"de":194,"en":195,"es":196,"fr":197,"it":198,"ru":199,"sr":200,"zh":201},"Präsenzplattformen — Unternehmenskommunikation","Presence Platforms — Enterprise Communication","Plataformas de presencia — Comunicación empresarial","Plateformes de présence — Communication d'entreprise","Presence Platforms — Comunicazione aziendale","Платформы присутствия — Корпоративные коммуникации","Presence platforme — Poslovna komunikacija","Presence 平台 — 企业通信","\u002Fportfolio\u002Fpresence-platforms-enterprise-communication-that-wins-without-competing","i-lucide-badge-percent",21,[],{"id":207,"title":208,"url":217,"target":61,"icon":218,"isActive":14,"type":188,"productId":10,"categoryId":10,"shopCategoryId":10,"articleId":10,"pageId":10,"portfolioId":219,"children":220},"menu-a781582e-e90d-418e-b8c9-b070bc37c37e",{"de":209,"en":210,"es":211,"fr":212,"it":213,"ru":214,"sr":215,"zh":216},"Maximierung des POS-ROI durch Konsistenz","Maximizing POS ROI Through Consistent","Maximizar el ROI del POS a través de constante","Maximiser le ROI du PDV grâce à la cohérence","Massimizzare il ROI del POS attraverso costante","Максимизация ROI POS за счет последовательного","Maksimizacija POS ROI-ja kroz dosledno","通过持续最大化 POS 投资回报率","\u002Fportfolio\u002Ffrom-vision-to-value-maximizing-pos-roi-through-consistent-ci-creation-and-strategic-branding","i-lucide-briefcase-business",11,[],{"statusCode":4,"data":222,"message":618},{"id":223,"title":224,"slug":225,"content":226,"contentJson":227,"excerpt":364,"featuredImage":365,"featuredImageAlt":366,"featuredImageCaption":10,"featuredImageTitle":10,"featuredImageCopyright":10,"featuredImageAuthor":10,"featuredImageSourceUrl":10,"featuredImageLicense":10,"featuredImageIsAiGenerated":43,"status":367,"publishedAt":368,"createdAt":369,"updatedAt":370,"seoLocalePaths":371,"categories":380,"author":384,"translations":389},"434","全面评估指南：精通LLM性能评估","evaluation-harness","\u003Cp># 评估工具指南\u003C\u002Fp>\n\u003Cp>## 评估工具介绍\u003C\u002Fp>\n\u003Cp>评估工具是一个功能强大、开源的框架，专门设计用于评估大型语言模型（LLMs）。由EleutherAI社区开发，它标准化了在不同任务、指标和数据集上对LLMs进行基准测试的过程。在企业LLMOps中，它作为模型选择、微调验证和持续监控的基石。\u003C\u002Fp>\n\u003Cp>主要优势包括：\n- **一致性**：跨模型和任务的统一评估协议。\n- **可扩展性**：高效处理海量数据集和多个模型。\n- **可扩展性**：支持自定义任务、数据集和指标。\n- **可复现性**：通过种子随机性和缓存实现确定性结果。\u003C\u002Fp>\n\u003Cp>非常适合从临时测试过渡到生产级LLM评估的团队。\u003C\u002Fp>\n\u003Cp>## 先决条件与安装\u003C\u002Fp>\n\u003Cp>在开始之前，请确保您的环境满足以下要求：\n- Python 3.10+。\n- GPU\u002FTPU 加速（推荐用于大型模型）。\n- 足够的内存（中型模型需要16GB以上）。\u003C\u002Fp>\n\u003Cp>### Step-by-Step Installation\n1. Clone the repository:\n   ```bash\ngit clone https:\u002F\u002Fgithub.com\u002FEleutherAI\u002Flm-evaluation-harness\ngit checkout main\n```\u003C\u002Fp>\n\u003Cp>2. Install dependencies:\n   ```bash\npip install -e .\npip install torch transformers datasets\n```\u003C\u002Fp>\n\u003Cp>3. For specific tasks (e.g., vision-language models):\n   ```bash\npip install timm pillow\n```\u003C\u002Fp>\n\u003Cp>4. Verify installation:\n   ```bash\nlm_eval --help\n```\u003C\u002Fp>\n\u003Cp>专业建议：使用虚拟环境（如 `venv` 或 `conda`）来隔离依赖项。\u003C\u002Fp>\n\u003Cp>## 核心概念\u003C\u002Fp>\n\u003Cp>### 任务与数据集\n评估工具包开箱即用支持200多个任务，分类如下：\n- **分类任务**：ARC、BoolQ、HellaSwag。\n- **生成任务**：AlpacaEval、MT-Bench。\n- **推理任务**：GSM8K、MATH。\n- **多模态任务**：MMMU、MathVista。\u003C\u002Fp>\n\u003Cp>数据集自动从Hugging Face Hub下载。\u003C\u002Fp>\n\u003Cp>### 指标\n常用指标包括：\n- **准确率**：分类任务的精确匹配。\n- **F1分数**：平衡精确率与召回率。\n- **困惑度**：用于生成流畅性评估。\n- **BLEU\u002FROUGE**：翻译和摘要任务指标。\u003C\u002Fp>\n\u003Cp>可通过`--metric`参数使用自定义指标。\u003C\u002Fp>\n\u003Cp>### 模型加载\n支持HF Transformers、Llama.cpp、vLLM等多种框架：\n- Hugging Face：`meta-llama\u002FLlama-2-7b-chat-hf`\n- 本地模型：支持自定义路径及量化（如4位量化）。\u003C\u002Fp>\n\u003Cp>## 运行基础评估\u003C\u002Fp>\n\u003Cp>### Command-Line Interface (CLI)\nStart with a simple benchmark:\n```bash\nlm_eval --model hf --model_args pretrained=model_name,trust_remote_code=True --tasks hellaswag,arc_easy --device cuda:0 --batch_size auto\n```\u003C\u002Fp>\n\u003Cp>详细说明：\n- `--model hf`：Hugging Face 加载器。\n- `--tasks`：逗号分隔的任务列表。\n- `--batch_size auto`：根据硬件自动优化。\u003C\u002Fp>\n\u003Cp>### 解读结果\n输出包括：\n- **acc**：准确率分数。\n- **acc_stderr**：标准误差。\n- 与排行榜兼容的JSON。\u003C\u002Fp>\n\u003Cp>Example output:\n```\nhellaswag: acc=0.9123 (±0.0012)\narc_easy: acc=0.7845 (±0.0021)\n```\u003C\u002Fp>\n\u003Cp>## 高级用法\u003C\u002Fp>\n\u003Cp>### Multi-Model Leaderboards\nCompare models:\n```bash\nlm_eval --model hf --model_args pretrained=model1 --tasks all --limit 1000\nlm_eval --model hf --model_args pretrained=model2 --tasks all --limit 1000\n```\nAggregate with `--save_jsonl` and external tools.\u003C\u002Fp>\n\u003Cp>### 自定义任务\n1. 在 `lm_eval\u002Ftasks\u002F` 中定义任务：\n   - 用于数据集的 YAML 配置。\n   - 用于少样本提示的 Python 处理器。\u003C\u002Fp>\n\u003Cp>2. Example custom task YAML:\n   ```yaml\ntask: my_custom_task\ndataset_path: huggingface\ndataset_name: my_dataset\ntraining_split: train\nfewshot_split: validation\nmetric_list:\n  - metric: acc\n    aggregation: mean\n    higher_is_better: true\n```\u003C\u002Fp>\n\u003Cp>3. 运行：`lm_eval --tasks my_custom_task`\u003C\u002Fp>\n\u003Cp>### 少样本和思维链提示\n- `--num_fewshot 5`：上下文示例。\n- 通过 `--gen_kwargs temperature=0.7` 自定义模板。\u003C\u002Fp>\n\u003Cp>对于思维链：使用类似 `gsm8k_cot` 的任务。\u003C\u002Fp>\n\u003Cp>## 优化与最佳实践\u003C\u002Fp>\n\u003Cp>### 性能调优\n- **批处理**：`--batch_size 32` 或 `auto`。\n- **量化**：`--model_args dtype=bfloat16,load_in_4bit=True`。\n- **分布式**：`--multi_gpu` 用于 Ray 集成。\u003C\u002Fp>\n\u003Cp>### 成本效率\n- 限制样本数量：`--limit 500`。\n- 使用较小的子集：`--subsample 0.1`。\n- 缓存结果：`--cache_dir \u002Fpath\u002Fto\u002Fcache`。\u003C\u002Fp>\n\u003Cp>### 可靠性提示\n- 运行多个种子：`--num_generations 8`。\n- 使用自助法计算置信区间。\n- 使用 `--log_samples` 记录所有内容。\u003C\u002Fp>\n\u003Cp>## 在LLMOps流水线中的集成\u003C\u002Fp>\n\u003Cp>Embed in CI\u002FCD:\n1. GitHub Actions YAML:\n   ```yaml\n   - name: Evaluate Model\n     run: lm_eval --model hf --model_args pretrained=${{ inputs.model }} --tasks core --batch_size auto > results.json\n   ```\u003C\u002Fp>\n\u003Cp>2. MLflow tracking:\n   ```python\nimport mlflow\nmlflow.log_metrics(results)\n```\u003C\u002Fp>\n\u003Cp>3. 用于仪表板的 Prometheus\u002FGrafana。\u003C\u002Fp>\n\u003Cp>## 常见问题排查\u003C\u002Fp>\n\u003Cp>- **OOM错误**：减小批次大小或使用梯度检查点。\n- **CUDA内存不足**：启用 `torch.backends.cuda.enable_flash_sdp(True)`。\n- **推理速度慢**：切换到vLLM加载器：`--model vllm`。\n- **数据集未找到**：检查HF访问令牌。\u003C\u002Fp>\n\u003Cp>## 结论与后续步骤\u003C\u002Fp>\n\u003Cp>评估工具将主观的LLM评估转变为数据驱动的过程。从核心任务开始，扩展到自定义评估，并集成到您的LLMOps工作流程中。\u003C\u002Fp>\n\u003Cp>资源：\n- GitHub：[EleutherAI\u002Flm-evaluation-harness](https:\u002F\u002Fgithub.com\u002FEleutherAI\u002Flm-evaluation-harness)\n- 排行榜：[Open LLM Leaderboard](https:\u002F\u002Fhuggingface.co\u002Fspaces\u002Fopen-llm-leaderboard\u002Fopen_llm_leaderboard)\n- Discord：EleutherAI社区。\u003C\u002Fp>\n\u003Cp>立即开始实验，解锁精确的模型洞察。\u003C\u002Fp>",{"time":228,"blocks":229,"version":363},1772701876779,[230,234,237,240,243,246,249,252,255,258,261,264,267,270,273,276,279,282,285,288,291,294,297,300,303,306,309,312,315,318,321,324,327,330,333,336,339,342,345,348,351,354,357,360],{"data":231,"type":233},{"text":232},"# 评估工具指南","paragraph",{"data":235,"type":233},{"text":236},"## 评估工具介绍",{"data":238,"type":233},{"text":239},"评估工具是一个功能强大、开源的框架，专门设计用于评估大型语言模型（LLMs）。由EleutherAI社区开发，它标准化了在不同任务、指标和数据集上对LLMs进行基准测试的过程。在企业LLMOps中，它作为模型选择、微调验证和持续监控的基石。",{"data":241,"type":233},{"text":242},"主要优势包括：\n- **一致性**：跨模型和任务的统一评估协议。\n- **可扩展性**：高效处理海量数据集和多个模型。\n- **可扩展性**：支持自定义任务、数据集和指标。\n- **可复现性**：通过种子随机性和缓存实现确定性结果。",{"data":244,"type":233},{"text":245},"非常适合从临时测试过渡到生产级LLM评估的团队。",{"data":247,"type":233},{"text":248},"## 先决条件与安装",{"data":250,"type":233},{"text":251},"在开始之前，请确保您的环境满足以下要求：\n- Python 3.10+。\n- GPU\u002FTPU 加速（推荐用于大型模型）。\n- 足够的内存（中型模型需要16GB以上）。",{"data":253,"type":233},{"text":254},"### Step-by-Step Installation\n1. Clone the repository:\n   ```bash\ngit clone https:\u002F\u002Fgithub.com\u002FEleutherAI\u002Flm-evaluation-harness\ngit checkout main\n```",{"data":256,"type":233},{"text":257},"2. Install dependencies:\n   ```bash\npip install -e .\npip install torch transformers datasets\n```",{"data":259,"type":233},{"text":260},"3. For specific tasks (e.g., vision-language models):\n   ```bash\npip install timm pillow\n```",{"data":262,"type":233},{"text":263},"4. Verify installation:\n   ```bash\nlm_eval --help\n```",{"data":265,"type":233},{"text":266},"专业建议：使用虚拟环境（如 `venv` 或 `conda`）来隔离依赖项。",{"data":268,"type":233},{"text":269},"## 核心概念",{"data":271,"type":233},{"text":272},"### 任务与数据集\n评估工具包开箱即用支持200多个任务，分类如下：\n- **分类任务**：ARC、BoolQ、HellaSwag。\n- **生成任务**：AlpacaEval、MT-Bench。\n- **推理任务**：GSM8K、MATH。\n- **多模态任务**：MMMU、MathVista。",{"data":274,"type":233},{"text":275},"数据集自动从Hugging Face Hub下载。",{"data":277,"type":233},{"text":278},"### 指标\n常用指标包括：\n- **准确率**：分类任务的精确匹配。\n- **F1分数**：平衡精确率与召回率。\n- **困惑度**：用于生成流畅性评估。\n- **BLEU\u002FROUGE**：翻译和摘要任务指标。",{"data":280,"type":233},{"text":281},"可通过`--metric`参数使用自定义指标。",{"data":283,"type":233},{"text":284},"### 模型加载\n支持HF Transformers、Llama.cpp、vLLM等多种框架：\n- Hugging Face：`meta-llama\u002FLlama-2-7b-chat-hf`\n- 本地模型：支持自定义路径及量化（如4位量化）。",{"data":286,"type":233},{"text":287},"## 运行基础评估",{"data":289,"type":233},{"text":290},"### Command-Line Interface (CLI)\nStart with a simple benchmark:\n```bash\nlm_eval --model hf --model_args pretrained=model_name,trust_remote_code=True --tasks hellaswag,arc_easy --device cuda:0 --batch_size auto\n```",{"data":292,"type":233},{"text":293},"详细说明：\n- `--model hf`：Hugging Face 加载器。\n- `--tasks`：逗号分隔的任务列表。\n- `--batch_size auto`：根据硬件自动优化。",{"data":295,"type":233},{"text":296},"### 解读结果\n输出包括：\n- **acc**：准确率分数。\n- **acc_stderr**：标准误差。\n- 与排行榜兼容的JSON。",{"data":298,"type":233},{"text":299},"Example output:\n```\nhellaswag: acc=0.9123 (±0.0012)\narc_easy: acc=0.7845 (±0.0021)\n```",{"data":301,"type":233},{"text":302},"## 高级用法",{"data":304,"type":233},{"text":305},"### Multi-Model Leaderboards\nCompare models:\n```bash\nlm_eval --model hf --model_args pretrained=model1 --tasks all --limit 1000\nlm_eval --model hf --model_args pretrained=model2 --tasks all --limit 1000\n```\nAggregate with `--save_jsonl` and external tools.",{"data":307,"type":233},{"text":308},"### 自定义任务\n1. 在 `lm_eval\u002Ftasks\u002F` 中定义任务：\n   - 用于数据集的 YAML 配置。\n   - 用于少样本提示的 Python 处理器。",{"data":310,"type":233},{"text":311},"2. Example custom task YAML:\n   ```yaml\ntask: my_custom_task\ndataset_path: huggingface\ndataset_name: my_dataset\ntraining_split: train\nfewshot_split: validation\nmetric_list:\n  - metric: acc\n    aggregation: mean\n    higher_is_better: true\n```",{"data":313,"type":233},{"text":314},"3. 运行：`lm_eval --tasks my_custom_task`",{"data":316,"type":233},{"text":317},"### 少样本和思维链提示\n- `--num_fewshot 5`：上下文示例。\n- 通过 `--gen_kwargs temperature=0.7` 自定义模板。",{"data":319,"type":233},{"text":320},"对于思维链：使用类似 `gsm8k_cot` 的任务。",{"data":322,"type":233},{"text":323},"## 优化与最佳实践",{"data":325,"type":233},{"text":326},"### 性能调优\n- **批处理**：`--batch_size 32` 或 `auto`。\n- **量化**：`--model_args dtype=bfloat16,load_in_4bit=True`。\n- **分布式**：`--multi_gpu` 用于 Ray 集成。",{"data":328,"type":233},{"text":329},"### 成本效率\n- 限制样本数量：`--limit 500`。\n- 使用较小的子集：`--subsample 0.1`。\n- 缓存结果：`--cache_dir \u002Fpath\u002Fto\u002Fcache`。",{"data":331,"type":233},{"text":332},"### 可靠性提示\n- 运行多个种子：`--num_generations 8`。\n- 使用自助法计算置信区间。\n- 使用 `--log_samples` 记录所有内容。",{"data":334,"type":233},{"text":335},"## 在LLMOps流水线中的集成",{"data":337,"type":233},{"text":338},"Embed in CI\u002FCD:\n1. GitHub Actions YAML:\n   ```yaml\n   - name: Evaluate Model\n     run: lm_eval --model hf --model_args pretrained=${{ inputs.model }} --tasks core --batch_size auto > results.json\n   ```",{"data":340,"type":233},{"text":341},"2. MLflow tracking:\n   ```python\nimport mlflow\nmlflow.log_metrics(results)\n```",{"data":343,"type":233},{"text":344},"3. 用于仪表板的 Prometheus\u002FGrafana。",{"data":346,"type":233},{"text":347},"## 常见问题排查",{"data":349,"type":233},{"text":350},"- **OOM错误**：减小批次大小或使用梯度检查点。\n- **CUDA内存不足**：启用 `torch.backends.cuda.enable_flash_sdp(True)`。\n- **推理速度慢**：切换到vLLM加载器：`--model vllm`。\n- **数据集未找到**：检查HF访问令牌。",{"data":352,"type":233},{"text":353},"## 结论与后续步骤",{"data":355,"type":233},{"text":356},"评估工具将主观的LLM评估转变为数据驱动的过程。从核心任务开始，扩展到自定义评估，并集成到您的LLMOps工作流程中。",{"data":358,"type":233},{"text":359},"资源：\n- GitHub：[EleutherAI\u002Flm-evaluation-harness](https:\u002F\u002Fgithub.com\u002FEleutherAI\u002Flm-evaluation-harness)\n- 排行榜：[Open LLM Leaderboard](https:\u002F\u002Fhuggingface.co\u002Fspaces\u002Fopen-llm-leaderboard\u002Fopen_llm_leaderboard)\n- Discord：EleutherAI社区。",{"data":361,"type":233},{"text":362},"立即开始实验，解锁精确的模型洞察。","2.31","本指南详细介绍了评估工具（Evaluation Harness），这是一个在企业级LLMOps流程中严格评估大型语言模型（LLM）能力的关键框架。您将学习其设置方法、最佳实践以及高级技巧，以确保模型基准测试与优化的可靠性。","\u002Fuploads\u002F2026\u002F04\u002Fevaluation-harness-1775466944495-4s0xv2.webp","evaluation-harness-1775466944495-4s0xv2","PUBLISHED","2026-03-01T17:50:00.000Z","2026-03-01T18:50:33.351Z","2026-04-06T09:49:14.730Z",{"en":372,"de":373,"sr":374,"es":375,"fr":376,"it":377,"ru":378,"zh":379},"\u002Fblog\u002Fevaluation-harness","\u002Fde\u002Fblog\u002Fevaluation-harness","\u002Fsr\u002Fblog\u002Fevaluation-harness","\u002Fes\u002Fblog\u002Fevaluation-harness","\u002Ffr\u002Fblog\u002Fevaluation-harness","\u002Fit\u002Fblog\u002Fevaluation-harness","\u002Fru\u002Fblog\u002Fevaluation-harness","\u002Fzh\u002Fblog\u002Fevaluation-harness",[381],{"id":382,"name":383,"slug":225},89,"评估框架",{"id":385,"login":386,"email":387,"displayName":388},"20","rooth8233","","Aleksandar Stajić",[390,527],{"lang":391,"title":392,"content":393,"contentJson":394,"excerpt":526},"en","Comprehensive Guide to Evaluation Harness: Mastering LLM Performance Evaluation","{\"time\":1772391033350,\"blocks\":[{\"data\":{\"text\":\"# Evaluation Harness Guide\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Introduction to Evaluation Harness\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Evaluation Harness is a powerful, open-source framework designed specifically for evaluating large language models (LLMs). Developed by the EleutherAI community, it standardizes the process of benchmarking LLMs across diverse tasks, metrics, and datasets. In enterprise LLMOps, it serves as a cornerstone for model selection, fine-tuning validation, and continuous monitoring.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Key benefits include:\\n- **Consistency**: Uniform evaluation protocols across models and tasks.\\n- **Scalability**: Handles massive datasets and multiple models efficiently.\\n- **Extensibility**: Supports custom tasks, datasets, and metrics.\\n- **Reproducibility**: Deterministic results with seeded randomness and caching.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Ideal for teams transitioning from ad-hoc testing to production-grade LLM evaluation.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Prerequisites and Installation\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Before diving in, ensure your environment meets these requirements:\\n- Python 3.10+.\\n- GPU\u002FTPU acceleration (recommended for large models).\\n- Sufficient RAM (16GB+ for mid-sized models).\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Step-by-Step Installation\\n1. Clone the repository: ```bash\\ngit clone https:\u002F\u002Fgithub.com\u002FEleutherAI\u002Flm-evaluation-harness\\ngit checkout main\\n```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"2. Install dependencies: ```bash\\npip install -e .\\npip install torch transformers datasets\\n```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"3. For specific tasks (e.g., vision-language models): ```bash\\npip install timm pillow\\n```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"4. Verify installation: ```bash\\nlm_eval --help\\n```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Pro tip: Use a virtual environment like `venv` or `conda` to isolate dependencies.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Core Concepts\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Tasks and Datasets\\nEvaluation Harness supports 200+ tasks out-of-the-box, categorized as:\\n- **Classification**: ARC, BoolQ, HellaSwag.\\n- **Generative**: AlpacaEval, MT-Bench.\\n- **Reasoning**: GSM8K, MATH.\\n- **Multimodal**: MMMU, MathVista.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Datasets auto-download from Hugging Face Hub.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Metrics\\nCommon metrics include:\\n- **Accuracy**: Exact match for classification.\\n- **F1**: Balanced precision\u002Frecall.\\n- **Perplexity**: For generative fluency.\\n- **BLEU\u002FROUGE**: Translation and summarization.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Custom metrics via `--metric` flag.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Model Loading\\nSupports HF Transformers, Llama.cpp, vLLM, and more:\\n- Hugging Face: `meta-llama\u002FLlama-2-7b-chat-hf`\\n- Local: Custom paths with quantization (e.g., 4-bit).\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Running Basic Evaluations\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Command-Line Interface (CLI)\\nStart with a simple benchmark:\\n```bash\\nlm_eval --model hf --model_args pretrained=model_name,trust_remote_code=True --tasks hellaswag,arc_easy --device cuda:0 --batch_size auto\\n```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Breakdown:\\n- `--model hf`: Hugging Face loader.\\n- `--tasks`: Comma-separated tasks.\\n- `--batch_size auto`: Optimizes for hardware.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Interpreting Results\\nOutput includes:\\n- **acc**: Accuracy score.\\n- **acc_stderr**: Standard error.\\n- Leaderboard-compatible JSON.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Example output:\\n```\\nhellaswag: acc=0.9123 (±0.0012)\\narc_easy: acc=0.7845 (±0.0021)\\n```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Advanced Usage\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Multi-Model Leaderboards\\nCompare models:\\n```bash\\nlm_eval --model hf --model_args pretrained=model1 --tasks all --limit 1000\\nlm_eval --model hf --model_args pretrained=model2 --tasks all --limit 1000\\n```\\nAggregate with `--save_jsonl` and external tools.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Custom Tasks\\n1. Define task in `lm_eval\u002Ftasks\u002F`: - YAML config for dataset. - Python processor for few-shot prompting.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"2. Example custom task YAML: ```yaml\\ntask: my_custom_task\\ndataset_path: huggingface\\ndataset_name: my_dataset\\ntraining_split: train\\nfewshot_split: validation\\nmetric_list: - metric: acc aggregation: mean higher_is_better: true\\n```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"3. Run: `lm_eval --tasks my_custom_task`\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Few-Shot and Chain-of-Thought Prompting\\n- `--num_fewshot 5`: In-context examples.\\n- Custom templates via `--gen_kwargs temperature=0.7`.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"For CoT: Use tasks like `gsm8k_cot`.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Optimization and Best Practices\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Performance Tuning\\n- **Batching**: `--batch_size 32` or `auto`.\\n- **Quantization**: `--model_args dtype=bfloat16,load_in_4bit=True`.\\n- **Distributed**: `--multi_gpu` for Ray integration.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Cost Efficiency\\n- Limit samples: `--limit 500`.\\n- Use smaller subsets: `--subsample 0.1`.\\n- Cache results: `--cache_dir \u002Fpath\u002Fto\u002Fcache`.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"### Reliability Tips\\n- Run multiple seeds: `--num_generations 8`.\\n- Bootstrap confidence intervals.\\n- Log everything with `--log_samples`.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Integration in LLMOps Pipelines\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Embed in CI\u002FCD:\\n1. GitHub Actions YAML: ```yaml - name: Evaluate Model run: lm_eval --model hf --model_args pretrained=${{ inputs.model }} --tasks core --batch_size auto > results.json ```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"2. MLflow tracking: ```python\\nimport mlflow\\nmlflow.log_metrics(results)\\n```\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"3. Prometheus\u002FGrafana for dashboards.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Troubleshooting Common Issues\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"- **OOM Errors**: Reduce batch size or use gradient checkpointing.\\n- **CUDA Out of Memory**: Enable `torch.backends.cuda.enable_flash_sdp(True)`.\\n- **Slow Inference**: Switch to vLLM loader: `--model vllm`.\\n- **Dataset Not Found**: Check HF access token.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"## Conclusion and Next Steps\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Evaluation Harness transforms subjective LLM assessment into a data-driven process. Start with core tasks, scale to custom evals, and integrate into your LLMOps workflow.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Resources:\\n- GitHub: [EleutherAI\u002Flm-evaluation-harness](https:\u002F\u002Fgithub.com\u002FEleutherAI\u002Flm-evaluation-harness)\\n- Leaderboard: [Open LLM Leaderboard](https:\u002F\u002Fhuggingface.co\u002Fspaces\u002Fopen-llm-leaderboard\u002Fopen_llm_leaderboard)\\n- Discord: EleutherAI community.\"},\"type\":\"paragraph\"},{\"data\":{\"text\":\"Experiment today to unlock precise model insights.\"},\"type\":\"paragraph\"}],\"version\":\"2.31\"}",{"time":395,"blocks":396,"version":363},1772391033350,[397,400,403,406,409,412,415,418,421,424,427,430,433,436,439,442,445,448,451,454,456,459,462,464,467,469,472,475,478,481,484,487,490,493,496,499,502,505,508,511,514,517,520,523],{"data":398,"type":233},{"text":399},"# Evaluation Harness Guide",{"data":401,"type":233},{"text":402},"## Introduction to Evaluation Harness",{"data":404,"type":233},{"text":405},"Evaluation Harness is a powerful, open-source framework designed specifically for evaluating large language models (LLMs). Developed by the EleutherAI community, it standardizes the process of benchmarking LLMs across diverse tasks, metrics, and datasets. In enterprise LLMOps, it serves as a cornerstone for model selection, fine-tuning validation, and continuous monitoring.",{"data":407,"type":233},{"text":408},"Key benefits include:\n- **Consistency**: Uniform evaluation protocols across models and tasks.\n- **Scalability**: Handles massive datasets and multiple models efficiently.\n- **Extensibility**: Supports custom tasks, datasets, and metrics.\n- **Reproducibility**: Deterministic results with seeded randomness and caching.",{"data":410,"type":233},{"text":411},"Ideal for teams transitioning from ad-hoc testing to production-grade LLM evaluation.",{"data":413,"type":233},{"text":414},"## Prerequisites and Installation",{"data":416,"type":233},{"text":417},"Before diving in, ensure your environment meets these requirements:\n- Python 3.10+.\n- GPU\u002FTPU acceleration (recommended for large models).\n- Sufficient RAM (16GB+ for mid-sized models).",{"data":419,"type":233},{"text":420},"### Step-by-Step Installation\n1. Clone the repository: ```bash\ngit clone https:\u002F\u002Fgithub.com\u002FEleutherAI\u002Flm-evaluation-harness\ngit checkout main\n```",{"data":422,"type":233},{"text":423},"2. Install dependencies: ```bash\npip install -e .\npip install torch transformers datasets\n```",{"data":425,"type":233},{"text":426},"3. For specific tasks (e.g., vision-language models): ```bash\npip install timm pillow\n```",{"data":428,"type":233},{"text":429},"4. Verify installation: ```bash\nlm_eval --help\n```",{"data":431,"type":233},{"text":432},"Pro tip: Use a virtual environment like `venv` or `conda` to isolate dependencies.",{"data":434,"type":233},{"text":435},"## Core Concepts",{"data":437,"type":233},{"text":438},"### Tasks and Datasets\nEvaluation Harness supports 200+ tasks out-of-the-box, categorized as:\n- **Classification**: ARC, BoolQ, HellaSwag.\n- **Generative**: AlpacaEval, MT-Bench.\n- **Reasoning**: GSM8K, MATH.\n- **Multimodal**: MMMU, MathVista.",{"data":440,"type":233},{"text":441},"Datasets auto-download from Hugging Face Hub.",{"data":443,"type":233},{"text":444},"### Metrics\nCommon metrics include:\n- **Accuracy**: Exact match for classification.\n- **F1**: Balanced precision\u002Frecall.\n- **Perplexity**: For generative fluency.\n- **BLEU\u002FROUGE**: Translation and summarization.",{"data":446,"type":233},{"text":447},"Custom metrics via `--metric` flag.",{"data":449,"type":233},{"text":450},"### Model Loading\nSupports HF Transformers, Llama.cpp, vLLM, and more:\n- Hugging Face: `meta-llama\u002FLlama-2-7b-chat-hf`\n- Local: Custom paths with quantization (e.g., 4-bit).",{"data":452,"type":233},{"text":453},"## Running Basic Evaluations",{"data":455,"type":233},{"text":290},{"data":457,"type":233},{"text":458},"Breakdown:\n- `--model hf`: Hugging Face loader.\n- `--tasks`: Comma-separated tasks.\n- `--batch_size auto`: Optimizes for hardware.",{"data":460,"type":233},{"text":461},"### Interpreting Results\nOutput includes:\n- **acc**: Accuracy score.\n- **acc_stderr**: Standard error.\n- Leaderboard-compatible JSON.",{"data":463,"type":233},{"text":299},{"data":465,"type":233},{"text":466},"## Advanced Usage",{"data":468,"type":233},{"text":305},{"data":470,"type":233},{"text":471},"### Custom Tasks\n1. Define task in `lm_eval\u002Ftasks\u002F`: - YAML config for dataset. - Python processor for few-shot prompting.",{"data":473,"type":233},{"text":474},"2. Example custom task YAML: ```yaml\ntask: my_custom_task\ndataset_path: huggingface\ndataset_name: my_dataset\ntraining_split: train\nfewshot_split: validation\nmetric_list: - metric: acc aggregation: mean higher_is_better: true\n```",{"data":476,"type":233},{"text":477},"3. Run: `lm_eval --tasks my_custom_task`",{"data":479,"type":233},{"text":480},"### Few-Shot and Chain-of-Thought Prompting\n- `--num_fewshot 5`: In-context examples.\n- Custom templates via `--gen_kwargs temperature=0.7`.",{"data":482,"type":233},{"text":483},"For CoT: Use tasks like `gsm8k_cot`.",{"data":485,"type":233},{"text":486},"## Optimization and Best Practices",{"data":488,"type":233},{"text":489},"### Performance Tuning\n- **Batching**: `--batch_size 32` or `auto`.\n- **Quantization**: `--model_args dtype=bfloat16,load_in_4bit=True`.\n- **Distributed**: `--multi_gpu` for Ray integration.",{"data":491,"type":233},{"text":492},"### Cost Efficiency\n- Limit samples: `--limit 500`.\n- Use smaller subsets: `--subsample 0.1`.\n- Cache results: `--cache_dir \u002Fpath\u002Fto\u002Fcache`.",{"data":494,"type":233},{"text":495},"### Reliability Tips\n- Run multiple seeds: `--num_generations 8`.\n- Bootstrap confidence intervals.\n- Log everything with `--log_samples`.",{"data":497,"type":233},{"text":498},"## Integration in LLMOps Pipelines",{"data":500,"type":233},{"text":501},"Embed in CI\u002FCD:\n1. GitHub Actions YAML: ```yaml - name: Evaluate Model run: lm_eval --model hf --model_args pretrained=${{ inputs.model }} --tasks core --batch_size auto > results.json ```",{"data":503,"type":233},{"text":504},"2. MLflow tracking: ```python\nimport mlflow\nmlflow.log_metrics(results)\n```",{"data":506,"type":233},{"text":507},"3. Prometheus\u002FGrafana for dashboards.",{"data":509,"type":233},{"text":510},"## Troubleshooting Common Issues",{"data":512,"type":233},{"text":513},"- **OOM Errors**: Reduce batch size or use gradient checkpointing.\n- **CUDA Out of Memory**: Enable `torch.backends.cuda.enable_flash_sdp(True)`.\n- **Slow Inference**: Switch to vLLM loader: `--model vllm`.\n- **Dataset Not Found**: Check HF access token.",{"data":515,"type":233},{"text":516},"## Conclusion and Next Steps",{"data":518,"type":233},{"text":519},"Evaluation Harness transforms subjective LLM assessment into a data-driven process. Start with core tasks, scale to custom evals, and integrate into your LLMOps workflow.",{"data":521,"type":233},{"text":522},"Resources:\n- GitHub: [EleutherAI\u002Flm-evaluation-harness](https:\u002F\u002Fgithub.com\u002FEleutherAI\u002Flm-evaluation-harness)\n- Leaderboard: [Open LLM Leaderboard](https:\u002F\u002Fhuggingface.co\u002Fspaces\u002Fopen-llm-leaderboard\u002Fopen_llm_leaderboard)\n- Discord: EleutherAI community.",{"data":524,"type":233},{"text":525},"Experiment today to unlock precise model insights.","This guide provides a detailed walkthrough of Evaluation Harness, an essential framework for rigorously assessing large language model (LLM) capabilities in enterprise LLMOps pipelines. Learn setup, best practices, and advanced techniques to ensure reliable model benchmarking and optimization.",{"lang":7,"title":224,"content":226,"contentJson":528,"excerpt":364},{"time":228,"blocks":529,"version":363},[530,532,534,536,538,540,542,544,546,548,550,552,554,556,558,560,562,564,566,568,570,572,574,576,578,580,582,584,586,588,590,592,594,596,598,600,602,604,606,608,610,612,614,616],{"data":531,"type":233},{"text":232},{"data":533,"type":233},{"text":236},{"data":535,"type":233},{"text":239},{"data":537,"type":233},{"text":242},{"data":539,"type":233},{"text":245},{"data":541,"type":233},{"text":248},{"data":543,"type":233},{"text":251},{"data":545,"type":233},{"text":254},{"data":547,"type":233},{"text":257},{"data":549,"type":233},{"text":260},{"data":551,"type":233},{"text":263},{"data":553,"type":233},{"text":266},{"data":555,"type":233},{"text":269},{"data":557,"type":233},{"text":272},{"data":559,"type":233},{"text":275},{"data":561,"type":233},{"text":278},{"data":563,"type":233},{"text":281},{"data":565,"type":233},{"text":284},{"data":567,"type":233},{"text":287},{"data":569,"type":233},{"text":290},{"data":571,"type":233},{"text":293},{"data":573,"type":233},{"text":296},{"data":575,"type":233},{"text":299},{"data":577,"type":233},{"text":302},{"data":579,"type":233},{"text":305},{"data":581,"type":233},{"text":308},{"data":583,"type":233},{"text":311},{"data":585,"type":233},{"text":314},{"data":587,"type":233},{"text":317},{"data":589,"type":233},{"text":320},{"data":591,"type":233},{"text":323},{"data":593,"type":233},{"text":326},{"data":595,"type":233},{"text":329},{"data":597,"type":233},{"text":332},{"data":599,"type":233},{"text":335},{"data":601,"type":233},{"text":338},{"data":603,"type":233},{"text":341},{"data":605,"type":233},{"text":344},{"data":607,"type":233},{"text":347},{"data":609,"type":233},{"text":350},{"data":611,"type":233},{"text":353},{"data":613,"type":233},{"text":356},{"data":615,"type":233},{"text":359},{"data":617,"type":233},{"text":362},"Post erfolgreich abgerufen",{"items":620,"source":649,"manualIds":650,"manualMatchedIds":651},[621,628,635,642],{"id":622,"slug":623,"title":624,"excerpt":625,"featuredImage":626,"publishedAt":627},"469","rag-failed-but-which-layer-actually-failed-a-diagnostic-method","RAG失败了——但究竟是哪一层真正失败了？一种诊断方法","当RAG答案出错时，将问题归咎于检索或模型过于笼统。这种诊断方法将来源覆盖、查询构建、检索、排序、上下文组装、生成、证据归因和时效性逐一隔离，从而使实际故障能够被复现并修复。","\u002Fuploads\u002F2026\u002F09\u002Frag-failed-but-which-layer-actually-failed-a-diagnostic-method-1790350847177-pior4c.webp","2026-09-24T19:39:00.000Z",{"id":629,"slug":630,"title":631,"excerpt":632,"featuredImage":633,"publishedAt":634},"475","managed-agent-harness-vs-self-hosted-agent-loop-what-you-gain-what-you-lose","托管代理框架与自托管代理循环：你得到什么，失去什么","“自托管代理”可能意味着截然不同的架构。本指南区分了托管式运行框架、自托管执行环境和完全自主运营的代理循环——并说明了团队实际需要哪种控制边界。","\u002Fuploads\u002F2026\u002F09\u002Fmanaged-agent-harness-vs-self-hosted-agent-loop-what-you-gain-what-you-lose-1790352403475-kj10jh.webp","2026-09-25T12:05:00.000Z",{"id":636,"slug":637,"title":638,"excerpt":639,"featuredImage":640,"publishedAt":641},"493","mlops-vs-llmops-what-changes-when-the-model-is-an-llm","MLOps 与 LLMOps：当模型是 LLM 时，会发生哪些变化","MLOps 运维机器学习系统；LLMOps 将这些实践扩展到围绕大型语言模型的提示、上下文、检索、提供商、工具、评估和运行时行为。","\u002Fuploads\u002F2026\u002F10\u002Fmlops-vs-llmops-what-changes-when-the-model-is-an-llm-1791487319869-2v7hxo.webp","2026-10-08T15:20:00.000Z",{"id":643,"slug":644,"title":645,"excerpt":646,"featuredImage":647,"publishedAt":648},"459","ollama-is-not-the-product-building-production-ready-open-llm-applications","Ollama 并非产品：构建可投入生产的开源大语言模型应用","使用Ollama运行本地模型很简单。但构建一个可用于生产环境的开源大语言模型（Open-LLM）应用则更具挑战性：它需要RAG（检索增强生成）、访问控制、供应商抽象、评估、日志记录、部署规范，以及围绕模型构建受控的应用层。","\u002Fuploads\u002F2026\u002F06\u002Follama-is-not-the-product-building-production-ready-open-llm-applications-1782679361640-h0usqf.webp","2026-06-28T16:39:00.000Z","fallback",[],[]]