# 对话模式
llama-cli -m model.gguf

# > hi, who are you?
# Hi there! I'm your helpful assistant! I'm an AI-powered chatbot designed to assist and provide information to users like you. I'm here to help answer your questions, provide guidance, and offer support on a wide range of topics. I'm a friendly and knowledgeable AI, and I'm always happy to help with anything you need. What's on your mind, and how can I assist you today?
#
# > what is 1+1?
# Easy peasy! The answer to 1+1 is... 2!

# 以定制聊天模版进行对话
# use the "chatml" template (use -h to see the list of supported templates)
llama-cli -m model.gguf -cnv --chat-template chatml

# use a custom template
llama-cli -m model.gguf -cnv --in-prefix 'User: ' --reverse-prompt 'User:'

# 补全续写方式
llama-cli -m model.gguf -p "I believe the meaning of life is" -n 128 -no-cnv

# I believe the meaning of life is to find your own truth and to live in accordance with it. For me, this means being true to myself and following my passions, even if they don't align with societal expectations. I think that's what I love about yoga – it's not just a physical practice, but a spiritual one too. It's about connecting with yourself, listening to your inner voice, and honoring your own unique journey.

# 通过自定义指令限制输出
llama-cli -m model.gguf -n 256 --grammar-file grammars/json.gbnf -p 'Request: schedule a call at 8pm; Command:'

# {"appointmentTime": "8pm", "appointmentDetails": "schedule a a call"}

# 基本启用
llama-server -m model.gguf --port 8080

# Basic web UI can be accessed via browser: http://localhost:8080
# Chat completion endpoint: http://localhost:8080/v1/chat/completions

# 多用户使用
# up to 4 concurrent requests, each with 4096 max context
llama-server -m model.gguf -c 16384 -np 4

# Enable speculative decoding（启用推测解码）是一种通过​​并行化推理过程​​来加速大语言模型（LLM）生成文本的技术，其核心思想是“先推测后验证”（Draft-then-Verify）。
# 双模型协作机制​​
# ​​目标模型（Target Model）​​：主模型（如LLaMA-70B），负责最终生成结果的准确性。
# ​​草稿模型（Draft Model）​​：小型辅助模型（如LLaMA-7B），快速生成候选词（draft tokens）。
# 两者需共享相同的分词器（Tokenizer）。
# the draft.gguf model should be a small variant of the target model.gguf
llama-server -m model.gguf -md draft.gguf


# “Serve an embedding model”（部署/服务嵌入模型）指将嵌入模型（Embedding Model）转化为可被其他系统调用的​​在线推理服务​​，使其能实时将文本、图像等非结构化数据转化为向量（Embeddings），供下游任务（如搜索、聚类、推荐）使用。
# 将嵌入模型封装为 RESTful 或 gRPC 接口，支持输入文本并返回向量。例如，使用 TensorFlow Serving 或 ONNX Runtime 将模型部署为 HTTP 端点。
# use the /embedding endpoint
llama-server -m model.gguf --embedding --pooling cls -ub 8192

# “Serve a reranking model”（部署/服务重排序模型）指将重排序模型（Reranking Model）转化为可被外部系统调用的​​在线推理服务​​，其核心功能是对初步检索到的候选结果（如文本段落、商品列表等）进行相关性评分与重新排序，以提升最终输出的精准度。
# use the /reranking endpoint
llama-server -m model.gguf --reranking

# 按照指令限制模型的所有输出
# custom grammar
llama-server -m model.gguf --grammar-file grammar.gbnf

# JSON
llama-server -m model.gguf --grammar-file grammars/json.gbnf

# 测试模型性能
llama-bench -m model.gguf

# Output:
# | model               |       size |     params | backend    | threads |          test |                  t/s |
# | ------------------- | ---------: | ---------: | ---------- | ------: | ------------: | -------------------: |
# | qwen2 1.5B Q4_0     | 885.97 MiB |     1.54 B | Metal,BLAS |      16 |         pp512 |      5765.41 ± 20.55 |
# | qwen2 1.5B Q4_0     | 885.97 MiB |     1.54 B | Metal,BLAS |      16 |         tg128 |        197.71 ± 0.81 |
#
# build: 3e0ba0e60 (4229)

llama-run granite-code

llama-simple -m model.gguf

# Hello my name is Kaitlyn and I am a 16 year old girl. I am a junior in high school and I am currently taking a class called "The Art of

# obtain the official LLaMA model weights and place them in ./models
ls ./models
llama-2-7b tokenizer_checklist.chk tokenizer.model
# [Optional] for models using BPE tokenizers
ls ./models
<folder containing weights and tokenizer json> vocab.json
# [Optional] for PyTorch .bin models like Mistral-7B
ls ./models
<folder containing weights and tokenizer json>

# install Python dependencies
python3 -m pip install -r requirements.txt

# convert the model to ggml FP16 format
python3 convert_hf_to_gguf.py models/mymodel/

# quantize the model to 4-bits (using Q4_K_M method)
./llama-quantize ./models/mymodel/ggml-model-f16.gguf ./models/mymodel/ggml-model-Q4_K_M.gguf Q4_K_M

# update the gguf filetype to current version if older version is now unsupported
./llama-quantize ./models/mymodel/ggml-model-Q4_K_M.gguf ./models/mymodel/ggml-model-Q4_K_M-v2.gguf COPY

# start inference on a gguf model
./llama-cli -m ./models/mymodel/ggml-model-Q4_K_M.gguf -cnv -p "You are a helpful assistant"