07 Quick Start
Environment Setup & Inference
Environment Setup
conda create -n moss_vl python=3.12 pip -y
conda activate moss_vl
pip install -i https://pypi.org/simple --no-build-isolation -r requirements.txt
Model Inference
For more ready-to-run inference examples and demo assets, see inference/README.md. Inference supports full-modality offline queries, including text-only, single-image, multi-image, single-video, multi-video, and interleaved image–video inputs via the messages format.
Single-query inference with offline_generate
import queue
import threading
import torch
from transformers import AutoModelForCausalLM, AutoProcessor
checkpoint = "/path/to/dummy-checkpoint"
processor = AutoProcessor.from_pretrained(
checkpoint,
trust_remote_code=True,
frame_extract_num_threads=1,
)
model = AutoModelForCausalLM.from_pretrained(
checkpoint,
trust_remote_code=True,
device_map="auto",
torch_dtype=torch.bfloat16,
attn_implementation="flash_attention_2",
)
query = {
"messages": [
{
"role": "user",
"content": [
{"type": "image", "image": "path/to/example.jpg"},
{"type": "text", "text": "Describe this image."},
],
}
],
"media_kwargs": {},
"generate_kwargs": {
"max_new_tokens": 256,
"do_sample": False,
"vision_chunked_length": 64,
},
}
input_queue = queue.Queue()
output_queue = queue.Queue()
worker = threading.Thread(
target=model.offline_generate,
args=(processor, input_queue, output_queue),
kwargs={"vision_chunked_length": 64},
daemon=True,
)
worker.start()
input_queue.put(query)
text_chunks = []
while True:
item = output_queue.get()
if item in {"<|round_start|>"}:
continue
if item == "<|round_end|>":
break
text_chunks.append(item)
print("".join(text_chunks))
input_queue.put({"stop_offline_generate": True})
worker.join()
For simple batch offline inference, you can also use offline_batch_generate directly:
Batch inference with offline_batch_generate
import torch
from transformers import AutoModelForCausalLM, AutoProcessor
checkpoint = "/path/to/dummy-checkpoint"
processor = AutoProcessor.from_pretrained(
checkpoint,
trust_remote_code=True,
frame_extract_num_threads=1,
)
model = AutoModelForCausalLM.from_pretrained(
checkpoint,
trust_remote_code=True,
device_map="auto",
torch_dtype=torch.bfloat16,
attn_implementation="flash_attention_2",
)
queries = [
{
"messages": [
{
"role": "user",
"content": [{"type": "text", "text": "Describe sample A."}],
}
],
"media_kwargs": {},
"generate_kwargs": {"max_new_tokens": 256, "do_sample": False},
},
{
"messages": [
{
"role": "user",
"content": [{"type": "text", "text": "Describe sample B."}],
}
],
"media_kwargs": {},
"generate_kwargs": {"max_new_tokens": 256, "do_sample": False},
},
]
with torch.no_grad():
result = model.offline_batch_generate(
processor,
queries,
vision_chunked_length=64,
)
texts = [item["text"] for item in result["results"]]
print(texts)
Fine-Tuning
We provide a lightweight SFT fine-tuning framework built on HuggingFace transformers.Trainer, supporting both full-parameter training and LoRA, with independent control over whether the vision encoder, language model, and LM head participate in training.
# Full-parameter SFT (vision encoder frozen by default)
bash finetune/scripts/run_sft.sh
# LoRA SFT
pip install -i https://pypi.org/simple peft
bash finetune/scripts/run_sft_lora.sh
Training data uses a JSON structure compatible with the inference query format, requiring only an extra response field:
[
{
"prompt": "Describe this image.",
"response": "A beautiful landscape painting surrounded by mountains.",
"images": ["path/to/image.jpg"],
"videos": []
}
]
Multi-turn dialogue formats are also supported; see finetune/README.md for full documentation.
