阿里开源多模态全能王 Qwen2.5-Omni：创新Thinker-Talker架构，全面超越Gemini-1.5-Pro等竞品

# 安装 transformers 和 accelerate pip uninstall transformers pip install git+https://github.com/huggingface/transformers@3a1ead0aabed473eafeeea8c197d pip install accelerate # 安装 Qwen2.5-Omni 工具包 pip install qwen-omni-utils[decord]

pip install qwen-omni-utils

import soundfile as sf from transformers import Qwen2_5OmniModel, Qwen2_5OmniProcessor from qwen_omni_utils import process_mm_info # 加载模型和处理器 model = Qwen2_5OmniModel.from_pretrained("Qwen/Qwen2.5-Omni-7B", torch_dtype="auto", device_map="auto") processor = Qwen2_5OmniProcessor.from_pretrained("Qwen/Qwen2.5-Omni-7B") # 定义对话内容 conversation = [ { "role": "system", "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.", }, { "role": "user", "content": [ {"type": "video", "video": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"}, ], }, ] # 数据预处理 text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False) audios, images, videos = process_mm_info(conversation, use_audio_in_video=True) inputs = processor(text=text, audios=audios, images=images, videos=videos, return_tensors="pt", padding=True) inputs = inputs.to(model.device).to(model.dtype) # 模型推理 text_ids, audio = model.generate(inputs, use_audio_in_video=True) # 解码输出 text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) print("生成的文本:", text) # 保存生成的音频 sf.write("output.wav", audio.reshape(-1).detach().cpu().numpy(), samplerate=24000)

pip install -U flash-attn --no-build-isolation

from transformers import Qwen2_5OmniModel model = Qwen2_5OmniModel.from_pretrained( "Qwen/Qwen2.5-Omni-7B", device_map="auto", torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2", )

# 示例对话 conversation1 = [{"role": "user", "content": [{"type": "video", "video": "/path/to/video.mp4"}]}] conversation2 = [{"role": "user", "content": [{"type": "audio", "audio": "/path/to/audio.wav"}]}] conversation3 = [{"role": "user", "content": "who are you?"}] conversation4 = [ {"role": "user", "content": [ {"type": "image", "image": "/path/to/image.jpg"}, {"type": "video", "video": "/path/to/video.mp4"}, {"type": "audio", "audio": "/path/to/audio.wav"}, {"type": "text", "text": "What are the elements can you see and hear in these medias?"} ]} ] # 批量处理 conversations = [conversation1, conversation2, conversation3, conversation4] text = processor.apply_chat_template(conversations, add_generation_prompt=True, tokenize=False) audios, images, videos = process_mm_info(conversations, use_audio_in_video=True) inputs = processor(text=text, audios=audios, images=images, videos=videos, return_tensors="pt", padding=True) inputs = inputs.to(model.device).to(model.dtype) # 批量推理 text_ids = model.generate(inputs, use_audio_in_video=True, return_audio=False) text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) print("批量生成的文本:", text)

import base64 import numpy as np import soundfile as sf from openai import OpenAI # 初始化客户端 client = OpenAI( api_key="your_api_key", base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", ) # 定义对话内容 messages = [ { "role": "system", "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.", }, { "role": "user", "content": [ {"type": "video_url", "video_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"}, ], }, ] # 流式推理 completion = client.chat.completions.create( model="qwen-omni-turbo", messages=messages, modalities=["text", "audio"], audio={"voice": "Chelsie", "format": "wav"}, stream=True, stream_options={"include_usage": True} ) # 处理流式输出 text, audio_string = [], "" for chunk in completion: if chunk.choices: if hasattr(chunk.choices[0].delta, "audio"): try: audio_string += chunk.choices[0].delta.audio["data"] except Exception as e: text.append(chunk.choices[0].delta.audio["transcript"]) else: print(chunk.usage) # 保存生成的文本和音频 print("".join(text)) wav_bytes = base64.b64decode(audio_string) wav_array = np.frombuffer(wav_bytes, dtype=np.int16) sf.write("output.wav", wav_array, samplerate=24000)

# 安装依赖 pip install -r requirements_web_demo.txt # 启动 Web 演示 python web_demo.py --flash-attn2

# 启动 Docker 容器 docker run --gpus all --ipc=host --network=host --rm --name qwen2.5-omni -it qwenllm/qwen-omni:2.5-cu121 bash # 启动 Web 演示 bash docker/docker_web_demo.sh --checkpoint /path/to/Qwen2.5-Omni-7B --flash-attn2