1.数据格式转换
模版格式:
[
{
"id": "identity_0",
"conversations": [
{
"from": "user",
"value": "你好"
},
{
"from": "assistant",
"value": "我是Qwen-VL,一个支持视觉输入的大模型。"
}
]
},
{
"id": "identity_1",
"conversations": [
{
"from": "user",
"value": "Picture 1: <img>https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg</img>\n图中的狗是什么品种?"
},
{
"from": "assistant",
"value": "图中是一只拉布拉多犬。"
},
{
"from": "user",
"value": "框出图中的格子衬衫"
},
{
"from": "assistant",
"value": "<ref>格子衬衫</ref><box>(588,499),(725,789)</box>"
}
]
},
{
"id": "identity_2",
"conversations": [
{
"from": "user",
"value": "Picture 1: <img>assets/mm_tutorial/Chongqing.jpeg</img>\nPicture 2: <img>assets/mm_tutorial/Beijing.jpeg</img>\n图中都是哪"
},
{
"from": "assistant",
"value": "第一张图片是重庆的城市天际线,第二张图片是北京的天际线。"
}
]
}
]
import json
# 打开并加载.json文件
with open(r"E:\comprehensive_library\e_commerce_lmm\data\openi-zh-prompt.json", 'r', encoding='utf-8') as f:
data = json.load(f)
# 数据格式转换
new_data = []
for i, d in enumerate(data):
new_conversations = dict()
new_conversations["id"] = f"identity_{i}"
new_conversations["conversations"] = [
{
"from": "user",
"value": "Picture 1: <img>" + d["img"] + "</img>\n" + d["prompt"]
},
{
"from": "assistant",
"value": d["label"]
}
]
new_data.append(new_conversations)
# 将新的数据写入新的.json文件中
with open("../data/openai-zh-qwenvl-prompt.json", 'w', encoding='utf-8') as f:
json.dump(new_data, f, ensure_ascii=False, indent=4)
2.微调
直接pip install -r requirments.txt
注意gcc要升级到9.3
yum -y install centos-release-scl
yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-binutils
scl enable devtoolset-9 bash
echo "source /opt/rh/devtoolset-9/enable" >>/etc/profile
lora在V100上显存不够,微调不起来,在a800上可以。用swift库可以在V100上微调Qwen-vl。
#!/bin/bash
export CUDA_DEVICE_MAX_CONNECTIONS=1
DIR=$(pwd)
GPUS_PER_NODE=$(python -c 'import torch; print(torch.cuda.device_count())')
NNODES=1
NODE_RANK=0
MASTER_ADDR=localhost
MASTER_PORT=6001
MODEL="/home/image_team/image_team_docker_home/lgd/e_commerce_lmm/weights/qwen-vl-caht/" #"Qwen/Qwen-VL-Chat"/"Qwen/Qwen-VL" Set the path if you do not want to load from huggingface directly
# ATTENTION: specify the path to your training data, which should be a json file consisting of a list of conversations.
# See the section for finetuning in README for more information.
DATA="/home/image_team/image_team_docker_home/lgd/e_commerce_lmm/data/openai-zh-qwenvl-prompt.json"
DISTRIBUTED_ARGS="
--nproc_per_node $GPUS_PER_NODE \
--nnodes $NNODES \
--node_rank $NODE_RANK \
--master_addr $MASTER_ADDR \
--master_port $MASTER_PORT
"
torchrun $DISTRIBUTED_ARGS finetune.py \
--model_name_or_path $MODEL \
--data_path $DATA \
--bf16 False \
--fp16 True \
--fix_vit True \
--output_dir output_qwen \
--num_train_epochs 5 \
--per_device_train_batch_size 1 \
--per_device_eval_batch_size 1 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 1000 \
--save_total_limit 10 \
--learning_rate 1e-5 \
--weight_decay 0.1 \
--adam_beta2 0.95 \
--warmup_ratio 0.01 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--report_to "none" \
--model_max_length 1024 \
--lazy_preprocess True \
--use_lora \
--gradient_checkpointing \
--deepspeed finetune/ds_config_zero2.json