>>> import os
>>> import torch
>>> from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
>>> os.environ["CUDA_VISIBLE_DEVICES"] = '0'
>>> tokenizer = AutoTokenizer.from_pretrained('../models/12B', trust_remote_code=True)
>>> model = AutoModelForCausalLM.from_pretrained('../models/12B', trust_remote_code=True, device_map="auto", torch_dtype=torch.float16)
>>> generate_config = GenerationConfig.from_pretrained('../models/12B')
>>> question="生抽与老抽的区别？"
>>> answer, history = model.chat(tokenizer = tokenizer, question=question, history=[], generation_config=generate_config, stream=False)
>>> print(answer)
生抽和老抽是两种不同的酱油，它们在风味、色泽和用途上都有所区别。

1. 颜色：生抽的颜色比较淡，而老抽的颜色较深。生抽的颜色呈红褐色或棕红色，而老抽的颜色则呈棕黑色。

2. 味道：生抽具有鲜美的咸味和微甜的味浅，而老抽浓郁，颜色较深。根据个人口味和烹饪需求选择不同的酱油类型可以获得更好的口感和菜肴效果。

{
  "datas/single_turn_example.jsonl": 2.0,
  "datas/multi_turn_example.jsonl": 1.0
}

python -u process_data.py \
   --data_path data.json \ # 数据配比文件路径
   --tokenizer_path ../../models/12B \ # 模型/tokenzier路径
   --data_output_path $DATA_OUTPUT_PATH \ # 处理后数据保存地址
   --max_seq_len $MAX_LEN \ # 数据长度
   --num_samples $NUM_SAMPLES \ # 最终生成拼接后的数据数量
   --num_workers 10 \ # 多进程个数
   --process_method multiple \ # 多进程&单进程处理
   --seed 42

deepspeed --master_port 29500 main.py \
   --data_path ${DATA_OUTPUT_PATH}/train_data.pt  \ # tokenzie后的数据文件
   --model_name_or_path ../../models/12B \
   --with_loss_mask \
   --per_device_train_batch_size 1 \
   --max_seq_len 4096 \
   --learning_rate 3e-5 \
   --weight_decay 0.0001 \
   --num_train_epochs 1 \
   --gradient_accumulation_steps 4 \
   --lr_scheduler_type cosine \
   --precision fp16 \ # 训练精度，fp16或bf16
   --warmup_proportion 0.1 \ 
   --gradient_checkpointing \
   --offload \
   --seed 1233 \
   --zero_stage $ZERO_STAGE \ 
   --save_steps 10 \
   --deepspeed \ 
   --output_dir $OUTPUT # 输出路径 

deepspeed --master_port 29500 --hostfile=my_hostfile main.py \
   --data_path ${DATA_OUTPUT_PATH}/train_data.pt  \ # tokenzie后的数据文件
   --model_name_or_path ../../models/12B \
   --with_loss_mask \
   --per_device_train_batch_size 1 \
   --max_seq_len 4096 \
   --learning_rate 3e-5 \
   --weight_decay 0.0001 \
   --num_train_epochs 1 \
   --gradient_accumulation_steps 4 \
   --lr_scheduler_type cosine \
   --precision fp16 \ # 训练精度，fp16或bf16
   --warmup_proportion 0.1 \ 
   --gradient_checkpointing \
   --offload \
   --seed 1233 \
   --zero_stage $ZERO_STAGE \ 
   --save_steps 10 \
   --deepspeed \ 
   --output_dir $OUTPUT # 输出路径 

>>> from transformers import AutoTokenizer
>>> from auto_gptq import BaseQuantizeConfig
>>> from modeling_telechat_gptq import TelechatGPTQForCausalLM
>>> tokenizer_path = '../models/7B'
>>> pretrained_model_dir = '../models/7B'
>>> quantized_model_dir = '../models/7B_8bit'
>>> tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True)
>>> calibration_text = ["auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm."]
>>> examples = [tokenizer(_) for _ in calibration_text]
>>> quantize_config = BaseQuantizeConfig( bits=8,  group_size=128,  desc_act=False )
>>> model = TelechatGPTQForCausalLM.from_pretrained(pretrained_model_dir, quantize_config,trust_remote_code=True)
>>> model.quantize(examples)
>>> model.save_quantized(quantized_model_dir)

>>> from transformers import AutoTokenizer, GenerationConfig
>>> from modeling_telechat_gptq import TelechatGPTQForCausalLM
>>> PATH = '../models/7B_8bit'
>>> tokenizer = AutoTokenizer.from_pretrained(PATH, trust_remote_code=True)
>>> model = TelechatGPTQForCausalLM.from_quantized(PATH, device="cuda:0", inject_fused_mlp=False, inject_fused_attention=False, trust_remote_code=True)
>>> generate_config = GenerationConfig.from_pretrained(PATH)
>>> model.eval()
>>> question = "生抽与老抽的区别？"
>>> answer, history = model.chat(tokenizer=tokenizer, question=question, history=[], generation_config=generate_config, stream=False)
>>> print("回答:", answer)
回答: 生抽和老抽是两种不同的酱油，它们的区别如下：

1. 原料不同：生抽是用大豆、面粉等为原料制成的；而老抽则是用豆豉、盐等为原料制成的。

2. 制作工艺不同：生抽是通过将大豆浸泡在水中，然后经过发酵、蒸煮等过程制成的；而老抽则是在生抽的基础上进行进一步的加工和处理，如加入盐、糖、味精等调料。

3. 口感和风味不同：生抽的口感相对较咸，适合用于烹调肉类、海鲜等；而老抽的风味相对较重，适合用于烹调红烧肉、酱爆鸡丁等菜品。

总的来说，生抽和老抽都是常见的酱油品种，它们在原料、制作工艺和口感等方面都有所不同。选择使用哪种酱油，可以根据个人口味和菜品需求来决定。

>>> from transformers import AutoTokenizer
>>> from auto_gptq import BaseQuantizeConfig
>>> from modeling_telechat_gptq import TelechatGPTQForCausalLM
>>> tokenizer_path = '../models/7B'
>>> pretrained_model_dir = '../models/7B'
>>> quantized_model_dir = '../models/7B_4bit'
>>> tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True)
>>> calibration_text = ["auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm."]
>>> examples = [tokenizer(_) for _ in calibration_text]
>>> quantize_config = BaseQuantizeConfig( bits=4, group_size=128, desc_act=False )
>>> model = TelechatGPTQForCausalLM.from_pretrained(pretrained_model_dir, quantize_config,trust_remote_code=True)
>>> model.quantize(examples)
>>> model.save_quantized(quantized_model_dir)

>>> from transformers import AutoTokenizer, GenerationConfig
>>> from modeling_telechat_gptq import TelechatGPTQForCausalLM
>>> PATH = '../models/7B_4bit'
>>> tokenizer = AutoTokenizer.from_pretrained(PATH, trust_remote_code=True)
>>> model = TelechatGPTQForCausalLM.from_quantized(PATH, device="cuda:0", inject_fused_mlp=False, inject_fused_attention=False, trust_remote_code=True)
>>> generate_config = GenerationConfig.from_pretrained(PATH)
>>> model.eval()
>>> question = "生抽与老抽的区别？"
>>> answer, history = model.chat(tokenizer=tokenizer, question=question, history=[], generation_config=generate_config, stream=False)
>>> print("回答:", answer)
回答: 生抽和老抽是两种不同的酱油，它们的区别主要体现在以下几个方面：

1. 原料不同：生抽是用大豆、小麦等制成的，而老抽则是用豆豉、盐等制成的。

2. 发酵方式不同：生抽是通过将大豆或小麦浸泡在水中，然后进行发酵制成的；而老抽则是在制作过程中直接将大豆或小麦炒熟后使用。

3. 味道不同：生抽的口感比较鲜美，有咸味和甜味；老抽的味道相对较重，有咸味和苦味。

4. 用途不同：生抽主要用于调味酱料、腌制肉类等；老抽则主要用于烹调菜肴、焖煮食材等。

@misc{wang2024telechat,
      title={TeleChat Technical Report}, 
      author={Zihan Wang and Xinzhang Liu and Shixuan Liu and Yitong Yao and Yuyao Huang and Zhongjiang He and Xuelong Li and Yongxiang Li and Zhonghao Che and Zhaoxi Zhang and Yan Wang and Xin Wang and Luwen Pu and Huihan Xu and Ruiyu Fang and Yu Zhao and Jie Zhang and Xiaomeng Huang and Zhilong Lu and Jiaxin Peng and Wenjun Zheng and Shiquan Wang and Bingkai Yang and Xuewei he and Zhuoru Jiang and Qiyi Xie and Yanhan Zhang and Zhongqiu Li and Lingling Shi and Weiwei Fu and Yin Zhang and Zilu Huang and Sishi Xiong and Yuxiang Zhang and Chao Wang and Shuangyong Song},
      year={2024},
      eprint={2401.03804},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}