Python＋OpenVINOでSafetensors形式のモデルを実行してみる

import os
import psutil
import time
from optimum.intel import OVModelForCausalLM
from transformers import AutoProcessor
process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()
model_path = r"H:\LLM_Models\safetensor\Qwen\Qwen2.5-Coder-7B-Instruct"
processor = AutoProcessor.from_pretrained(model_path)
model = OVModelForCausalLM.from_pretrained(model_path)
messages = [
    {"role": "system", "content": "あなたは有能なAIアシスタントです。"},
    {"role": "user", "content": "面白いジョークを話して。またどこが面白いかも解説してください。"},
]
inputs = processor.apply_chat_template(
    messages,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    add_generation_prompt=True,
    enable_thinking=False
).to(model.device)
input_len = inputs["input_ids"].shape[-1]
# Generate output
outputs = model.generate(**inputs, max_new_tokens=1024)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)
mem_after = process.memory_info().rss / 1024**2  # MB
end = time.time()
print(response)
print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"実行時間: {end - start:.3f}")
# OpenVINO IR形式を保存
# 次回以降model_pathに保存済みのOpenVINO IR形式へのpathを渡せば変換が省略される。
open_vino_save_path = r"H:\LLM_Models\open_vino\Qwen\Qwen2.5-Coder-7B-Instruct"
model.save_pretrained(open_vino_save_path)
processor.save_pretrained(open_vino_save_path)

process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()
processor = AutoProcessor.from_pretrained(open_vino_save_path)
model = OVModelForCausalLM.from_pretrained(open_vino_save_path)
messages = [
    {"role": "system", "content": "あなたは有能なAIアシスタントです。"},
    {"role": "user", "content": "面白いジョークを話して。またどこが面白いかも解説してください。"},
]
inputs = processor.apply_chat_template(
    messages,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    add_generation_prompt=True,
    enable_thinking=False
).to(model.device)
input_len = inputs["input_ids"].shape[-1]
outputs = model.generate(**inputs, max_new_tokens=1024)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)
mem_after = process.memory_info().rss / 1024**2  # MB
end = time.time()
print(response)
print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"実行時間: {end - start:.3f} 秒")

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()
quantization_config = BitsAndBytesConfig(load_in_8bit=True)
model = AutoModelForCausalLM.from_pretrained(
    model_path,
    quantization_config=quantization_config,
    torch_dtype="auto",
    device_map="auto",
)
tokenizer = AutoTokenizer.from_pretrained(model_path)
quantization_time = time.time()
messages = [
    {"role": "system", "content": "あなたは有能なAIアシスタントです。"},
    {"role": "user", "content": "面白いジョークを話して。またどこが面白いかも解説してください。"},
]
text = tokenizer.apply_chat_template(
    messages,
    temperature=0.1,
    tokenize=False,
    add_generation_prompt=True
)
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
generated_ids = model.generate(
    **model_inputs,
    max_new_tokens=512
)
generated_ids = [
    output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
]
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
mem_after = process.memory_info().rss / 1024**2  # MB
end = time.time()
print(response)
print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"差引メモリ使用量: {mem_after - mem_before:.2f} MB")
print(f"実行時間: {end - start:.3f} 秒")
print(f"量子化処理を除いた推論時間: {end - quantization_time:.3f} 秒")
del model, tokenizer

import os
import psutil
import time
from optimum.intel import OVModelForCausalLM
from transformers import AutoProcessor
model_path = r"H:\LLM_Models\safetensor\Qwen\Qwen2.5-Coder-7B-Instruct"
# モデル量子化用に追加でロード
from nncf import compress_weights, CompressWeightsMode
process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()
model = OVModelForCausalLM.from_pretrained(model_path,
                                           load_in_8bit=False, # これを指定することでデフォルトの8ビット量子化を無効化
                                           compile=False,   # この段階でモデルをコンパイルしても、その後4ビットモデルを作成⇒再コンパイルすることとなり無駄となる。
                                                            # Falseを指定することでコンパイルをキャンセルできる。
                                           )
# modeを指定することで量子化モードを指定できる。指定できるパラメータは以下のマークダウンを参照。
model.model = compress_weights(model.model,
                               mode=CompressWeightsMode.INT4_ASYM,
                              )
# プロセッサも保存する必要があるので読み込み
processor = AutoProcessor.from_pretrained(model_path)
# 量子化したモデルをOpenVINO IR形式で保存
open_vino_save_path_int4a = r"H:\LLM_Models\open_vino\Qwen\Qwen2.5-Coder-7B-Instruct_INT4-ASYM"
model.save_pretrained(open_vino_save_path_int4a)
processor.save_pretrained(open_vino_save_path_int4a)
end = time.time()
print(f"実行時間: {end - start:.3f} 秒")
del model, processor

process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()
processor = AutoProcessor.from_pretrained(open_vino_save_path_int4a)
model = OVModelForCausalLM.from_pretrained(open_vino_save_path_int4a)
messages = [
    {"role": "system", "content": "あなたは有能なAIアシスタントです。"},
    {"role": "user", "content": "面白いジョークを話して。またどこが面白いかも解説してください。"},
]
processor = AutoProcessor.from_pretrained(model_path)
inputs = processor.apply_chat_template(
    messages,
    temperature=0.1,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    add_generation_prompt=True,
    enable_thinking=False,
).to(model.device)
input_len = inputs["input_ids"].shape[-1]
outputs = model.generate(**inputs, max_new_tokens=1024)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)
mem_after = process.memory_info().rss / 1024**2  # MB
end = time.time()
print(response)
print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"差引メモリ使用量: {mem_after - mem_before:.2f} MB")
print(f"実行時間: {end - start:.3f} 秒")
del model, processor

# KVキャッシュの量子化を無効化してみる
process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()
processor = AutoProcessor.from_pretrained(open_vino_save_path_int4a)
model = OVModelForCausalLM.from_pretrained(open_vino_save_path_int4a,
                                           ov_config={"KV_CACHE_PRECISION": "f16",
                                                      "PERFORMANCE_HINT": "LATENCY",
                                                      },
                                           )
messages = [
    {"role": "system", "content": "あなたは有能なコーディングAIです。"},
    {"role": "user", "content": "transformersとFAISSとstreamlitを使った、QAチャットボットをpythonで構築して。"},
]
processor = AutoProcessor.from_pretrained(model_path)
inputs = processor.apply_chat_template(
    messages,
    temperature=0.1,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    add_generation_prompt=True,
    enable_thinking=False,
).to(model.device)
input_len = inputs["input_ids"].shape[-1]
outputs = model.generate(**inputs, max_new_tokens=4096)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)
mem_after = process.memory_info().rss / 1024**2  # MB
end = time.time()
print(response)
print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"差引メモリ使用量: {mem_after - mem_before:.2f} MB")
print(f"実行時間: {end - start:.3f} 秒")
del model, processor

# KVキャッシュの量子化をu4にしてみる
process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()
processor = AutoProcessor.from_pretrained(open_vino_save_path_int4a)
model = OVModelForCausalLM.from_pretrained(open_vino_save_path_int4a,
                                           ov_config={"KV_CACHE_PRECISION": "u4",
                                                      "DYNAMIC_QUANTIZATION_GROUP_SIZE": "32",
                                                      "PERFORMANCE_HINT": "LATENCY",
                                                      },
                                           )
messages = [
    {"role": "system", "content": "あなたは有能なコーディングAIです。"},
    {"role": "user", "content": "transformersとFAISSとstreamlitを使った、QAチャットボットをpythonで構築して。"},
]
processor = AutoProcessor.from_pretrained(model_path)
inputs = processor.apply_chat_template(
    messages,
    temperature=0.1,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    add_generation_prompt=True,
    enable_thinking=False,
).to(model.device)
input_len = inputs["input_ids"].shape[-1]
outputs = model.generate(**inputs, max_new_tokens=4096)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)
mem_after = process.memory_info().rss / 1024**2  # MB
end = time.time()
print(response)
print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"差引メモリ使用量: {mem_after - mem_before:.2f} MB")
print(f"実行時間: {end - start:.3f} 秒")
del model, processor

# NPUが認識されるかチェック
# ['CPU', 'GPU', 'NPU']のようにNPUが表示されればOK
import openvino as ov
core = ov.Core()
print("利用可能なデバイス:", core.available_devices)
for device in core.available_devices:
    print(device, core.get_property(device, "FULL_DEVICE_NAME"))
# 必要なライブラリのインポート
import openvino
import openvino_genai
import transformers
import nncf
print("OpenVINO   :", openvino.__version__)
print("Transformers:", transformers.__version__)
print("NNCF       :", nncf.__version__)

optimum-cli export openvino -m "H:\LLM_Models\safetensor\Qwen\Qwen2.5-Coder-7B-Instruct" --task text-generation-with-past --weight-format int4 --sym --ratio 1.0 --group-size -1 "H:\LLM_Models\open_vino\Qwen\Qwen2.5-Coder-7B-Instruct-int4-npu"

import os
import time
import psutil
import openvino_genai as ov_genai
from transformers import AutoTokenizer
model_path = (
    r"H:\LLM_Models\open_vino\Qwen"
    r"\Qwen2.5-Coder-7B-Instruct-int4-npu"
)
process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2
start = time.time()
tokenizer = AutoTokenizer.from_pretrained(
    model_path,
    trust_remote_code=True,
)
messages = [
    {
        "role": "system",
        "content": "あなたは有能なAIアシスタントです。",
    },
    {
        "role": "user",
        "content": (
            "Pythonで高速な素数判定関数を書いて、"
            "処理内容も説明してください。"
        ),
    },
]
prompt = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
)
pipeline_config = {
    "MAX_PROMPT_LEN": 1024,
    "MIN_RESPONSE_LEN": 128,
    "GENERATE_HINT": "BEST_PERF",
    "CACHE_DIR": os.path.join(model_path, "npu_cache"),
}
pipe = ov_genai.LLMPipeline(
    model_path,
    "NPU",
    pipeline_config,
)
result = pipe.generate(
    prompt,
    max_new_tokens=512,
    do_sample=False,
)
end = time.time()
mem_after = process.memory_info().rss / 1024**2
print(result)
print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"メモリ増加量      : {mem_after - mem_before:+.2f} MB")
print(f"実行時間          : {end - start:.3f} 秒")