Python＋OpenVINOでLLMを実行してみた

import os
import psutil
import time


from optimum.intel import OVModelForCausalLM
from transformers import AutoProcessor


process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()

model_path = r"H:\LLM_Models\safetensor\Qwen\Qwen2.5-Coder-7B-Instruct"
processor = AutoProcessor.from_pretrained(model_path)

model = OVModelForCausalLM.from_pretrained(model_path)

messages = [
    {"role": "system", "content": "あなたは有能なAIアシスタントです。"},
    {"role": "user", "content": "面白いジョークを話して。またどこが面白いかも解説してください。"},
]

inputs = processor.apply_chat_template(
    messages,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    add_generation_prompt=True,
    enable_thinking=False
).to(model.device)
input_len = inputs["input_ids"].shape[-1]

# Generate output
outputs = model.generate(**inputs, max_new_tokens=1024)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)

mem_after = process.memory_info().rss / 1024**2  # MB
end = time.time()

print(response)

print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"実行時間: {end - start:.3f}")

# OpenVINO IR形式を保存
# 次回以降model_pathに保存済みのOpenVINO IR形式へのpathを渡せば変換が省略される。
open_vino_save_path = r"H:\LLM_Models\open_vino\Qwen\Qwen2.5-Coder-7B-Instruct"
model.save_pretrained(open_vino_save_path)
processor.save_pretrained(open_vino_save_path)

process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2  # MB
start = time.time()

processor = AutoProcessor.from_pretrained(open_vino_save_path)
model = OVModelForCausalLM.from_pretrained(open_vino_save_path)

messages = [
    {"role": "system", "content": "あなたは有能なAIアシスタントです。"},
    {"role": "user", "content": "面白いジョークを話して。またどこが面白いかも解説してください。"},
]

inputs = processor.apply_chat_template(
    messages,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    add_generation_prompt=True,
    enable_thinking=False
).to(model.device)
input_len = inputs["input_ids"].shape[-1]


outputs = model.generate(**inputs, max_new_tokens=1024)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)

mem_after = process.memory_info().rss / 1024**2  # MB

end = time.time()

print(response)
print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"実行時間: {end - start:.3f} 秒")

# NPUが認識されるかチェック
# ['CPU', 'GPU', 'NPU']のようにNPUが表示されればOK
import openvino as ov

core = ov.Core()

print("利用可能なデバイス:", core.available_devices)

for device in core.available_devices:
    print(device, core.get_property(device, "FULL_DEVICE_NAME"))

# 必要なライブラリのインポート
import openvino
import openvino_genai
import transformers
import nncf

print("OpenVINO   :", openvino.__version__)
print("Transformers:", transformers.__version__)
print("NNCF       :", nncf.__version__)

optimum-cli export openvino -m "H:\LLM_Models\safetensor\Qwen\Qwen2.5-Coder-7B-Instruct" --task text-generation-with-past --weight-format int4 --sym --ratio 1.0 --group-size -1 "H:\LLM_Models\open_vino\Qwen\Qwen2.5-Coder-7B-Instruct-int4-npu"

import os
import time
import psutil
import openvino_genai as ov_genai
from transformers import AutoTokenizer

model_path = (
    r"H:\LLM_Models\open_vino\Qwen"
    r"\Qwen2.5-Coder-7B-Instruct-int4-npu"
)

process = psutil.Process(os.getpid())
mem_before = process.memory_info().rss / 1024**2
start = time.time()

tokenizer = AutoTokenizer.from_pretrained(
    model_path,
    trust_remote_code=True,
)

messages = [
    {
        "role": "system",
        "content": "あなたは有能なAIアシスタントです。",
    },
    {
        "role": "user",
        "content": (
            "Pythonで高速な素数判定関数を書いて、"
            "処理内容も説明してください。"
        ),
    },
]

prompt = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
)

pipeline_config = {
    "MAX_PROMPT_LEN": 1024,
    "MIN_RESPONSE_LEN": 128,
    "GENERATE_HINT": "BEST_PERF",
    "CACHE_DIR": os.path.join(model_path, "npu_cache"),
}

pipe = ov_genai.LLMPipeline(
    model_path,
    "NPU",
    pipeline_config,
)

result = pipe.generate(
    prompt,
    max_new_tokens=512,
    do_sample=False,
)

end = time.time()
mem_after = process.memory_info().rss / 1024**2

print(result)

print("=" * 60)
print(f"実行前メモリ使用量: {mem_before:.2f} MB")
print(f"実行後メモリ使用量: {mem_after:.2f} MB")
print(f"メモリ増加量      : {mem_after - mem_before:+.2f} MB")
print(f"実行時間          : {end - start:.3f} 秒")