added usage example

This commit is contained in:
Falko Victor Habel 2025-02-11 19:55:32 +01:00
parent ccdb645bb8
commit 9327ac80df
1 changed files with 15 additions and 0 deletions

15
example.py Normal file
View File

@ -0,0 +1,15 @@
from transformers import AutoModelForCausalLM, AutoTokenizer
location = "Godala-moe"
device = "cuda" # cpu when not using gpu
tokenizer = AutoTokenizer.from_pretrained(location)
model = AutoModelForCausalLM.from_pretrained(location).to(device)
messages = [{"role": "user", "content": "What can you tell me about Godot?"}]
input_text=tokenizer.apply_chat_template(messages, tokenize=False)
inputs = tokenizer.encode(input_text, return_tensors="pt").to(device)
outputs = model.generate(inputs, max_new_tokens=510, temperature=0.2, top_p=0.9, do_sample=True)
print(tokenizer.decode(outputs[0]))