from transformers import AutoModelForCausalLM, AutoTokenizer


location = "Godala-moe"

device = "gpu" # or cpu
tokenizer = AutoTokenizer.from_pretrained(location)

model = AutoModelForCausalLM.from_pretrained(location).to(device)

messages = [{"role": "user", "content": "What can you tell me about Godot?"}]
input_text=tokenizer.apply_chat_template(messages, tokenize=False)
inputs = tokenizer.encode(input_text, return_tensors="pt").to(device)
outputs = model.generate(inputs, max_new_tokens=510, temperature=0.2, top_p=0.9, do_sample=True)
print(tokenizer.decode(outputs[0]))