example.py

@@ -0,0 +1,15 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+
+location = "Godala-moe"
+
+device = "cuda" # cpu when not using gpu
+tokenizer = AutoTokenizer.from_pretrained(location)
+
+model = AutoModelForCausalLM.from_pretrained(location).to(device)
+
+messages = [{"role": "user", "content": "What can you tell me about Godot?"}]
+input_text=tokenizer.apply_chat_template(messages, tokenize=False)
+inputs = tokenizer.encode(input_text, return_tensors="pt").to(device)
+outputs = model.generate(inputs, max_new_tokens=510, temperature=0.2, top_p=0.9, do_sample=True)
+print(tokenizer.decode(outputs[0]))