Update app.py

app.py

@@ -1,13 +1,36 @@
-import gradio as gr
+import os
+from huggingface_hub import InferenceClient, login
+from transformers import AutoTokenizer
+from langchain.chat_models import ChatOpenAI
 
+# access token with permission to access the model and PRO subscription
+hf_token = "YOUR_HF_TOKEN" # <https://huggingface.co/settings/tokens>
+login(token=hf_token)
 
-#demo = gr.load("mistralai/Mixtral-8x7B-Instruct-v0.1", src="models")
+# tokenizer for generating prompt
+tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-70b-chat-hf")
 
-description = "Test"
-title = "Test"
-interface = gr.Interface.load("mistralai/Mixtral-8x7B-Instruct-v0.1", 
-            description=description,
-            title = title,
-            examples=[["american robin"]]
-)
-interface.launch(share=True)
+# inference client
+client = InferenceClient("https://api-inference.huggingface.co/models/meta-llama/Llama-2-70b-chat-hf")
+
+# generate function
+def generate(text):
+    payload = tokenizer.apply_chat_template([{"role":"user","content":text}],tokenize=False)
+    res = client.text_generation(
+                    payload,
+                    do_sample=True,
+                    return_full_text=False,
+                    max_new_tokens=2048,
+                    top_p=0.9,
+                    temperature=0.6,
+                )
+    return res.strip()
+
+# test client
+assert generate("What is 2+2?") == "The answer to 2+2 is 4."
+
+# create evaluator
+os.environ["OPENAI_API_KEY"] = "YOUR_OPENAI_API_KEY" # https://platform.openai.com/account/api-keys
+assert os.environ.get("OPENAI_API_KEY") is not None, "Please set OPENAI_API_KEY environment variable"
+
+evaluation_llm = ChatOpenAI(model="gpt-4")