| import os | |
| import gradio as gr | |
| from huggingface_hub import InferenceClient | |
| import requests | |
| url = "https://huggingface.co/models-json?num_parameters=min%3A64B&inference_provider=novita&sort=modified&withCount=true" | |
| res = requests.get(url).json() | |
| model_ids = [m['id'] for m in res.get('models', [])] | |
| client = InferenceClient(provider="novita", api_key=os.environ["HF_TOKEN"]) | |
| def respond(message, history, model): | |
| history.append((message, "")) | |
| yield "", history | |
| full_reply = "" | |
| for chunk in client.chat.completions.create( | |
| model=model, | |
| messages=[{"role": "user", "content": message}], | |
| stream=True | |
| ): | |
| if chunk.choices and len(chunk.choices) > 0: | |
| delta = chunk.choices[0].delta | |
| if hasattr(delta, 'content') and delta.content: | |
| full_reply += delta.content | |
| history[-1] = (message, full_reply) | |
| yield "", history | |
| with gr.Blocks() as demo: | |
| gr.Markdown("## AI") | |
| chatbot = gr.Chatbot(height=400) | |
| msg = gr.Textbox(label="Ask me smth") | |
| model_dd = gr.Dropdown(model_ids, label="Model", value=model_ids[0] if model_ids else "") | |
| with gr.Row(): | |
| submit_btn = gr.Button("Send") | |
| clear = gr.Button("Clear") | |
| msg.submit(respond, [msg, chatbot, model_dd], [msg, chatbot]) | |
| submit_btn.click(respond, [msg, chatbot, model_dd], [msg, chatbot]) | |
| clear.click(lambda: None, None, chatbot, queue=False) | |
| demo.queue().launch() |