added option to disable streaming output

app.py

@@ -76,7 +76,8 @@ with gr.Blocks(theme = gr.themes.Soft()) as demo:
       with gr.Accordion("Advanced Settings", open=False):
         with gr.Row(equal_height = True):
           llm_model = gr.Dropdown(choices = ['mistralai/Mixtral-8x7B-Instruct-v0.1','mistralai/Mistral-7B-Instruct-v0.2', 'None'], value = 'mistralai/Mistral-7B-Instruct-v0.2', label = 'LLM Model')
-          llm_results = gr.Slider(minimum=4, maximum=10, value=5, step=1, interactive=True, label="Top n results to sent as context")
+          llm_results = gr.Slider(minimum=4, maximum=10, value=5, step=1, interactive=True, label="Top n results as context")
+          stream_results = gr.Checkbox(value = True, label = "Stream output")
 
     output_text = gr.Textbox(show_label = True, container = True, label = 'LLM Answer', visible = True, placeholder = output_placeholder)
     input = gr.Textbox(show_label = False, visible = False)
@@ -100,31 +101,35 @@ with gr.Blocks(theme = gr.themes.Soft()) as demo:
         prompt = get_prompt_text(message, '\n\n'.join(rag_cleaner(out) for out in rag_out[:llm_results_use]))
         return md_text_updated, prompt
 
-    def ask_llm(prompt, llm_model_picked = 'mistralai/Mistral-7B-Instruct-v0.2'):
+    def ask_llm(prompt, llm_model_picked = 'mistralai/Mistral-7B-Instruct-v0.2', stream_outputs = False):
        model_disabled_text = "LLM Model is disabled"
        output = ""
        if llm_model_picked == 'None':
-          for out in model_disabled_text:
-            output += out
-            yield output
-          return output 
-           
+          if stream_outputs:
+              for out in model_disabled_text:
+                output += out
+                yield output
+              return output 
+          else:
+              return model_disabled_text
+              
        client = InferenceClient(llm_model_picked)
-       #output = client.text_generation(prompt, **generate_kwargs,  stream=False, details=False, return_full_text=False)
        try:
-           stream = client.text_generation(prompt, **generate_kwargs,  stream=True, details=True, return_full_text=False)
+           stream = client.text_generation(prompt, **generate_kwargs,  stream=stream_outputs, details=False, return_full_text=False)
        except:
            gr.Warning("LLM Inference rate limit reached, try again later!")
            return ""
        #output = output.lstrip(' \n') if output.lstrip().startswith('\n') else output
        
+       if stream_outputs:
+           for response in stream:
+              output += response
+              yield output
+           return output
+       else:
+           return stream
 
-       for response in stream:
-          output += response.token.text
-          yield output
-       return output
-       #return gr.Textbox(output, visible = True)
 
-    msg.submit(update_with_rag_md, [msg, llm_results], [gr_md, input]).success(ask_llm, [input, llm_model], output_text)
+    msg.submit(update_with_rag_md, [msg, llm_results], [gr_md, input]).success(ask_llm, [input, llm_model, stream_results], output_text)
 
-demo.launch(debug = True)
+demo.queue(default_concurrency_limit=10).launch()