update

utils.py

@@ -7,6 +7,9 @@ import io
 from transformers import DynamicCache
 import os
 import spaces
+import httpx
+import tqdm
+
 
 os.makedirs("tmp", exist_ok=True)
 
@@ -54,7 +57,7 @@ def generate_answer(
         )
         new_id = outputs.logits[0, -1].argmax()
         generated_ids.append(new_id)
-        if new_id.item() in model.generation_config.eos_token_id:
+        if new_id.item() == model.generation_config.eos_token_id:
             break
 
     answer = tokenizer.decode(torch.stack(generated_ids), skip_special_tokens=True)
@@ -84,6 +87,8 @@ def get_condense_kv_cache(context: str):
     response = requests.post(url, json=payload, headers=headers).json()
     print(response)
     numpy_kv_cache, error = load_npy_from_url(response["compressed_kv_url"])
+    if error:
+        print(error)
     kv_cache = DynamicCache.from_legacy_cache(
         torch.from_numpy(numpy_kv_cache).to("cuda").to(torch.bfloat16)
     )