kengboon commited on
Commit
c011c20
·
1 Parent(s): 81917a3

First commit

Browse files
Files changed (11) hide show
  1. .gitignore +163 -0
  2. README.md +7 -2
  3. agent.py +17 -0
  4. api.py +77 -0
  5. app.py +83 -185
  6. app_ori.py +196 -0
  7. images/huggingface_logo-noborder.svg +11 -0
  8. main.py +13 -0
  9. requirements.txt +4 -1
  10. secret.py +10 -0
  11. tools.py +0 -0
.gitignore ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+ *.c
9
+
10
+ # Distribution / packaging
11
+ .Python
12
+ build/
13
+ develop-eggs/
14
+ dist/
15
+ downloads/
16
+ eggs/
17
+ .eggs/
18
+ lib/
19
+ lib64/
20
+ parts/
21
+ sdist/
22
+ var/
23
+ wheels/
24
+ share/python-wheels/
25
+ *.egg-info/
26
+ .installed.cfg
27
+ *.egg
28
+ MANIFEST
29
+
30
+ # PyInstaller
31
+ # Usually these files are written by a python script from a template
32
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
33
+ *.manifest
34
+ *.spec
35
+
36
+ # Installer logs
37
+ pip-log.txt
38
+ pip-delete-this-directory.txt
39
+
40
+ # Unit test / coverage reports
41
+ htmlcov/
42
+ .tox/
43
+ .nox/
44
+ .coverage
45
+ .coverage.*
46
+ .cache
47
+ nosetests.xml
48
+ coverage.xml
49
+ *.cover
50
+ *.py,cover
51
+ .hypothesis/
52
+ .pytest_cache/
53
+ cover/
54
+
55
+ # Translations
56
+ *.mo
57
+ *.pot
58
+
59
+ # Django stuff:
60
+ *.log
61
+ local_settings.py
62
+ db.sqlite3
63
+ db.sqlite3-journal
64
+
65
+ # Flask stuff:
66
+ instance/
67
+ .webassets-cache
68
+
69
+ # Scrapy stuff:
70
+ .scrapy
71
+
72
+ # Sphinx documentation
73
+ docs/_build/
74
+
75
+ # PyBuilder
76
+ .pybuilder/
77
+ target/
78
+
79
+ # Jupyter Notebook
80
+ .ipynb_checkpoints
81
+
82
+ # IPython
83
+ profile_default/
84
+ ipython_config.py
85
+
86
+ # pyenv
87
+ # For a library or package, you might want to ignore these files since the code is
88
+ # intended to run in multiple environments; otherwise, check them in:
89
+ # .python-version
90
+
91
+ # pipenv
92
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
93
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
94
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
95
+ # install all needed dependencies.
96
+ #Pipfile.lock
97
+
98
+ # poetry
99
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
100
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
101
+ # commonly ignored for libraries.
102
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
103
+ #poetry.lock
104
+
105
+ # pdm
106
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
107
+ #pdm.lock
108
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
109
+ # in version control.
110
+ # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
111
+ .pdm.toml
112
+ .pdm-python
113
+ .pdm-build/
114
+
115
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
116
+ __pypackages__/
117
+
118
+ # Celery stuff
119
+ celerybeat-schedule
120
+ celerybeat.pid
121
+
122
+ # SageMath parsed files
123
+ *.sage.py
124
+
125
+ # Environments
126
+ .env
127
+ .venv
128
+ env/
129
+ venv/
130
+ ENV/
131
+ env.bak/
132
+ venv.bak/
133
+
134
+ # Spyder project settings
135
+ .spyderproject
136
+ .spyproject
137
+
138
+ # Rope project settings
139
+ .ropeproject
140
+
141
+ # mkdocs documentation
142
+ /site
143
+
144
+ # mypy
145
+ .mypy_cache/
146
+ .dmypy.json
147
+ dmypy.json
148
+
149
+ # Pyre type checker
150
+ .pyre/
151
+
152
+ # pytype static type analyzer
153
+ .pytype/
154
+
155
+ # Cython debug symbols
156
+ cython_debug/
157
+
158
+ # PyCharm
159
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
160
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
161
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
162
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
163
+ #.idea/
README.md CHANGED
@@ -1,15 +1,20 @@
1
  ---
2
- title: Template Final Assignment
3
  emoji: 🕵🏻‍♂️
4
- colorFrom: indigo
5
  colorTo: indigo
6
  sdk: gradio
7
  sdk_version: 5.25.2
8
  app_file: app.py
 
9
  pinned: false
10
  hf_oauth: true
11
  # optional, default duration is 8 hours/480 minutes. Max duration is 30 days/43200 minutes.
12
  hf_oauth_expiration_minutes: 480
 
 
 
 
13
  ---
14
 
15
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: Agent Evaluator - GAIA Benchmark
3
  emoji: 🕵🏻‍♂️
4
+ colorFrom: purple
5
  colorTo: indigo
6
  sdk: gradio
7
  sdk_version: 5.25.2
8
  app_file: app.py
9
+ short_description: Evaluate AI Agent on subset of GAIA benchmark
10
  pinned: false
11
  hf_oauth: true
12
  # optional, default duration is 8 hours/480 minutes. Max duration is 30 days/43200 minutes.
13
  hf_oauth_expiration_minutes: 480
14
+ tags:
15
+ - agent
16
+ - langgraph
17
+ - agent-course
18
  ---
19
 
20
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
agent.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain.schema import HumanMessage
2
+ from langchain_google_genai import ChatGoogleGenerativeAI
3
+
4
+ class Agent:
5
+ def __init__(self, model_name: str="gemini-2.0-flash", tools: list | None=None):
6
+ llm = ChatGoogleGenerativeAI(model=model_name)
7
+ if tools:
8
+ self.llm = llm.bind_tools(tools)
9
+ else:
10
+ self.llm = llm
11
+
12
+ def run(self, prompt: str):
13
+ message = [
14
+ HumanMessage(prompt)
15
+ ]
16
+ response = self.llm.invoke(message)
17
+ return response.content.strip()
api.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Communicate with HF Agent Course API to evaluate the AI agent.
3
+ """
4
+ import requests
5
+
6
+ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
7
+
8
+ def get_questions(api_url: str | None=None) -> list | str:
9
+ if not api_url:
10
+ api_url = DEFAULT_API_URL
11
+ questions_url = f"{api_url}/questions"
12
+
13
+ print("Fetching questions from:", questions_url)
14
+ try:
15
+ response = requests.get(questions_url, timeout=15)
16
+ response.raise_for_status()
17
+ questions_data = response.json()
18
+ if not questions_data:
19
+ print("Fetched questions list is empty.")
20
+ return "Fetched questions list is empty or invalid format.", None
21
+ print(f"Fetched {len(questions_data)} questions.")
22
+ return questions_data
23
+ except requests.exceptions.RequestException as e:
24
+ print(f"Error fetching questions: {e}")
25
+ return f"Error fetching questions: {e}", None
26
+ except requests.exceptions.JSONDecodeError as e:
27
+ print(f"Error decoding JSON response from questions endpoint: {e}")
28
+ print(f"Response text: {response.text[:500]}")
29
+ return f"Error decoding server response for questions: {e}", None
30
+ except Exception as e:
31
+ print(f"An unexpected error occurred fetching questions: {e}")
32
+ return f"An unexpected error occurred fetching questions: {e}", None
33
+
34
+ def submit(submission_data: list, api_url: str | None=None) -> str:
35
+ if not api_url:
36
+ api_url = DEFAULT_API_URL
37
+ submit_url = f"{api_url}/submit"
38
+
39
+ print(f"Submitting {len(submission_data["answers"])} answers to: {submit_url}")
40
+ try:
41
+ response = requests.post(submit_url, json=submission_data, timeout=60)
42
+ response.raise_for_status()
43
+ result_data = response.json()
44
+ final_status = (
45
+ f"Submission Successful!\n"
46
+ f"User: {result_data.get('username')}\n"
47
+ f"Overall Score: {result_data.get('score', 'N/A')}% "
48
+ f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
49
+ f"Message: {result_data.get('message', 'No message received.')}"
50
+ )
51
+ print("Submission successful.")
52
+ return final_status
53
+ except requests.exceptions.HTTPError as e:
54
+ error_detail = f"Server responded with status {e.response.status_code}."
55
+ try:
56
+ error_json = e.response.json()
57
+ error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
58
+ except requests.exceptions.JSONDecodeError:
59
+ error_detail += f" Response: {e.response.text[:500]}"
60
+ status_message = f"Submission Failed: {error_detail}"
61
+ print(status_message)
62
+ return status_message
63
+ except requests.exceptions.Timeout:
64
+ status_message = "Submission Failed: The request timed out."
65
+ print(status_message)
66
+ return status_message
67
+ except requests.exceptions.RequestException as e:
68
+ status_message = f"Submission Failed: Network error - {e}"
69
+ print(status_message)
70
+ return status_message
71
+ except Exception as e:
72
+ status_message = f"An unexpected error occurred during submission: {e}"
73
+ print(status_message)
74
+ return status_message
75
+
76
+ if __name__ == "__main__":
77
+ pass
app.py CHANGED
@@ -1,196 +1,94 @@
1
  import os
2
- import gradio as gr
3
- import requests
4
- import inspect
5
- import pandas as pd
6
-
7
- # (Keep Constants as is)
8
- # --- Constants ---
9
- DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
10
-
11
- # --- Basic Agent Definition ---
12
- # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
13
- class BasicAgent:
14
- def __init__(self):
15
- print("BasicAgent initialized.")
16
- def __call__(self, question: str) -> str:
17
- print(f"Agent received question (first 50 chars): {question[:50]}...")
18
- fixed_answer = "This is a default answer."
19
- print(f"Agent returning fixed answer: {fixed_answer}")
20
- return fixed_answer
21
-
22
- def run_and_submit_all( profile: gr.OAuthProfile | None):
23
- """
24
- Fetches all questions, runs the BasicAgent on them, submits all answers,
25
- and displays the results.
26
- """
27
- # --- Determine HF Space Runtime URL and Repo URL ---
28
- space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
29
 
30
- if profile:
31
- username= f"{profile.username}"
32
- print(f"User logged in: {username}")
33
- else:
34
- print("User not logged in.")
35
- return "Please Login to Hugging Face with the button.", None
36
 
37
- api_url = DEFAULT_API_URL
38
- questions_url = f"{api_url}/questions"
39
- submit_url = f"{api_url}/submit"
40
 
41
- # 1. Instantiate Agent ( modify this part to create your agent)
42
- try:
43
- agent = BasicAgent()
44
- except Exception as e:
45
- print(f"Error instantiating agent: {e}")
46
- return f"Error initializing agent: {e}", None
47
- # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
48
- agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
49
- print(agent_code)
50
 
51
- # 2. Fetch Questions
52
- print(f"Fetching questions from: {questions_url}")
53
  try:
54
- response = requests.get(questions_url, timeout=15)
55
- response.raise_for_status()
56
- questions_data = response.json()
57
- if not questions_data:
58
- print("Fetched questions list is empty.")
59
- return "Fetched questions list is empty or invalid format.", None
60
- print(f"Fetched {len(questions_data)} questions.")
61
- except requests.exceptions.RequestException as e:
62
- print(f"Error fetching questions: {e}")
63
- return f"Error fetching questions: {e}", None
64
- except requests.exceptions.JSONDecodeError as e:
65
- print(f"Error decoding JSON response from questions endpoint: {e}")
66
- print(f"Response text: {response.text[:500]}")
67
- return f"Error decoding server response for questions: {e}", None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  except Exception as e:
69
- print(f"An unexpected error occurred fetching questions: {e}")
70
- return f"An unexpected error occurred fetching questions: {e}", None
71
-
72
- # 3. Run your Agent
73
- results_log = []
74
- answers_payload = []
75
- print(f"Running agent on {len(questions_data)} questions...")
76
- for item in questions_data:
77
- task_id = item.get("task_id")
78
- question_text = item.get("question")
79
- if not task_id or question_text is None:
80
- print(f"Skipping item with missing task_id or question: {item}")
81
- continue
82
- try:
83
- submitted_answer = agent(question_text)
84
- answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
85
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
86
- except Exception as e:
87
- print(f"Error running agent on task {task_id}: {e}")
88
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
89
-
90
- if not answers_payload:
91
- print("Agent did not produce any answers to submit.")
92
- return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
93
-
94
- # 4. Prepare Submission
95
- submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
96
- status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
97
- print(status_update)
98
-
99
- # 5. Submit
100
- print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
101
- try:
102
- response = requests.post(submit_url, json=submission_data, timeout=60)
103
- response.raise_for_status()
104
- result_data = response.json()
105
- final_status = (
106
- f"Submission Successful!\n"
107
- f"User: {result_data.get('username')}\n"
108
- f"Overall Score: {result_data.get('score', 'N/A')}% "
109
- f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
110
- f"Message: {result_data.get('message', 'No message received.')}"
111
  )
112
- print("Submission successful.")
113
- results_df = pd.DataFrame(results_log)
114
- return final_status, results_df
115
- except requests.exceptions.HTTPError as e:
116
- error_detail = f"Server responded with status {e.response.status_code}."
117
- try:
118
- error_json = e.response.json()
119
- error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
120
- except requests.exceptions.JSONDecodeError:
121
- error_detail += f" Response: {e.response.text[:500]}"
122
- status_message = f"Submission Failed: {error_detail}"
123
- print(status_message)
124
- results_df = pd.DataFrame(results_log)
125
- return status_message, results_df
126
- except requests.exceptions.Timeout:
127
- status_message = "Submission Failed: The request timed out."
128
- print(status_message)
129
- results_df = pd.DataFrame(results_log)
130
- return status_message, results_df
131
- except requests.exceptions.RequestException as e:
132
- status_message = f"Submission Failed: Network error - {e}"
133
- print(status_message)
134
- results_df = pd.DataFrame(results_log)
135
- return status_message, results_df
136
- except Exception as e:
137
- status_message = f"An unexpected error occurred during submission: {e}"
138
- print(status_message)
139
- results_df = pd.DataFrame(results_log)
140
- return status_message, results_df
141
-
142
-
143
- # --- Build Gradio Interface using Blocks ---
144
- with gr.Blocks() as demo:
145
- gr.Markdown("# Basic Agent Evaluation Runner")
146
- gr.Markdown(
147
- """
148
- **Instructions:**
149
-
150
- 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
151
- 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
152
- 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
153
-
154
- ---
155
- **Disclaimers:**
156
- Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
157
- This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
158
- """
159
- )
160
-
161
- gr.LoginButton()
162
-
163
- run_button = gr.Button("Run Evaluation & Submit All Answers")
164
-
165
- status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
166
- # Removed max_rows=10 from DataFrame constructor
167
- results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
168
 
169
- run_button.click(
170
- fn=run_and_submit_all,
171
- outputs=[status_output, results_table]
172
- )
 
 
173
 
174
  if __name__ == "__main__":
175
- print("\n" + "-"*30 + " App Starting " + "-"*30)
176
- # Check for SPACE_HOST and SPACE_ID at startup for information
177
- space_host_startup = os.getenv("SPACE_HOST")
178
- space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
179
-
180
- if space_host_startup:
181
- print(f"✅ SPACE_HOST found: {space_host_startup}")
182
- print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
183
- else:
184
- print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
185
-
186
- if space_id_startup: # Print repo URLs if SPACE_ID is found
187
- print(f"✅ SPACE_ID found: {space_id_startup}")
188
- print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
189
- print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
190
- else:
191
- print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
192
-
193
- print("-"*(60 + len(" App Starting ")) + "\n")
194
-
195
- print("Launching Gradio Interface for Basic Agent Evaluation...")
196
- demo.launch(debug=True, share=False)
 
1
  import os
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
 
3
+ import gradio as gr
 
 
 
 
 
4
 
5
+ from api import get_questions, submit
6
+ from secret import check_secret, USERNAME
 
7
 
8
+ def run_app(submission_password: str):
9
+ chat_history = []
 
 
 
 
 
 
 
10
 
 
 
11
  try:
12
+ # Get the list of questions from the API
13
+ questions_data = get_questions()
14
+ if type(questions_data) == str:
15
+ err_msg = f"*{questions_data}*"
16
+ chat_history.append(gr.ChatMessage(err_msg, role="assistant"))
17
+ yield chat_history
18
+ return
19
+
20
+ # Initialize the agent
21
+ from agent import Agent
22
+ agent = Agent(model_name="gemini-2.5-flash-preview-04-17")
23
+
24
+ # Loop through the questions and simulate the chat
25
+ answers_payload = []
26
+ for i, item in enumerate(questions_data):
27
+ task_id = item.get("task_id")
28
+ question_text = item.get("question")
29
+
30
+ chat_history.append(gr.ChatMessage(f"**Question {i+1}:** {question_text}", role="user"))
31
+ yield chat_history
32
+
33
+ response = agent.run(question_text)
34
+ chat_history.append(gr.ChatMessage(response, role="assistant"))
35
+ yield chat_history
36
+
37
+ # Append to the answers payload
38
+ answers_payload.append({
39
+ "task_id": task_id,
40
+ "submitted_answer": response
41
+ })
42
+
43
+ # Check required to submit answers
44
+ if answers_payload and submission_password:
45
+ if check_secret(submission_password):
46
+ username = USERNAME
47
+ agent_code = f"https://huggingface.co/spaces/{os.getenv('SPACE_ID')}/tree/main"
48
+ answers_payload = {
49
+ "username": username,
50
+ "answers": answers_payload,
51
+ "agent_code": agent_code
52
+ }
53
+ submit(answers_payload)
54
  except Exception as e:
55
+ err_msg = f"An error occurred: {str(e)}"
56
+ chat_history.append(gr.ChatMessage(err_msg, role="assistant"))
57
+ yield chat_history
58
+ return
59
+
60
+ def get_ui():
61
+ with gr.Blocks() as ui_block:
62
+ gr.Markdown("# AI Agent Evaluation Runner 🕵🏻‍♂️")
63
+ gr.Markdown(
64
+ """
65
+ Evaluate an AI agent on a subset of validation questions from the **General AI Assistants (GAIA) Benchmark**.
66
+
67
+ **Note**: This space run on minimal setup and takes time to answer the questions.
68
+
69
+ [API Information](https://huggingface.co/learn/agents-course/unit4/hands-on)
70
+ """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  )
72
+ chatbot = gr.Chatbot(
73
+ label="AI Agent",
74
+ type="messages",
75
+ avatar_images=(
76
+ None,
77
+ "images/huggingface_logo-noborder.svg",
78
+ ),
79
+ resizable=True,
80
+ scale=1
81
+ )
82
+ run_button = gr.Button("Run Evaluation")
83
+ subm_pw = gr.Textbox("", type="password", placeholder="Submission Password", label="Submission Password")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
 
85
+ run_button.click(
86
+ fn=run_app,
87
+ inputs=[subm_pw],
88
+ outputs=[chatbot]
89
+ )
90
+ return ui_block
91
 
92
  if __name__ == "__main__":
93
+ from main import main as main_fn
94
+ main_fn()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
app_ori.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gradio as gr
3
+ import requests
4
+ import inspect
5
+ import pandas as pd
6
+
7
+ # (Keep Constants as is)
8
+ # --- Constants ---
9
+ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
10
+
11
+ # --- Basic Agent Definition ---
12
+ # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
13
+ class BasicAgent:
14
+ def __init__(self):
15
+ print("BasicAgent initialized.")
16
+ def __call__(self, question: str) -> str:
17
+ print(f"Agent received question (first 50 chars): {question[:50]}...")
18
+ fixed_answer = "This is a default answer."
19
+ print(f"Agent returning fixed answer: {fixed_answer}")
20
+ return fixed_answer
21
+
22
+ def run_and_submit_all( profile: gr.OAuthProfile | None):
23
+ """
24
+ Fetches all questions, runs the BasicAgent on them, submits all answers,
25
+ and displays the results.
26
+ """
27
+ # --- Determine HF Space Runtime URL and Repo URL ---
28
+ space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
29
+
30
+ if profile:
31
+ username= f"{profile.username}"
32
+ print(f"User logged in: {username}")
33
+ else:
34
+ print("User not logged in.")
35
+ return "Please Login to Hugging Face with the button.", None
36
+
37
+ api_url = DEFAULT_API_URL
38
+ questions_url = f"{api_url}/questions"
39
+ submit_url = f"{api_url}/submit"
40
+
41
+ # 1. Instantiate Agent ( modify this part to create your agent)
42
+ try:
43
+ agent = BasicAgent()
44
+ except Exception as e:
45
+ print(f"Error instantiating agent: {e}")
46
+ return f"Error initializing agent: {e}", None
47
+ # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
48
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
49
+ print(agent_code)
50
+
51
+ # 2. Fetch Questions
52
+ print(f"Fetching questions from: {questions_url}")
53
+ try:
54
+ response = requests.get(questions_url, timeout=15)
55
+ response.raise_for_status()
56
+ questions_data = response.json()
57
+ if not questions_data:
58
+ print("Fetched questions list is empty.")
59
+ return "Fetched questions list is empty or invalid format.", None
60
+ print(f"Fetched {len(questions_data)} questions.")
61
+ except requests.exceptions.RequestException as e:
62
+ print(f"Error fetching questions: {e}")
63
+ return f"Error fetching questions: {e}", None
64
+ except requests.exceptions.JSONDecodeError as e:
65
+ print(f"Error decoding JSON response from questions endpoint: {e}")
66
+ print(f"Response text: {response.text[:500]}")
67
+ return f"Error decoding server response for questions: {e}", None
68
+ except Exception as e:
69
+ print(f"An unexpected error occurred fetching questions: {e}")
70
+ return f"An unexpected error occurred fetching questions: {e}", None
71
+
72
+ # 3. Run your Agent
73
+ results_log = []
74
+ answers_payload = []
75
+ print(f"Running agent on {len(questions_data)} questions...")
76
+ for item in questions_data:
77
+ task_id = item.get("task_id")
78
+ question_text = item.get("question")
79
+ if not task_id or question_text is None:
80
+ print(f"Skipping item with missing task_id or question: {item}")
81
+ continue
82
+ try:
83
+ submitted_answer = agent(question_text)
84
+ answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
85
+ results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
86
+ except Exception as e:
87
+ print(f"Error running agent on task {task_id}: {e}")
88
+ results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
89
+
90
+ if not answers_payload:
91
+ print("Agent did not produce any answers to submit.")
92
+ return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
93
+
94
+ # 4. Prepare Submission
95
+ submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
96
+ status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
97
+ print(status_update)
98
+
99
+ # 5. Submit
100
+ print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
101
+ try:
102
+ response = requests.post(submit_url, json=submission_data, timeout=60)
103
+ response.raise_for_status()
104
+ result_data = response.json()
105
+ final_status = (
106
+ f"Submission Successful!\n"
107
+ f"User: {result_data.get('username')}\n"
108
+ f"Overall Score: {result_data.get('score', 'N/A')}% "
109
+ f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
110
+ f"Message: {result_data.get('message', 'No message received.')}"
111
+ )
112
+ print("Submission successful.")
113
+ results_df = pd.DataFrame(results_log)
114
+ return final_status, results_df
115
+ except requests.exceptions.HTTPError as e:
116
+ error_detail = f"Server responded with status {e.response.status_code}."
117
+ try:
118
+ error_json = e.response.json()
119
+ error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
120
+ except requests.exceptions.JSONDecodeError:
121
+ error_detail += f" Response: {e.response.text[:500]}"
122
+ status_message = f"Submission Failed: {error_detail}"
123
+ print(status_message)
124
+ results_df = pd.DataFrame(results_log)
125
+ return status_message, results_df
126
+ except requests.exceptions.Timeout:
127
+ status_message = "Submission Failed: The request timed out."
128
+ print(status_message)
129
+ results_df = pd.DataFrame(results_log)
130
+ return status_message, results_df
131
+ except requests.exceptions.RequestException as e:
132
+ status_message = f"Submission Failed: Network error - {e}"
133
+ print(status_message)
134
+ results_df = pd.DataFrame(results_log)
135
+ return status_message, results_df
136
+ except Exception as e:
137
+ status_message = f"An unexpected error occurred during submission: {e}"
138
+ print(status_message)
139
+ results_df = pd.DataFrame(results_log)
140
+ return status_message, results_df
141
+
142
+
143
+ # --- Build Gradio Interface using Blocks ---
144
+ with gr.Blocks() as demo:
145
+ gr.Markdown("# Basic Agent Evaluation Runner")
146
+ gr.Markdown(
147
+ """
148
+ **Instructions:**
149
+
150
+ 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
151
+ 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
152
+ 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
153
+
154
+ ---
155
+ **Disclaimers:**
156
+ Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
157
+ This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
158
+ """
159
+ )
160
+
161
+ gr.LoginButton()
162
+
163
+ run_button = gr.Button("Run Evaluation & Submit All Answers")
164
+
165
+ status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
166
+ # Removed max_rows=10 from DataFrame constructor
167
+ results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
168
+
169
+ run_button.click(
170
+ fn=run_and_submit_all,
171
+ outputs=[status_output, results_table]
172
+ )
173
+
174
+ if __name__ == "__main__":
175
+ print("\n" + "-"*30 + " App Starting " + "-"*30)
176
+ # Check for SPACE_HOST and SPACE_ID at startup for information
177
+ space_host_startup = os.getenv("SPACE_HOST")
178
+ space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
179
+
180
+ if space_host_startup:
181
+ print(f"✅ SPACE_HOST found: {space_host_startup}")
182
+ print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
183
+ else:
184
+ print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
185
+
186
+ if space_id_startup: # Print repo URLs if SPACE_ID is found
187
+ print(f"✅ SPACE_ID found: {space_id_startup}")
188
+ print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
189
+ print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
190
+ else:
191
+ print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
192
+
193
+ print("-"*(60 + len(" App Starting ")) + "\n")
194
+
195
+ print("Launching Gradio Interface for Basic Agent Evaluation...")
196
+ demo.launch(debug=True, share=False)
images/huggingface_logo-noborder.svg ADDED
main.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ def main():
4
+ # Load environment variables from .env file
5
+ if os.path.exists('.env'):
6
+ from dotenv import load_dotenv
7
+ load_dotenv()
8
+
9
+ from app import get_ui
10
+ get_ui().launch(debug=True, share=False)
11
+
12
+ if __name__ == "__main__":
13
+ main()
requirements.txt CHANGED
@@ -1,2 +1,5 @@
1
  gradio
2
- requests
 
 
 
 
1
  gradio
2
+ requests
3
+ langgraph
4
+ langchain-core
5
+ langchain-google-genai
secret.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ USERNAME = "kengboon"
4
+ SUBMISSION_PW = "rMDm1z6Z^-L~yo$.3zHCnL%+=!"
5
+
6
+ def check_secret(input: str) -> bool | None:
7
+ f1 = os.getenv("SUBMISSION_PW")
8
+ f2 = f"{USERNAME}-{SUBMISSION_PW}"
9
+ if (f1 is not None) and ("k" in f1) and (f1[6] == f2[22]) and (ord(f1[7].upper()) == ord(f2[10])+1):
10
+ return input == f1 == f2
tools.py ADDED
File without changes