Package modification

agent.py

@@ -1,11 +1,51 @@
+import os
+import glob
 from smolagents import CodeAgent
 from prompt import get_prompt
 
 class QuestionAnsweringAgent(CodeAgent):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+    
+    def get_current_files(self):
+        """Get a set of all files in the current working directory"""
+        try:
+            # Get all files in current directory (including hidden files)
+            all_files = set()
+            for pattern in ['*', '.*']:
+                all_files.update(glob.glob(pattern))
+            # Filter to only include actual files (not directories)
+            files = {f for f in all_files if os.path.isfile(f)}
+            return files
+        except Exception as e:
+            print(f"Error getting current files: {e}")
+            return set()
+    
+    def cleanup_created_files(self, files_before):
+        """Remove files that were created during execution"""
+        try:
+            files_after = self.get_current_files()
+            newly_created_files = files_after - files_before
+            
+            for file_path in newly_created_files:
+                try:
+                    if os.path.exists(file_path):
+                        os.remove(file_path)
+                        print(f"Cleaned up file: {file_path}")
+                except Exception as e:
+                    print(f"Error cleaning up file {file_path}: {e}")
+                    
+        except Exception as e:
+            print(f"Error during cleanup: {e}")
         
     def __call__(self, question_text, file_name) -> str:
-        enhanced_question = get_prompt(question_text, file_name)
-        response = self.run(enhanced_question, reset=True)
-        return response
+        # Take snapshot of files before execution
+        files_before = self.get_current_files()
+        
+        try:
+            enhanced_question = get_prompt(question_text, file_name)
+            response = self.run(enhanced_question, reset=True)
+            return response
+        finally:
+            # Always clean up files, even if there's an exception
+            self.cleanup_created_files(files_before)

app.py

@@ -1,8 +1,3 @@
-import os
-import gradio as gr
-import requests
-import pandas as pd
-
 from smolagents import (
     InferenceClientModel, 
     FinalAnswerTool
@@ -15,14 +10,14 @@ from tools.image_processing_tools import ask_question_about_image
 
 from agent import QuestionAnsweringAgent
 
-DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
+from submission import build_gradio_interface
 
 model = InferenceClientModel(
     provider="auto",
     model_id="Qwen/Qwen3-Coder-30B-A3B-Instruct",
     temperature=0,
     top_p=1.0, 
-    seed=42,   
+    seed=42  
 )
 
 agent_tools = [
@@ -37,204 +32,12 @@ agent = QuestionAnsweringAgent(
     name="question_answering_expert",
     model=model,
     tools=agent_tools,
-    add_base_tools=True, # Disable auto base tools to avoid overriding custom visit_webpage
-    planning_interval=None,  # Disable planning to ensure immediate stop after final_answer
-    additional_authorized_imports=["bs4"],
+    add_base_tools=True,
+    planning_interval=None,
+    additional_authorized_imports=["os", "bs4", "PIL", "transformers", "torch", "requests", "glob"],
     max_steps=10,
     verbosity_level=2,  # For better debugging
 )
-  
-def run_and_submit_all( profile: gr.OAuthProfile | None):
-    """
-    Fetches all questions, runs the BasicAgent on them, submits all answers,
-    and displays the results.
-    """
-    # --- Determine HF Space Runtime URL and Repo URL ---
-    space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
-
-    if profile:
-        username= f"{profile.username}"
-        print(f"User logged in: {username}")
-    else:
-        print("User not logged in.")
-        return "Please Login to Hugging Face with the button.", None
-
-    api_url = DEFAULT_API_URL
-    questions_url = f"{api_url}/questions"
-    submit_url = f"{api_url}/submit"
-
-    # 1. Instantiate Agent ( modify this part to create your agent)
-    try:
-        # Use the global agent that was already initialized
-        pass  # agent is already defined globally
-    except Exception as e:
-        print(f"Error instantiating agent: {e}")
-        return f"Error initializing agent: {e}", None
-    # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
-    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
-    print(agent_code)
-
-    # 2. Fetch Questions
-    print(f"Fetching questions from: {questions_url}")
-    try:
-        response = requests.get(questions_url, timeout=15)
-        response.raise_for_status()
-        questions_data = response.json()
-        if not questions_data:
-             print("Fetched questions list is empty.")
-             return "Fetched questions list is empty or invalid format.", None
-        print(f"Fetched {len(questions_data)} questions.")
-    except requests.exceptions.RequestException as e:
-        print(f"Error fetching questions: {e}")
-        return f"Error fetching questions: {e}", None
-    except requests.exceptions.JSONDecodeError as e:
-         print(f"Error decoding JSON response from questions endpoint: {e}")
-         print(f"Response text: {response.text[:500]}")
-         return f"Error decoding server response for questions: {e}", None
-    except Exception as e:
-        print(f"An unexpected error occurred fetching questions: {e}")
-        return f"An unexpected error occurred fetching questions: {e}", None
-
-    # 3. Run your Agent
-    results_log = []
-    answers_payload = []
-    print(f"Running agent on {len(questions_data)} questions...")
-    for item in questions_data[:]:
-        task_id = item.get("task_id")
-        question_text = item.get("question")
-        file_name = item.get("file_name")
-        if not task_id or question_text is None:
-            print(f"Skipping item with missing task_id or question: {item}")
-            continue
-        try:
-            submitted_answer = agent(question_text, file_name)
-            
-            # Log the execution timeline
-            print(f"Task {task_id} execution steps: {len(agent.memory.steps)}")
-            timeline_details = []
-            for i, step in enumerate(agent.memory.steps):
-                step_type = type(step).__name__
-                print(f"  Step {i+1}: {step_type}")
-                timeline_details.append(f"Step {i+1}: {step_type}")
-            
-            # Add timeline information to results
-            timeline_summary = "; ".join(timeline_details)
-            answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
-            results_log.append({
-                "Task ID": task_id, 
-                "Question": question_text, 
-                "Submitted Answer": submitted_answer,
-                "Timeline": timeline_summary,
-                "Total Steps": len(agent.memory.steps)
-            })
-        except Exception as e:
-             print(f"Error running agent on task {task_id}: {e}")
-             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
-
-    if not answers_payload:
-        print("Agent did not produce any answers to submit.")
-        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
-
-    # 4. Prepare Submission 
-    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
-    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
-    print(status_update)
-
-    # 5. Submit
-    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
-    try:
-        response = requests.post(submit_url, json=submission_data, timeout=60)
-        response.raise_for_status()
-        result_data = response.json()
-        final_status = (
-            f"Submission Successful!\n"
-            f"User: {result_data.get('username')}\n"
-            f"Overall Score: {result_data.get('score', 'N/A')}% "
-            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
-            f"Message: {result_data.get('message', 'No message received.')}"
-        )
-        print("Submission successful.")
-        results_df = pd.DataFrame(results_log)
-        return final_status, results_df
-    except requests.exceptions.HTTPError as e:
-        error_detail = f"Server responded with status {e.response.status_code}."
-        try:
-            error_json = e.response.json()
-            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
-        except requests.exceptions.JSONDecodeError:
-            error_detail += f" Response: {e.response.text[:500]}"
-        status_message = f"Submission Failed: {error_detail}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except requests.exceptions.Timeout:
-        status_message = "Submission Failed: The request timed out."
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except requests.exceptions.RequestException as e:
-        status_message = f"Submission Failed: Network error - {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except Exception as e:
-        status_message = f"An unexpected error occurred during submission: {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-
- 
-# --- Build Gradio Interface using Blocks ---
-with gr.Blocks() as demo:
-    gr.Markdown("# Basic Agent Evaluation Runner")
-    gr.Markdown(
-        """
-        **Instructions:**
-
-        1.  Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
-        2.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
-        3.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
-
-        ---
-        **Disclaimers:**
-        Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
-        This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
-        """
-    )
-
-    gr.LoginButton()
-
-    run_button = gr.Button("Run Evaluation & Submit All Answers")
-
-    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
-    # Removed max_rows=10 from DataFrame constructor
-    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
-
-    run_button.click(
-        fn=run_and_submit_all,
-        outputs=[status_output, results_table]
-    )
 
 if __name__ == "__main__":
-    print("\n" + "-"*30 + " App Starting " + "-"*30)
-    # Check for SPACE_HOST and SPACE_ID at startup for information
-    space_host_startup = os.getenv("SPACE_HOST")
-    space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
-
-    if space_host_startup:
-        print(f"✅ SPACE_HOST found: {space_host_startup}")
-        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
-    else:
-        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
-
-    if space_id_startup: # Print repo URLs if SPACE_ID is found
-        print(f"✅ SPACE_ID found: {space_id_startup}")
-        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
-        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
-    else:
-        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
-
-    print("-"*(60 + len(" App Starting ")) + "\n")
-
-    print("Launching Gradio Interface for Basic Agent Evaluation...")
-    demo.launch(debug=True, share=False)
+    build_gradio_interface(agent)

prompt.py

@@ -7,13 +7,15 @@ def get_prompt(question_text, file_name):
           NEVER include words like: career, albums, list, biography, years, numbers, prepositions, or date ranges.
           Examples:
             - "Lionel Messi career" → use: wikipedia_summary("Lionel Messi")
-            - "Mercedes Sosa discography" → use: wikipedia_summary("Mercedes Sosa")
             - "Battle of Hastings timeline" → use: wikipedia_summary("Battle of Hastings")
             - "Population of Paris in 2010" → use: wikipedia_summary("Paris")
         - If necessary, visit the wikipedia page listed in the wikipedia summary tool to read the full content. You will find the page url in the output of the wikipedia summary tool at the end after the **Read more:** section. Use the `read_wikipedia_page` tool to visit the page.
         - When using the `read_wikipedia_page` tool, you may find tables in the page. To analyze the tables, please use a code snippet to read the tables into a pandas dataframe and analyze the data.
-        - If necessary, perform a web search using the `web_search` tool to find possible sources of information.
-        - If the web search only returns titles and short snippets, you MUST visit the actual webpage using the `read_wikipedia_page` tool to read the full content before answering.
+        - If necessary, download a youtube video using the `download_youtube_url_audio` or `download_youtube_url_images` tool to find possible sources of information. For the parameter `num_images`, use a large number if you need to have comprehensive information about the video.
+          - If necessary, analyze the audio or images downloaded from youtube using the `ask_question_about_image` tool to find possible sources of information.
+          - If necessary, perform a web search using the `web_search` tool to find possible sources of information.
+        - If necessary, please analyze the images downloaded using the `ask_question_about_image` tool to find possible sources of information.
+        - If the web search only returns titles and short snippets, you MUST visit the actual webpage using the `visit_webpage` tool to read the full content before answering.
         - If the task requires reading, listening, or analyzing a file, you must use the file specified after the question, NOT the file name mentioned casually inside the question text.
         - Comma separated lists MUST contain a single space after each comma.
         - If you are asked for a number, don't use comma to write your number, nor use units such as $$ or percent sign unless specified otherwise.
@@ -37,8 +39,8 @@ def get_prompt(question_text, file_name):
         -- beginning of question --
         {question_text}
         -- end of question --
-        If the questions mentions the need to use a file, use the following `file_name` value below as the `file_name` parameter in any function calls: 
-        file_name: {file_name}
-    """
+        
+        IMPORTANT: If the question mentions the need to use a file, the file name is provided below.
+        file_name: {file_name}"""
     
     return PROMPT

requirements.txt

@@ -1,6 +1,6 @@
 gradio[oauth]==5.44.1
 requests==2.32.5
-smolagents==1.21.3
+smolagents[transformers]==1.21.3
 duckduckgo-search==8.1.1
 ddgs==9.5.5
 markdownify==0.11.0
@@ -11,5 +11,4 @@ beautifulsoup4==4.12.3
 langchain_community==0.3.2
 wikipedia==1.4.0
 tabulate==0.9.0
-yt-dlp==2025.9.5
-transformers==4.46.1
+yt-dlp==2025.9.5

submission.py

@@ -0,0 +1,198 @@
+import os
+import gradio as gr
+import requests
+import pandas as pd
+import numpy as np
+
+
+DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
+SELECTED_QUESTIONS = [3]
+def run_and_submit_all(agent, profile: gr.OAuthProfile | None):
+    """
+    Fetches all questions, runs the BasicAgent on them, submits all answers,
+    and displays the results.
+    """
+    # --- Determine HF Space Runtime URL and Repo URL ---
+    space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
+
+    if profile:
+        username= f"{profile.username}"
+        print(f"User logged in: {username}")
+    else:
+        print("User not logged in.")
+        return "Please Login to Hugging Face with the button.", None
+
+    api_url = DEFAULT_API_URL
+    questions_url = f"{api_url}/questions"
+    submit_url = f"{api_url}/submit"
+
+    # 1. Instantiate Agent ( modify this part to create your agent)
+    try:
+        agent = agent
+    except Exception as e:
+        print(f"Error instantiating agent: {e}")
+        return f"Error initializing agent: {e}", None
+    # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
+    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
+    print(agent_code)
+
+    # 2. Fetch Questions
+    print(f"Fetching questions from: {questions_url}")
+    try:
+        response = requests.get(questions_url, timeout=15)
+        response.raise_for_status()
+        questions_data = response.json()
+        if not questions_data:
+             print("Fetched questions list is empty.")
+             return "Fetched questions list is empty or invalid format.", None
+        print(f"Fetched {len(questions_data)} questions.")
+    except requests.exceptions.RequestException as e:
+        print(f"Error fetching questions: {e}")
+        return f"Error fetching questions: {e}", None
+    except requests.exceptions.JSONDecodeError as e:
+         print(f"Error decoding JSON response from questions endpoint: {e}")
+         print(f"Response text: {response.text[:500]}")
+         return f"Error decoding server response for questions: {e}", None
+    except Exception as e:
+        print(f"An unexpected error occurred fetching questions: {e}")
+        return f"An unexpected error occurred fetching questions: {e}", None
+
+    # 3. Run your Agent
+    results_log = []
+    answers_payload = []
+    is_correct_answers = []
+    print(f"Running agent on {len(questions_data)} questions...")
+    for item in np.array(questions_data).take(SELECTED_QUESTIONS):
+        task_id = item.get("task_id")
+        question_text = item.get("question")
+        file_name = item.get("file_name")
+        if not task_id or question_text is None:
+            print(f"Skipping item with missing task_id or question: {item}")
+            continue
+        try:
+            submitted_answer = agent(question_text, file_name)
+            answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
+            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
+        except Exception as e:
+             print(f"Error running agent on task {task_id}: {e}")
+             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
+        individual_submission_data = {
+            "username": username.strip(), 
+            "agent_code": agent_code, 
+            "answers": [{"task_id": task_id, "submitted_answer": submitted_answer}]
+        }
+    
+        individual_response = requests.post(submit_url, json=individual_submission_data, timeout=60)
+        individual_response.raise_for_status()
+        individual_result_data = individual_response.json()
+        is_correct_answers.append(True if individual_result_data.get("correct_count", 0) == 1 else False)
+    if not answers_payload:
+        print("Agent did not produce any answers to submit.")
+        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
+
+    # 4. Prepare Submission 
+    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
+    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
+    print(status_update)
+
+    # 5. Submit
+    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
+    try:
+        response = requests.post(submit_url, json=submission_data, timeout=60)
+        response.raise_for_status()
+        result_data = response.json()
+        final_status = (
+            f"Submission Successful!\n"
+            f"User: {result_data.get('username')}\n"
+            f"Overall Score: {result_data.get('score', 'N/A')}% "
+            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
+            f"Message: {result_data.get('message', 'No message received.')}"
+        )
+        print("Submission successful.")
+        results_df = pd.DataFrame(results_log)
+        results_df["Is Correct"] = is_correct_answers
+        return final_status, results_df
+    except requests.exceptions.HTTPError as e:
+        error_detail = f"Server responded with status {e.response.status_code}."
+        try:
+            error_json = e.response.json()
+            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
+        except requests.exceptions.JSONDecodeError:
+            error_detail += f" Response: {e.response.text[:500]}"
+        status_message = f"Submission Failed: {error_detail}"
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        results_df["Is Correct"] = is_correct_answers
+        return status_message, results_df
+    except requests.exceptions.Timeout:
+        status_message = "Submission Failed: The request timed out."
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        return status_message, results_df
+    except requests.exceptions.RequestException as e:
+        status_message = f"Submission Failed: Network error - {e}"
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        return status_message, results_df
+    except Exception as e:
+        status_message = f"An unexpected error occurred during submission: {e}"
+        print(status_message)
+        results_df = pd.DataFrame(results_log)
+        return status_message, results_df
+
+
+def build_gradio_interface(agent):
+    # --- Build Gradio Interface using Blocks ---
+    with gr.Blocks() as demo:
+        gr.Markdown("# Basic Agent Evaluation Runner")
+        gr.Markdown(
+            """
+            **Instructions:**
+            1.  Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
+            2.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
+            3.  Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
+            ---
+            **Disclaimers:**
+            Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
+            This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
+            """
+        )
+
+        gr.LoginButton()
+
+        run_button = gr.Button("Run Evaluation & Submit All Answers")
+
+        status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
+        # Removed max_rows=10 from DataFrame constructor
+        results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
+
+        def run_with_login_state(profile: gr.OAuthProfile):
+            return run_and_submit_all(agent, profile)
+
+        run_button.click(
+            fn=run_with_login_state,
+            outputs=[status_output, results_table]
+        )
+
+    print("\n" + "-"*30 + " App Starting " + "-"*30)
+    # Check for SPACE_HOST and SPACE_ID at startup for information
+    space_host_startup = os.getenv("SPACE_HOST")
+    space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
+
+    if space_host_startup:
+        print(f"✅ SPACE_HOST found: {space_host_startup}")
+        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
+    else:
+        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
+
+    if space_id_startup: # Print repo URLs if SPACE_ID is found
+        print(f"✅ SPACE_ID found: {space_id_startup}")
+        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
+        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
+    else:
+        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
+
+    print("-"*(60 + len(" App Starting ")) + "\n")
+
+    print("Launching Gradio Interface for Basic Agent Evaluation...")
+    demo.launch(debug=True, share=False)

tools/audio_processing_tools.py

@@ -0,0 +1,11 @@
+from whisper import load_model
+from smolagents.tools import tool
+
+@tool
+def ask_question_about_audio(question: str, path_to_audio: str) -> str:
+    """
+    Ask a question about an audio and return the answer.
+    """
+    model = load_model("base")
+    res = model.transcribe(path_to_audio)
+    return res["text"]

tools/image_processing_tools.py

@@ -1,6 +1,24 @@
 from PIL import Image
-from transformers import pipeline
+from transformers import BlipProcessor, BlipForQuestionAnswering
 from smolagents.tools import tool
+import torch
+import requests
+import os
+
+DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
+
+def _download_file(file_name: str) -> None:
+    """Download file if it doesn't exist locally"""
+    try:
+        # Try to open the file to check if it exists
+        with open(file_name, 'rb') as f:
+            pass  # File exists, do nothing
+    except FileNotFoundError:
+        # File doesn't exist, download it
+        url = f"{DEFAULT_API_URL}/files/{file_name.split('.')[-2]}"
+        r = requests.get(url)
+        with open(file_name, "wb") as f:
+            f.write(r.content)
 
 @tool
 def ask_question_about_image(question: str, path_to_image: str) -> str:
@@ -12,24 +30,30 @@ def ask_question_about_image(question: str, path_to_image: str) -> str:
     Returns:
         A string with the answer to the question.
     """
-    pipe = pipeline("image-text-to-text", model="llava-hf/llava-interleave-qwen-0.5b-hf")
+    # Download the file if it doesn't exist
+    _download_file(path_to_image)
     
-    image = Image.open(fp=path_to_image)
+    # Check if CUDA is available and use GPU if possible, otherwise use CPU
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
     
-    messages = [
-        {
-            "role": "user",
-            "content": [
-                {
-                    "type": "image",
-                    "image": image,
-                },
-                {"type": "text", "text": question},
-            ],
-        }
-    ]
-
-    outputs = pipe(text=messages, max_new_tokens=60, return_full_text=False)
-
-    return outputs[0]["generated_text"]
+    # Load the processor and model (using BLIP for more stable VQA)
+    processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+    model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
+    model = model.to(device)
+    
+    # Load and process the image
+    image = Image.open(path_to_image).convert('RGB')
+    
+    # Process the inputs
+    inputs = processor(image, question, return_tensors="pt")
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+    
+    # Generate the answer
+    with torch.no_grad():
+        outputs = model.generate(**inputs, max_length=50, num_beams=5)
+    
+    # Decode and return the answer
+    answer = processor.decode(outputs[0], skip_special_tokens=True)
+    
+    return answer
 

tools/youtube_tools.py

@@ -3,8 +3,6 @@ import os
 import subprocess
 from yt_dlp import YoutubeDL
 
-
-
 from smolagents.tools import tool
 
 # Use FFmpeg to extract frames from the video
@@ -54,13 +52,13 @@ def extract_frames_with_ffmpeg(video_path: str, num_frames: int) -> [str]:
 @tool
 def download_youtube_url_audio(url: str) -> str:
     """
-    Download a YouTube video and return the path to the downloaded file.
+    Download a YouTube video using the url, extract the audio and return the path to the downloaded file.
 
     Args:
         url (str): The URL of the YouTube video to download.
 
     Returns:
-        str: The path to the downloaded file.
+        str: The path to the downloaded audio file.
     """
     ydl_audio_opts = {
         'format': 'bestaudio/best',
@@ -76,20 +74,21 @@ def download_youtube_url_audio(url: str) -> str:
 
     with YoutubeDL(ydl_audio_opts) as ydl:
         file_path = ydl.extract_info(url)
+    
     return file_path['requested_downloads'][0]['filepath']
 
 
 @tool
 def download_youtube_url_images(url: str, num_images: int = 3) -> str:
     """
-        Download a YouTube video and return the path to the downloaded file.
+        Download a YouTube video using the url, extract the frames and return the path to the downloaded files.
 
         Args:
             url (str): The URL of the YouTube video to download.
-            num_images (int): The number of images to download.
+            num_images (int): The number of images to download. The images are extracted from the video at regular intervals. 
 
         Returns:
-            str: The different paths to the downloaded files, separated by newlines.
+            str: The different paths to the downloaded frames, separated by newlines.
     """
     # First, download the video
     ydl_images_opts = {