Delete Mohamed Maged (221101329) (Ais) Task 7 Advanced machine.ipynb

Mohamed Maged (221101329) (Ais) Task 7 Advanced machine.ipynb

@@ -1,454 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Initial State: (0, 0)\n"
-     ]
-    }
-   ],
-   "source": [
-    "import numpy as np\n",
-    "\n",
-    "# Create a grid environment\n",
-    "class Gridworld:\n",
-    "    def __init__(self):\n",
-    "        self.grid_size = 5\n",
-    "        self.start_state = (0, 0)\n",
-    "        self.goal_state = (4, 4)\n",
-    "        self.obstacles = [(2, 2), (3, 3)]\n",
-    "        self.state = self.start_state\n",
-    "\n",
-    "    def reset(self):\n",
-    "        self.state = self.start_state\n",
-    "        return self.state\n",
-    "\n",
-    "    def step(self, action):\n",
-    "        actions = {\n",
-    "            0: (-1, 0),  # Up\n",
-    "            1: (1, 0),   # Down\n",
-    "            2: (0, -1),  # Left\n",
-    "            3: (0, 1)    # Right\n",
-    "        }\n",
-    "        next_state = (self.state[0] + actions[action][0],\n",
-    "                      self.state[1] + actions[action][1])\n",
-    "\n",
-    "        # Ensure the agent doesn't move out of bounds\n",
-    "        if 0 <= next_state[0] < self.grid_size and 0 <= next_state[1] < self.grid_size:\n",
-    "            self.state = next_state\n",
-    "\n",
-    "        # Check for rewards or penalties\n",
-    "        if self.state == self.goal_state:\n",
-    "            return self.state, 100, True  # Goal reached\n",
-    "        elif self.state in self.obstacles:\n",
-    "            return self.state, -10, False  # Hit obstacle\n",
-    "        else:\n",
-    "            return self.state, -1, False  # Normal step\n",
-    "\n",
-    "# Test environment\n",
-    "env = Gridworld()\n",
-    "print(\"Initial State:\", env.reset())\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Q-Learning parameters\n",
-    "episodes = 500\n",
-    "alpha = 0.1   # Learning rate\n",
-    "gamma = 0.9   # Discount factor\n",
-    "epsilon = 0.2  # Exploration rate\n",
-    "actions = [0, 1, 2, 3]  # Up, Down, Left, Right\n",
-    "\n",
-    "# Initialize Q-table\n",
-    "Q_table = np.zeros((5, 5, len(actions)))\n",
-    "\n",
-    "# Q-Learning function\n",
-    "def train_gridworld(env):\n",
-    "    for episode in range(episodes):\n",
-    "        state = env.reset()\n",
-    "        done = False\n",
-    "\n",
-    "        while not done:\n",
-    "            # Epsilon-greedy action selection\n",
-    "            if np.random.uniform(0, 1) < epsilon:\n",
-    "                action = np.random.choice(actions)\n",
-    "            else:\n",
-    "                action = np.argmax(Q_table[state[0], state[1], :])\n",
-    "\n",
-    "            # Take action\n",
-    "            next_state, reward, done = env.step(action)\n",
-    "\n",
-    "            # Update Q-value\n",
-    "            Q_table[state[0], state[1], action] = Q_table[state[0], state[1], action] + \\\n",
-    "                alpha * (reward + gamma * np.max(Q_table[next_state[0], next_state[1], :]) -\n",
-    "                         Q_table[state[0], state[1], action])\n",
-    "\n",
-    "            state = next_state\n",
-    "\n",
-    "train_gridworld(env)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "['↓', '↓', '↓', '↓', '↓']\n",
-      "['→', '↓', '→', '→', '↓']\n",
-      "['→', '↓', '←', '→', '↓']\n",
-      "['→', '→', '↓', '→', '↓']\n",
-      "['→', '→', '→', '→', '↑']\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Display optimal policy\n",
-    "policy = np.argmax(Q_table, axis=2)\n",
-    "actions_mapping = {0: '↑', 1: '↓', 2: '←', 3: '→'}\n",
-    "\n",
-    "for row in policy:\n",
-    "    print([actions_mapping[action] for action in row])\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "State space: Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
-      "Action space: Discrete(3)\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "array([[[255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        ...,\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255]],\n",
-       "\n",
-       "       [[255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        ...,\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255]],\n",
-       "\n",
-       "       [[255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        ...,\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255]],\n",
-       "\n",
-       "       ...,\n",
-       "\n",
-       "       [[255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        ...,\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255]],\n",
-       "\n",
-       "       [[255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        ...,\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255]],\n",
-       "\n",
-       "       [[255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        ...,\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255],\n",
-       "        [255, 255, 255]]], dtype=uint8)"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import gym\n",
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "# Initialize environment\n",
-    "env = gym.make(\"MountainCar-v0\", render_mode=\"rgb_array\")\n",
-    "\n",
-    "# Inspect state and action spaces\n",
-    "print(\"State space:\", env.observation_space)\n",
-    "print(\"Action space:\", env.action_space)\n",
-    "\n",
-    "# Visualize environment\n",
-    "state = env.reset()\n",
-    "env.render()\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "C:\\Users\\moham\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
-      "  if not isinstance(terminated, (bool, np.bool8)):\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Model saved as 'q_learning_model.pkl'\n"
-     ]
-    }
-   ],
-   "source": [
-    "import pickle\n",
-    "\n",
-    "# Initialize Q-table\n",
-    "state_bins = [20, 20]  # Number of bins for discretizing the state space\n",
-    "action_space = env.action_space.n\n",
-    "Q_table = np.random.uniform(low=-1, high=1, size=(state_bins[0], state_bins[1], action_space))\n",
-    "\n",
-    "# Helper to discretize state space\n",
-    "def discretize_state(state):\n",
-    "    state_low = env.observation_space.low\n",
-    "    state_high = env.observation_space.high\n",
-    "    bins = [np.linspace(state_low[i], state_high[i], state_bins[i]) for i in range(len(state))]\n",
-    "    state_indices = [np.digitize(state[i], bins[i]) - 1 for i in range(len(state))]\n",
-    "    return tuple(state_indices)\n",
-    "\n",
-    "# Initialize Q-learning parameters\n",
-    "alpha = 0.1\n",
-    "gamma = 0.99\n",
-    "epsilon = 0.2\n",
-    "episodes = 5000\n",
-    "epsilon_decay = 0.995\n",
-    "\n",
-    "# Track total rewards for plotting\n",
-    "total_rewards = []\n",
-    "\n",
-    "# Train the agent\n",
-    "for episode in range(episodes):\n",
-    "    state, _ = env.reset()\n",
-    "    state = discretize_state(state)\n",
-    "    done = False\n",
-    "    total_reward = 0\n",
-    "\n",
-    "    while not done:\n",
-    "        # Epsilon-greedy action selection\n",
-    "        if np.random.uniform(0, 1) < epsilon:\n",
-    "            action = np.random.choice(action_space)\n",
-    "        else:\n",
-    "            action = np.argmax(Q_table[state])\n",
-    "\n",
-    "        # Take action\n",
-    "        next_state, reward, done, _, _ = env.step(action)\n",
-    "        next_state = discretize_state(next_state)\n",
-    "        total_reward += reward\n",
-    "\n",
-    "        # Update Q-value\n",
-    "        Q_table[state + (action,)] += alpha * (\n",
-    "            reward + gamma * np.max(Q_table[next_state]) - Q_table[state + (action,)]\n",
-    "        )\n",
-    "        state = next_state\n",
-    "\n",
-    "    # Decay epsilon\n",
-    "    epsilon = max(0.01, epsilon * epsilon_decay)\n",
-    "\n",
-    "    # Log total reward\n",
-    "    total_rewards.append(total_reward)\n",
-    "\n",
-    "# Save the model as a .pkl file\n",
-    "model_data = {\n",
-    "    \"Q_table\": Q_table,\n",
-    "    \"state_bins\": state_bins,\n",
-    "    \"alpha\": alpha,\n",
-    "    \"gamma\": gamma,\n",
-    "    \"epsilon_decay\": epsilon_decay,\n",
-    "}\n",
-    "with open(\"q_learning_model.pkl\", \"wb\") as f:\n",
-    "    pickle.dump(model_data, f)\n",
-    "\n",
-    "print(\"Model saved as 'q_learning_model.pkl'\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Total reward: -140.0\n"
-     ]
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAFeCAYAAAAYIxzjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwIklEQVR4nO3dd3RU5aL+8Wdm0iYBISGBACGhSEsAAQExtEBUWgARBRG4ENRrQVHxqNdyztF7LcfjAbGB7dAEBARFOkoQEAuIgDSDtAASIKEkkDqTmf37wwM/C2qASXYm+/tZK2u5AjPzLJlkP/Put9gMwzAEAAAsy252AAAAYC7KAAAAFkcZAADA4igDAABYHGUAAACLowwAAGBxlAEAACyOMgAAgMVRBgAAsDjKAAAAFkcZAADA4igDAABYHGUAAACLowwAAGBxlAEAACyOMgAAgMVRBgAAsDjKAAAAFkcZAADA4igDAABYHGUAAACLCzA7AAAAVmEYxgW/b7PZyjnJL1EGAAAoJx7PSe3c2UJhYR0UGtpBYWHtFRraVjZbkGy2ANlsgf/5Kt9yYDN+r6YAAACfcruztW1bzV99N0BOZys5na0UGtpKTmdLBQTUkMNR7fyXzVa2n90pAwAAlJMLl4HfCgqqr6CghgoObqjg4AYKCqqnwMB6Cgqqq6CgurLbQ32aizIAAEA5KW0Z+DWHI1wBATUVEBClwMAoBQU1VEhIc4WENJPT2UwBATUuKxdzBgAAqOA8ntPyeE6ruHi3JMlmC5LdHia7PVR2e5hiYv6l6tX7XfLzUwYAAKjgfppUGCybLVh2e7CCgxsrLOwahYa2V1hYewUFxVzW81MGAACoYByO6nI4IhQQEC6HI0IhIU3ldLaU05kgp7OFHI5qPn09ygAAAKayKygo9mdf9RQU1EBBQXEKDq6voKA42e3BZZqAMgAAQDmy2ULkdCYoJCRBTme8QkKaKyAgUg5HDQUERCggoIZstvLdIJgyAABAOTlzRnr22daaO3fhf+7/h/xnLgA7EAIAYAler3TyZJACA2uZHeUXOKgIAACLowwAAGBxlAEAACyOMgAAgMUxgRAAgMtw4MABnT17tlR/NycnR/n5+dq2bZvPczRp0kQhISGX9FgOKgIA4DL07t1bK1asMDuGtm3bppYtW17SY7lNAACAxVEGAACwOMoAAAAWRxkAAMDiKAMAAFgcZQAAAIujDAAAYHGUAQAALI4dCAEAKANVq1ZVlSpVFBgYKElyuVzKy8tTXl5eqR4fEBCgatWqyel0yuFwyOPxqLCwUGfOnJHb7fZpVsoAAAA+FBwcrCuvvFLx8fGqW7euqlatKsMwlJeXp4MHD2rRokUKCAhQcXHx7z5HjRo11LJlSzVq1EiRkZEKCQlRcXGxTp48qX379mnHjh3KysryWWbKAAAAPhISEqJrrrlGHTp0UGhoqGw22/k/q169ugIDAxUZGalmzZppw4YNKioq+s1z1KxZUykpKapbt64cDscvnrtu3bqKjo5Wo0aNtHTpUmVmZvokN3MGAADwkfr16ysxMVFhYWG/KAI/FxISosTERMXFxV3wz/v376969er9ogj8nMPhUN26dXXjjTf6KjZlAAAAX6hdu7ZuueUWBQcH/+nfDQ4O1uDBgxUdHX3+ew6HQ0OHDlXdunV/t0j8XFRUlEaNGvW7peFiUAYAALhMdrtdiYmJF3Vhdjgc6tSp0/kLf8OGDVWrVq1SFQFJstlsqlGjhpo2bXpJmX+OMgAAwGWy2Wxq0aLFRT8uISFBdvtPl+I6deqoevXqF/X4qlWrKjY29qJf99coAwAAWBxlAACAyzBt2jTt3r271MP7P2e32/XDDz/owIEDeuihhy7p9VNTU3XgwIHLul3A0kIAAC5DrVq1VKNGDTmdThUWFl7UY51Op+Li4uRwOJSZmXl+c6HSstvtioyMVP369S8y9a+e57IeDQAAZLfblZycfNGPS05OPj9noFmzZr9YXVAa5zYnulyUAQAALpPdblfz5s2VkJBQ6se0aNFCzZs3P397ISIiQomJiQoJCSnV4wMCAtS5c2dFRUVdUuafowwAAOADoaGh6tq1q2JiYv7079arV09dunRRaGjoL74fHx+vbt26ler1unTpolatWl1S1l+jDAAA4CORkZHq27evYmNj5XA4fjGp0GazyeFwqGbNmurbt68iIyMv+Bzt2rVTv3795HQ6fzMp0WazKSQkRL169VLHjh19lttmGIbhs2cDAMDizl1Wd+3apd27d+vkyZOSfrq/X6NGDf3tb3/TmjVr5Ha7VVRUJIfDIcMwZBiGPB6PXC6XnE6n8vLytGPHDv34448qLi5WaGioYmJidNVVVykiIkKSLmkFw4WwmgAAAB86d4FOSEhQ06ZNdfLkSR09elTHjh3T8ePHlZ2drZkzZyo3N1dHjhxRdHS0DMOQ1+tVXl6eDh8+rPj4eAUGBiogIOD8aMAVV1whSTpw4ICys7MVGBio+vXr+2Q7YsoAAAA+kp+fr82bN2vr1q06c+aMjh8/ruDgYLlcLp09e1Z16tSRy+VSTk7O+aOOr7jiCjkcjvMjBI0aNZLT6ZTL5VJxcbGKiop05swZnThxQm63Wy6XS3l5edq3b58aNGiguLg4NWrU6Px/h4WFXXRubhMAAHCJDMPQ5s2btWPHDn399dfKyMhQRESEatasqWuvvVYxMTGqUqWKQkNDFRQUpNDQUGVmZiohIeGiPtEbhqHCwsLzXwUFBcrJyVFhYaEyMjKUkZGhvXv3KiMjQ506dVJSUpISExPPjyb8GcoAAAClYBiG3G633G63Tp48qcWLF2vhwoXKyclR7969lZSUpJYtW8rpdMrhcCgwMPA3kwjLIlNJScn5r8LCQq1fv15paWnauHGjmjRpolmzZv3p81AGAAD4A16vVydOnFBGRoZWr16tPXv26NChQ+rdu7dSUlLUqFGj8xsHSb6b1Hepzl3WXS6XNm/erGuvvfZPH0MZAADgAs6ePat9+/bpu+++048//qhTp04pKipKycnJateunekXfV+iDAAA8DMFBQX65JNPtGLFCkVERCguLk7x8fFKSEg4v6SvsqEMAAAs79ylcPny5Zo5c6YiIiLUu3dvXXXVVapZs6aCgoJMTli2KAMAAMs6t7Z/xYoVeuedd9SsWTPdfffduvLKKxUYGPiLuQCVGWUAAGBJR48e1bZt27RkyRKVlJTovvvuU/PmzS1TAH6OMgAAsJTMzEx9/vnn2rNnj0pKSpSSkqI2bdr4ZCc/f0UZAABYgsvl0ieffKKFCxcqPj5eiYmJatOmjYKDg82OZjrKAACgUjMMQ/n5+XrmmWd09uxZDR06VFdffbXCwsIq1fLAy8HZBACASqmkpES5ublatWqVXnnlFf3P//yPevfuff7wH/x/lAEAQKXjdruVlpamDz74QA0bNtSKFStKvU+/FXGbAABQqRw8eFBz585VcXGxOnTooKSkJOYF/AlGBgAAlYJhGFqxYoU++eQTde3aVYmJiapVq5bZsfwCZQAA4NcMw9Dhw4f19NNPq1q1ahozZowaNGhg6aWCF4vbBAAAv+V2u7V//36NHz9eCQkJGjNmTJkfG1wZUQYAAH4pKytLa9eu1erVq5WamqoOHTqYHclvUQYAAH5n9+7d+vjjjxUWFqYhQ4YoMjLS7Eh+jTkDAAC/4fV6lZaWpjlz5ui2225TYmKinE6n2bH8HmUAAOAX3G63Zs6cqQ0bNujFF19UeHg4kwR9hDIAAKjQvF6vjh8/rn//+98KDg7W5MmTJYlJgj5EGQAAVFhFRUX64osvtHbtWrVu3Vr9+vWjBJQBygAAoELyer36+OOPtWLFCt17771q06aNAgK4bJUFVhMAACqkV199Vfn5+Ro8eLAaNWpkdpxKjTIAAKgwDMNQUVGRnn/+ecXExOi//uu/WC1QDhhvAQBUCB6PRz/88INmzZqlli1bauDAgQoKCjI7liVQBgAApjMMQ5s2bdJbb72lIUOG6LrrrmPZYDniNgEAwHSfffaZVq9ereTkZCUlJZkdx3IoAwAA0xiGoYULF2rr1q26/fbbFRsba3YkS6IMAABM4Xa7tXjxYu3evVt33HGHIiMj2UPAJJQBAEC5MgxDbrdbCxYs0OHDh5WamqqoqCizY1kaEwgBAOVu0qRJOnPmjMaOHavq1aubHcfyGBkAAJSb4uJi/fWvf1WbNm3Ur18/ValSxexIEGUAAFAODMNQQUGBnnvuOXXt2lXXXXcdWwtXIJQBAECZMgxDp0+f1tSpU9WwYUMNGDBAdrvd7Fj4GWoZAKBMZWVlacqUKYqJidHAgQPNjoMLoJoBAMpMVlaW3nzzTUVHR2vEiBFmx8HvYGQAAFAmjh8/rkmTJqlbt27q3r272XHwBygDAACfMgxDJ0+e1DvvvKPk5GR17tyZzYQqOMoAAMBnzhWB2bNnq3Xr1urSpQtFwA9QBgAAPpORkaE5c+aoYcOGSklJMTsOSokJhAAAnzh16pQmTpyounXrasiQIWbHwUVgnwEAwGU7e/asXnjhBSUnJ6tHjx7cGvAz3CYAAFwywzBUVFSkN954Q507d1ZSUhJFwA9RBgAAl8zlcmnWrFmKjIxU7969KQJ+ijkDAIBL4vV6NX36dOXk5Gj06NEUAT/GyAAA4JJMmDBBNptN999/P2cN+DkmEAIALtrkyZNlt9s1YsQIhYaGmh0Hl4mRAQBAqXk8Hi1atEgej0e33XabnE6n2ZHgA4zrAABKxePx6IsvvtC+fft00003qVq1aswTqCQoAwCAP2UYhjZt2qT169erf//+qlOnjtmR4EOUAQDAn1qyZIkmTpyogQMHqkmTJmbHgY8xZwAA8LsMw9DBgwc1f/58PfXUU2revLnZkVAGWE0AALggwzCUnZ2tf/zjH7r99tsVHx/PHIFKipEBAMAFnT17VtOnT1dycrISEhLMjoMyxJwBAMBvuFwuzZ49WzVr1tR1111ndhyUMUYGAAC/8eabb8put2vQoEEKDg42Ow7KGGUAAHCeYRh67rnnlJ6erjfffFNVqlQxOxLKAWUAACDpp02F1q1bp8LCQk2ePJkiYCHMGQAAyOv1aufOnVq3bp3uvPNOVa1a1exIKEeUAQCAsrKytGDBAvXu3Vv169c3Ow7KGWUAACzO5XJp4sSJSkxM1NVXX212HJiAMgAAFubxePTMM8+odevW6tGjhxwOh9mRYALKAABYVHFxsZ544gkdPXpUgwcPVmBgoNmRYBLKAABYkNvt1qpVqxQREaHXXntNdjuXAyvjXx8ALGj79u3atGmThg0bprCwMLPjwGSlLgOzZ88uyxwAgHKSlZWluXPnqn///qpbt67ZcVABlLoMZGdn67333pPX6y3LPACAMlRcXKyXXnpJycnJatWqFacQQtJFlIERI0YoIyND69evl8fjKctMAIAykJubq/Hjxys+Pl7XX389KwdwXqnLQEREhG6++WZ99tln2r9/vwzDKMtcAAAfKi4u1rvvvqucnByNHDmSEQH8wkVNIGzevLk6deqkmTNnKi8vr6wyAQB8bPXq1SosLNQTTzzBygH8xkW/I7p166b4+HhNnDiR0QEA8APp6enauHGjBg8erGrVqpkdBxXQRZeBwMBADRo0SC6XSy+//LLcbndZ5AIAXCbDMHTy5EnNmzdPSUlJuvLKK7k9gAu6pLGigIAAPfXUU/r22281f/58VhgAQAVUXFys6dOnKzY2Vl27duX2AH7XJb8zgoOD9eKLL2r79u3asWOHLzMBAHxg2rRpcrlcGjVqFCMC+EOXVRNr166tvn37atmyZTp69KivMgEALtPMmTO1detW3X///WZHgR+4rDLgcDjUvn17xcXFacGCBSouLvZVLgDAJTAMQ5s3b9aePXv00EMPKTQ01OxI8AOXfQMpKChIgwcP1rFjx7RkyRLmDwCASQzD0LFjx7Ry5UrdcMMNatKkCbcHUCo+mU3icDj07LPP6v3339eXX37pi6cEAFwkt9utjz76SNHR0erUqRNFAKXm06mlzz//vN59911t3rzZl08LAPgThmFo0aJFys7O1tChQ82OAz/j0zLQqFEjjRo1SsuWLdOPP/7oy6cGAPyB1atX69tvv9WYMWMUEhJidhz4GZ+WAYfDoU6dOqlBgwZavny5CgsLffn0AIBfMQxD33zzjd544w3de++9ioyMNDsS/JDPd6AIDAzUrbfeql27dmnTpk1sWQwAZejUqVOaNm2annzyScXExJgdB36qTLajcjgceuSRRzR37lzt3LmzLF4CACyvoKBACxcuVGJiolq0aMGEQVyyMtubsk6dOrrzzjs1ZcoUZWRklNXLAIAleTweff755zp9+rR69uyp4OBgsyPBj5XpRtWtWrXSgAED9Mwzzyg7O7ssXwoALCUjI0MLFizQkCFDmCeAy2Yzyvimvsvl0qxZs5Sbm6uxY8dyUAYAXKaSkhL17NlTU6ZMUVxcnNlxUAmU+ZU5MDBQKSkpcrvdWrt2rTweT1m/JABUWrm5uXr00Uf16KOPKjY21uw4qCTKvAzYbDZFRUWpZ8+eWrt2rTIyMlhhAACXoKCgQFOmTFFISIi6dOnChEH4TLmN2bdq1UqdO3fWK6+8wvkFAHCRDMPQpk2blJOTo7Fjx3IAEXyqXG/gd+/eXa1atdJLL71Uni8LAH4vKytLy5Yt08CBAxUdHW12HFQy5VoGHA6HRowYoaKiIs2bN4/5AwBQCi6XS5MnT1bHjh3VsmVLs+OgEir3qf1BQUG66667NGvWLKWlpTF/AAD+gMfj0cyZMxUcHKwBAwbI4XCYHQmVULmXAZvNptq1a+uJJ57Ql19+qaysrPKOAAB+Y82aNdq0aZMee+wxJgyizJi26P+qq65SkyZNtGjRIg40AoALWL9+vd577z2NGzeOIoAyZVoZCAkJUUpKig4fPqz169dzuwAAfubo0aNasWKFBg0apPr161MGUKbKfAfCP1NUVKSePXvqgw8+UM2aNc2MAgAVgtvt1gcffKATJ07o3nvvVUBAgNmRUMmZvjdwSEiI3n77bT366KPMHwBgeYZhaMuWLdqwYYNSU1MpAigXppcBSWrcuLH69u2rf/3rX8rMzDQ7DgCYZt++fZo1a5buueceVa1a1ew4sIgKUQbsdrt69uypKlWq6NNPP2X/AQCWlJubq/Hjx+u2225Ts2bNzI4DC6kQZUCSrrjiCqWmpmr37t1KT09nQiEASzEMQxMnTlRycrLat29vdhxYTIUpA5IUExOjvn37atasWTp9+jSFAIAleDwezZgxQ0ePHlW3bt1YOYByV6HKgM1mU6dOndS4cWO9++67KikpMTsSAJQpwzCUnp6u9PR03X///YqKiqIMoNxVqDJwTmpqqk6dOqWPPvrI7CgAUKYKCws1f/58de3aVQkJCWbHgUVVyDIgSQ8//LC2bt2q9evXmx0FAMqEYRiaOnWqoqKilJycbHYcWFiFLQORkZHq16+fxo8fr++++475AwAqFcMw9Omnn2rfvn0aOXKkgoKCzI4EC6uwZcBms6ljx44aMmSI1q5dq6KiIrMjAYDP7N+/X//+97/1v//7vwoLCzM7DiyuwpYB6adC0KdPH+Xl5WndunXsPwCgUjh69Khee+01PfjggwoNDTU7DlCxy4D00/4Dt956qz799FPt3r3b7DgAcFnOnj2ruXPnKiYmRgkJCbLbK/yvYViAX7wLGzZsqNTUVD311FMqLi42Ow4AXBKv16sdO3YoMzNTI0eO1BVXXGF2JECSn5QBSYqPj9fIkSP1t7/9TV6v1+w4AHDR8vLy9Oqrr+quu+5SVFSU2XGA8/ymDNhsNt1www2Kjo7W7NmzGSEA4Ffy8/P1zDPPKDU1VQ0bNjQ7DvALflMGJMnpdColJUXr1q3Tt99+y3JDAH7B7XZr2rRpatiwoa6//np2GESF41dlQPrpuONbb71Vq1ev1unTp82OAwB/Ki0tTVlZWUpNTaUIoELyuzIgSZ06dVJ0dLTmzZvHckMAFdrWrVu1ePFi3XTTTXI6nWbHAS7IL8tAcHCwRowYwXbFACoswzB08uRJvf/++0pMTFSLFi0YFUCF5ZdlQPqpELz++uv6+9//rszMTLPjAMAvGIahL7/8UoGBgbrtttvkcDjMjgT8Lr8tA5LkcDj04osvatKkSTp27JjZcQDgvM2bN2vlypV64IEHGBFAhefXZcBms6l169Zq2rSpZsyYodzcXLMjAYAOHjyoqVOnasyYMewnAL/g12VA+ul2Qd++fXXy5Elt2bLF7DgALM7j8ej555/X8OHD1axZM7PjAKXi92VAksLDwzVy5EgtWrRIR44cYf8BAKYoKSnR9OnT1b59e7Vp04bbA/AblaIM2Gw2xcfHKykpSW+99ZYKCgrMjgTAYjwej9auXaudO3cqOTlZISEhZkcCSq1SlIFz+vfvr+rVq2vKlClmRwFgMadOndK8efOUkpKiBg0amB0HuCiVqgxI0t13363MzEytXLnS7CgALMLr9Wrq1Km69tprlZSUZHYc4KJVujLgdDp1++236/PPP9fu3buZPwCgTBmGofnz56uwsFBDhgxhngD8UqUrAzabTY0aNVLHjh01ffp0zi8AUKa+++47LV68WI899hjbDcNvVboyIP1UCLp37y6n06nVq1erpKTE7EgAKqGsrCy9/fbb+utf/6rg4GCz4wCXrFKWAUkKCwtTamqqtmzZoi1btnC7AIBP5eTkaN68eerRo4fi4uK4PQC/VmnLgCTFxMRoxIgRevnll9mdEIDPuN1uLVmyREePHlVycjKjAvB7lboMSFKzZs1055136tFHH5XX6zU7DgA/ZxiGTp8+rcWLF+uee+5ReHi42ZGAy1bpy4Akde7cWW3bttW0adPkdrvNjgPAj+Xn5+vvf/+7HnjgAdWtW9fsOIBPWKIMBAYGauDAgcrMzNSGDRsYIQBwSYqKivTmm2+qQ4cOSkxMZJ4AKg1LlAFJqlWrlpKSkrRgwQIdOXLE7DgA/NCSJUtUVFSk4cOHmx0F8CnLlAFJuvrqq3XVVVdp/vz5crlcZscB4Ee2bNmiXbt2afjw4QoICDA7DuBTlioDTqdTt956q7Kzs7Vy5UqWGwL4U4ZhKCsrS59++qk6d+6s2NhYbg+g0rFUGZCkkJAQPfvss3rttde0b98+s+MAqODcbrfee+89FRYWqnv37rLbLfdrExZgyXe1zWbTK6+8oldffVU//vij2XEAVGDffPONDh06pHHjxjEigErLsmWgcePGuu666/Thhx8qJyfH7EgAKqDvv/9ec+bM0dixY1W1alWz4wBlxpJlQJICAgKUnJyskpISffbZZyw3BPALZ8+e1YQJEzR69Gg1atTI7DhAmbJsGZB+Or+gZ8+emjlzpr7//nsmFAKQJHk8Hk2ePFk9e/ZUy5YtzY4DlDlLlwFJio+P14MPPshxxwAkSSUlJUpLS1NAQIC6d+8uh8NhdiSgzFm+DNhsNnXp0kWtWrXS66+/znHHgIUZhqH09HStWbNGPXv2VI0aNZg0CEuwfBk4Z9iwYZKkOXPmmJwEgFncbrfeeustdejQQQkJCWbHAcoNZeBn/vu//1t79+7Vl19+yfwBwGIMw9CkSZMUHx+vPn36mB0HKFeUgf+w2WyqVauW+vXrp1WrVunIkSMUAsAivF6vVq5cqcOHD+v2229XUFCQ2ZGAckUZ+Bmbzaarr75aderU0QcffKDCwkKzIwEoB7t379bChQv15JNPUgRgSZSBC0hJSdGhQ4e0atUqRgeASu748eP66KOPNHz4cFWvXt3sOIApKAMXEB0drXHjxiktLU3btm0zOw6AMlJYWKgVK1YoJiZGbdu25dwBWBbv/N9Rr149Pfzww3rqqad09uxZs+MA8DHDMLR69WqtW7dON954o0JDQ82OBJjGZjAO/rsMw9CaNWu0bNkyPfvsswoODjY7EgAfOXXqlIYMGaKZM2eqVq1aZscBTMXIwB+w2Wzq0KGDGjdurIULF6q4uNjsSAB8IDs7W88884z+7//+jyIAiDLwp8LCwtSnTx/t2bNHW7Zs4UAjwM/l5eXpvffeU2Jiojp06GB2HKBCoAyUQkxMjLp3767Jkydz3DHg55YsWaLAwED169ePCYPAf/CTUEodOnRQ//799dJLLzE6APghwzC0fft2paenq1+/fnI6nWZHAioMykApBQYGauDAgYqIiNDUqVM50AjwI4Zh6OjRo5o3b56Sk5MVFxfHAUTAz1AGLoLdbtcDDzyg9PR0rV692uw4AErJ6/XqueeeU5UqVdSlSxeKAPArlIGLFBgYqDFjxmjNmjXatWuX2XEAlMKcOXMUHh6uxx57zOwoQIVEGbhINptNMTExuu6667R8+XJlZ2ebHQnAH1i1apXS09M1btw4s6MAFRZl4BIEBASoc+fOstvtWrBggVwul9mRAPyKYRjas2ePPvvsM40YMULh4eFmRwIqLMrAJQoKCtKDDz6oNWvWaMOGDRxoBFQwubm5WrBggbp166bGjRszTwD4A5SBy2Cz2fTqq69q2rRpzB8AKhC3262lS5cqNDRU3bp1owgAf4IycJlq1qyp+++/X7NmzdL+/fvNjgNYnmEYmjVrltasWaMhQ4ZwpghQCpQBH2jZsqWSkpI0Z84cnTlzxuw4gKXt2bNHS5cu1SOPPMK5A0ApUQZ8wOFwKCkpSdWqVdPixYvZoRAwSX5+vsaNG6eJEyeqSZMmZscB/AZlwEcCAwN1yy23aOfOnVq/fj2FAChneXl5mjBhgh544AFFR0ebHQfwK5QBH7HZbKpZs6ZSUlL01ltvaefOnWZHAiyjqKhIy5cvV506ddSxY0c5HA6zIwF+hTLgY4mJiRo9erQmTJig3Nxcs+MAlZ7X69XWrVuVnp6u3r17q2rVqmZHAvyOzWCBvM8ZhqGVK1dqxYoVevnll1nWBJShwsJC3XLLLXrjjTcUFxdndhzALzEyUEaSkpLUokULTZs2jRMOgTJSVFSkQYMGaezYsYqNjTU7DuC3KANlwGazKSQkRH379tWJEyf01VdfyePxmB0LqFTOnDmjiRMnavTo0br++usZgQMuA2WgDNWuXVu9evXSqlWrdPDgQbYsBnykqKhIixcvVnh4uFJSUigCwGWiDJSxli1bqlOnTnr66acZHQB8wDAMbdiwQRkZGbr55psVEhJidiTA71EGykH37t3Vp08fPfHEE4wOAJfBMAxlZmZq4cKFuuWWW1SjRg2zIwGVAmWgHAQGBurmm29W/fr19e6778rtdpsdCfBLp0+f1gsvvKCbbrpJjRs3NjsOUGlQBspJQECAhg0bplOnTumzzz6jEAAX6cyZM3rkkUcUFRWlLl26ME8A8CH2GShnBw4c0IwZMzRo0CAlJCTwCw0oBZfLpXfeeUc2m01333237HY+xwC+xE9UOWvQoIF69eqlGTNmsEMhUEqLFy+Wy+XSyJEjKQJAGeCnygTt2rVTQkKC7rvvPlYYAH/AMAxt3rxZO3fu1M0336zQ0FCzIwGVEmXABA6HQyNGjFDTpk319NNPq6ioyOxIQIVjGIaOHDmijz76SCkpKYqJieG2GlBGKAMmsdvtevzxx1W1alV9/PHHKi4uNjsSUKEcP35cb7zxhjp27Ki2bdtSBIAyRBkwUUBAgO68804dOHBA69atYw8C4D8KCwv14osv6sorr1Tfvn3NjgNUepQBk4WHh2vw4MFau3atfvjhB7PjABXC66+/rlatWmnUqFFmRwEsgTJQAdSvX1833nijJk+erNOnT5sdBzCN1+vVhx9+qJCQEA0aNIiVA0A54SetArDb7br66qt1zTXX6JFHHtHJkyfNjgSUO6/Xq02bNik9PV0DBw5U1apVmScAlBPKQAVhs9k0dOhQNWnSRBMnTmQPAliKYRjau3evli9frj59+rByAChnlIEK5qGHHlKDBg00b948lhzCMg4dOqTx48frhhtuUOvWrc2OA1gOZaCCCQwM1JAhQ5Sfn6/ly5ezwgCVXkFBgR577DGNHDlS1157rdlxAEuiDFRAYWFhGjFihL744gtt27aNQoBKy+1269lnn9Xtt9+ujh07mh0HsCzKQAUVERGh++67T5MmTVJ6errZcQCfKyoq0uzZs9W0aVNOIQRMRhmooGw2m+rXr6/hw4fr+eef1+bNm82OBPhMSUmJPvnkE+Xm5iolJUUhISGUAcBEAWYHwB/r3Lmz8vLy9MEHH6h69epq2LCh2ZGAy7Z69Wrt2LFDo0aNUo0aNcyOA1iezeCGdIXn8Xi0bt06ffPNN0pNTVVkZCSfouCXDMPQsmXLNGXKFE2aNEm1atUyOxIAcZvALzgcDnXt2lVxcXFasGCB8vPzmVQIv+P1erVlyxbNnj1bb7zxhmrWrGl2JAD/QRnwEw6H4/ySw/nz58vr9ZodCSg1wzB04MABffTRR/rrX/+q6OhoRreACoQy4Gcefvhhff/995oxY4bZUYBSO3HihGbOnKlevXqpWbNmZscB8CuUAT/05JNP6vDhw3r77bfNjgL8KbfbrX/84x+69tprlZiYaHYcABdAGfBDVatW1ZgxY1RUVKQPP/yQ+QOosAzD0H333ac+ffooOTmZWwNABUUZ8EM2m00REREaMmSI9uzZo88//1wej8fsWMAvFBQUaPTo0WrcuLG6d+8uh8NhdiQAv4OlhX7uwIED5+/FtmvXjk9eqBByc3M1b948Va9eXQMGDFBQUJDZkQD8AUYG/FyDBg10yy23aNGiRfriiy/MjgOooKBAixYtUmhoqHr27EkRAPwAZaASaNasmYYOHarJkydr8eLFZseBhXm9Xs2ePVtut1t9+/bVFVdcYXYkAKVAGagkmjdvrscff1ybNm3ipEOYwuv1atq0acrLy9OQIUNUvXp1syMBKCXOJqgkbDabEhISZBiGli5dquDgYDVu3Fh2O30PZa+wsFBvvfWWdu7cqUmTJikwMNDsSAAuAleKSsRms6lly5bq1q2bFi1apL179zJCgDJXUFCgZcuWqbCwUP/85z8pAoAfYjVBJfXVV19p1apVuuGGG3TNNdeYHQeVlMvl0ooVK3T8+HH179+fg4cAP8XIQCV17bXXql+/fho/frzS0tLMjoNKyDAMffzxxzp48KAGDBhAEQD8GCMDlZhhGNq2bZs++OADDR8+XE2bNmUfAvhESUmJ3n//fR06dEgPPvigwsLCzI4E4DJQBio5wzC0ZcsWrVy5UgMHDlSTJk2YVIjLkp+fr9dff105OTl65pln2EcAqAS4KlRyNptNbdu2VVJSkj7++GNt377d7EjwY3l5efr444/l9Xr18MMPUwSASoKRAQvZsmWLli1bprZt26p3795mx4GfcblcmjVrliSpb9++qlmzpsmJAPgK+wxYSOvWreV0OvXPf/5ThmGoT58+ZkeCH3n55ZcVHR2tAQMGsKEQUMkwMmAxhmFo9+7deueddzRgwAB17tyZOQT4Q0VFRXr22WeVkJCgQYMGcWsAqIQoAxZkGIYOHTqkd999Vz169FC3bt0oBLig06dP67nnnlNCQoKGDx/OhkJAJUUZsLDMzEy9/fbbatGihW6++Waz46ACMQxDx44d08yZMxUeHq5bbrlF1apVMzsWgDJCGbC4nJwcvffeeyoqKtK9997LenFIkg4ePKhXXnlFvXr1UlJSErcGgEqOMmBxhmGooKBAc+fO1eHDh/XAAw+oWrVqbE5kUYZhaMeOHRo/frwee+wxNWvWjPcCYAGUAcgwDHk8Hs2bN08HDx7U7bffrqioKC4CFlNSUqL58+dr6dKlevHFF1W7dm3eA4BFUAbwCwsWLND27ds1bNgwNW7c2Ow4KCfFxcVKS0vTkiVLdM8996hly5ZmRwJQjigD+I1169Zp9erV6tKli5KTk82OgzLmcrk0depUFRYWauDAgYqLizM7EoByRhnAb5y7bzx9+nS1bdtWN910k0JCQsyOhTJw/PhxvfDCC2rXrp369OmjiIgIsyMBMAFlABfk9Xp15MgRTZ06VZGRkRo9ejSFoBIxDEPr16/Xu+++qzvvvFMdOnRgxQBgYZQB/K5zEwsnT56srKwsjRs3TuHh4WbHwmUwDENer1ebNm3Syy+/rDvvvFM9evRgoiBgcZQBlMqiRYu0bt063XbbbUpISFBwcLDZkXAJcnNzlZaWpq+++kqpqamKj483OxKACoAygFI592lyzpw5uvrqqzVgwABVqVLF7Fi4CHv37tWiRYvkcrl0xx13KDIy0uxIACoIygBKzev16uDBg1q8eLHy8vI0btw45hH4ibS0NC1fvlw9evRQr169OIsCwC/wGwGlZrfb1aBBA40ePVoJCQnq37+/9u3bJ6/Xa3Y0/A6Xy6VZs2ZpwoQJSklJ0Q033EARAPAbjAzgkpxbbfCXv/xF/fr104033qiwsDAmolUQXq9X2dnZeuutt1RcXKwnnnhCoaGh/PsAuCDKAC5LZmamXnrpJUVFRemmm25SkyZN+ORpslOnTmnjxo365JNP1KlTJw0aNMjsSAAqOMoALltWVpZWrlypffv2cRyyiQzDUFZWliZNmqT8/HyNGjVK8fHxlDMAf4oyAJ/54YcftHTpUqWnp+svf/kLZxuUs08++URTpkxR7969dcMNN6h27dpmRwLgJygD8JlzxyF/9dVXmjhxosaOHavu3bsrICCAe9VlxOv16syZM5o4caJycnL0+OOPKzw8nN0EAVwUygB86tzbadOmTXr++efVpk0bjRgxQnXr1uUC5UOGYejs2bP67LPP9M4772jw4MG69dZb+X8M4JJQBlCm5s+fr/Xr16tDhw7q2rWrYmJizI7k9zwejzZv3qx169Zp3759GjZsmDp16mR2LAB+jDKAMmUYhg4dOqQlS5Zo79696tSpk/r168d2xpfowIEDmjt3rlwul1q3bq3OnTtz0iCAy0YZQLkoKCjQjh07tGrVKqWnp2vs2LFq166d2bH8RmFhoaZOnaqNGzeqf//+uuaaa1SnTh3mYgDwCcoAyo1hGCoqKtLu3bs1YcIE1a1bV3fccYdiY2OZZHgBXq9XLpdLGzZs0IQJE1S/fn3dddddatKkiQICAsyOB6ASoQyg3J0rBQsWLNCMGTPUt29f9ezZU7GxsQoNDTU7num8Xq9OnDihXbt2aenSpcrPz9f999+vZs2aSRKlCYDPUQZgqu3bt+vTTz/V8ePHFRcXpzZt2qhly5aWPRExMzNTGzZs0Jdffqn8/HwNGjRInTt3Zo4FgDJFGUCFcOjQIX399dfatWuXTp06pZ49e6pXr15yOBxmRysXmZmZWrhwoTIyMlS7dm01adJEbdu2ZeMgAOWCMoAKw+PxKDs7W2lpadq0aZN27NihUaNGqW/fvqpWrZqkyjFE/vMfuf379+u1117TgQMHlJKSosTERMXGxqpq1aomJgRgNZQBVDgej0dut1snTpzQ22+/rY0bN6pp06a67777VLt2bYWEhPjlBLpzcyWKioq0detWvffeezp8+LB69eqlYcOGqUaNGkykBGAKygAqvMzMTL3++uv65ptv1KpVK7Vv317NmjVTeHi46tatW6GLwbktmo8dO6Zjx47p008/1caNGxUbG6uhQ4eqU6dOFTo/AGugDMBvuN1ubdy4UV9//bWysrKUlZWl5s2bq3nz5mrcuLHi4uLkdDrNjilJOnnypLZv364DBw4oOztbeXl5ys/PV7169dStWze1adPG7IgAcB5lAH7n3NK7LVu2nC8Fx44dU15enpxOp5KSktSqVSvFxMSUy6duwzDkdru1detWpaena+vWrSooKFB+fr4aN26stm3bqlGjRmrQoIGCg4O5DQCgwqEMwK+dO7DnzJkzOnHihObOnav8/Hzt379fubm5io+P1xVXXKEePXqoadOmql279i+W6ZX2wvzzHxOXy6Vdu3Zpz5492rVrl3bu3Kk9e/YoNjb2/Kf+evXqyel0qnr16kwGBFDhUQZQaRiGIY/HI8Mw5PV6lZubq++++06zZ8+Wy+XS0aNHdfLkSYWHh6u4uFgJCQmqWbOmqlSpoipVquj48eOKjo5WUFCQ3G633G63du7cqaCgIBUVFSkrK0vHjx9Xbm6u6tWrp/bt2yshIUEJCQlq0qSJgoKCZLfbZbfbZbPZGAEA4DcoA7AUl8ul48eP6+uvv1ZAQIA8Ho/y8/OVl5en3bt3KyIiQtWqVVNgYKACAgJ07NgxNWjQQA0bNlRUVJSioqJUvXr18xd8AKgMKAMAAFic3ewAAADAXJQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALC4/weJ1hqyIXcPWwAAAABJRU5ErkJggg==",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "from IPython.display import display, clear_output\n",
-    "import time\n",
-    "\n",
-    "# Initialize environment\n",
-    "state, _ = env.reset()\n",
-    "state = discretize_state(state)\n",
-    "done = False\n",
-    "total_reward = 0\n",
-    "\n",
-    "# Test the policy\n",
-    "while not done:\n",
-    "    action = np.argmax(Q_table[state])\n",
-    "    next_state, reward, done, truncated, _ = env.step(action)\n",
-    "    state = discretize_state(next_state)\n",
-    "    total_reward += reward\n",
-    "\n",
-    "    # Render frame\n",
-    "    frame = env.render()\n",
-    "    plt.imshow(frame)\n",
-    "    plt.axis(\"off\")\n",
-    "    display(plt.gcf())\n",
-    "    clear_output(wait=True)\n",
-    "    time.sleep(0.3)  # Add delay for smoother animation\n",
-    "\n",
-    "print(\"Total reward:\", total_reward)\n",
-    "env.close()\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Testing Episode 1\n"
-     ]
-    },
-    {
-     "ename": "TypeError",
-     "evalue": "MountainCarEnv.render() got an unexpected keyword argument 'mode'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[8], line 38\u001b[0m\n\u001b[0;32m     35\u001b[0m     total_reward \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m reward\n\u001b[0;32m     37\u001b[0m     \u001b[38;5;66;03m# Native rendering\u001b[39;00m\n\u001b[1;32m---> 38\u001b[0m     \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhuman\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m     39\u001b[0m     time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m0.03\u001b[39m)  \u001b[38;5;66;03m# Delay for smooth rendering\u001b[39;00m\n\u001b[0;32m     41\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTotal reward for Episode \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepisode\u001b[38;5;250m \u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtotal_reward\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
-      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\gym\\core.py:329\u001b[0m, in \u001b[0;36mWrapper.render\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    325\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrender\u001b[39m(\n\u001b[0;32m    326\u001b[0m     \u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[0;32m    327\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Optional[Union[RenderFrame, List[RenderFrame]]]:\n\u001b[0;32m    328\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Renders the environment.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 329\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\gym\\wrappers\\order_enforcing.py:51\u001b[0m, in \u001b[0;36mOrderEnforcing.render\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_disable_render_order_enforcing \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_has_reset:\n\u001b[0;32m     47\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ResetNeeded(\n\u001b[0;32m     48\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot call `env.render()` before calling `env.reset()`, if this is a intended action, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     49\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mset `disable_render_order_enforcing=True` on the OrderEnforcer wrapper.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     50\u001b[0m     )\n\u001b[1;32m---> 51\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\gym\\wrappers\\env_checker.py:55\u001b[0m, in \u001b[0;36mPassiveEnvChecker.render\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m     53\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m env_render_passive_checker(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m     54\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 55\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
-      "\u001b[1;31mTypeError\u001b[0m: MountainCarEnv.render() got an unexpected keyword argument 'mode'"
-     ]
-    }
-   ],
-   "source": [
-    "import pickle\n",
-    "import time\n",
-    "\n",
-    "# Load the saved model\n",
-    "with open(\"q_learning_model.pkl\", \"rb\") as f:\n",
-    "    model_data = pickle.load(f)\n",
-    "\n",
-    "# Extract the Q-table and state bins\n",
-    "Q_table = model_data[\"Q_table\"]\n",
-    "state_bins = model_data[\"state_bins\"]\n",
-    "\n",
-    "# Helper to discretize state space\n",
-    "def discretize_state(state):\n",
-    "    state_low = env.observation_space.low\n",
-    "    state_high = env.observation_space.high\n",
-    "    bins = [np.linspace(state_low[i], state_high[i], state_bins[i]) for i in range(len(state))]\n",
-    "    state_indices = [np.digitize(state[i], bins[i]) - 1 for i in range(len(state))]\n",
-    "    return tuple(state_indices)\n",
-    "\n",
-    "# Number of episodes to test\n",
-    "test_episodes = 10\n",
-    "\n",
-    "for episode in range(test_episodes):\n",
-    "    state, _ = env.reset()\n",
-    "    state = discretize_state(state)\n",
-    "    done = False\n",
-    "    total_reward = 0\n",
-    "\n",
-    "    print(f\"Testing Episode {episode + 1}\")\n",
-    "\n",
-    "    while not done:\n",
-    "        action = np.argmax(Q_table[state])  # Choose best action\n",
-    "        next_state, reward, done, truncated, _ = env.step(action)\n",
-    "        state = discretize_state(next_state)\n",
-    "        total_reward += reward\n",
-    "\n",
-    "        # Native rendering\n",
-    "        env.render(mode=\"human\")\n",
-    "        time.sleep(0.03)  # Delay for smooth rendering\n",
-    "\n",
-    "    print(f\"Total reward for Episode {episode + 1}: {total_reward}\")\n",
-    "\n",
-    "env.close()\n"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}