{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1208d732cfadaae1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2026-06-10T09:05:23.848013Z",
     "start_time": "2026-06-10T09:05:23.658956Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import pickle\n",
    "\n",
    "# ------------------------------\n",
    "# Tic-Tac-Toe Environment\n",
    "# ------------------------------\n",
    "class TicTacToeEnv:\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.board = np.zeros((3, 3), dtype=int)\n",
    "        self.current_player = 1  # X starts\n",
    "        self.done = False\n",
    "        self.winner = None\n",
    "        return self.get_state()\n",
    "\n",
    "    def get_state(self):\n",
    "        return tuple(self.board.flatten())  # immutable for Q-table keys\n",
    "\n",
    "    def available_actions(self):\n",
    "        return [i for i in range(9) if self.board.flatten()[i] == 0]\n",
    "\n",
    "    def step(self, action):\n",
    "        if self.done:\n",
    "            raise ValueError(\"Game is over. Reset first.\")\n",
    "        row, col = divmod(action, 3)\n",
    "        if self.board[row, col] != 0:\n",
    "            raise ValueError(\"Invalid move\")\n",
    "        self.board[row, col] = self.current_player\n",
    "\n",
    "        self.winner = self.check_winner()\n",
    "        self.done = self.winner is not None or len(self.available_actions()) == 0\n",
    "\n",
    "        if self.done:\n",
    "            if self.winner == 1:\n",
    "                reward = 1\n",
    "            elif self.winner == -1:\n",
    "                reward = -1\n",
    "            else:\n",
    "                reward = 0\n",
    "        else:\n",
    "            reward = 0\n",
    "\n",
    "        self.current_player *= -1\n",
    "        return self.get_state(), reward, self.done\n",
    "\n",
    "    def check_winner(self):\n",
    "        lines = []\n",
    "        lines.extend(self.board)          # rows\n",
    "        lines.extend(self.board.T)        # columns\n",
    "        lines.append(np.diag(self.board)) # main diag\n",
    "        lines.append(np.diag(np.fliplr(self.board))) # anti diag\n",
    "\n",
    "        for line in lines:\n",
    "            if np.all(line == 1):\n",
    "                return 1\n",
    "            if np.all(line == -1):\n",
    "                return -1\n",
    "        return None\n",
    "\n",
    "    def render(self):\n",
    "        symbols = {1: 'X', -1: 'O', 0: ' '}\n",
    "        for row in self.board:\n",
    "            print('|'.join(symbols[x] for x in row))\n",
    "            print('-'*5)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dd3d81bd-14fd-422a-bd5e-4c436c15a6d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " | | \n",
      "-----\n",
      " | | \n",
      "-----\n",
      " | | \n",
      "-----\n",
      "\n",
      "X| | \n",
      "-----\n",
      " | | \n",
      "-----\n",
      " | | \n",
      "-----\n",
      "\n",
      "Reward: 0\n",
      "Done: False\n",
      "Winner: None\n",
      "X| | \n",
      "-----\n",
      " |O| \n",
      "-----\n",
      " | | \n",
      "-----\n",
      "\n",
      "Reward: 0\n",
      "Done: False\n",
      "Winner: None\n",
      "X|X| \n",
      "-----\n",
      " |O| \n",
      "-----\n",
      " | | \n",
      "-----\n",
      "\n",
      "Reward: 0\n",
      "Done: False\n",
      "Winner: None\n",
      "X|X| \n",
      "-----\n",
      " |O| \n",
      "-----\n",
      " | |O\n",
      "-----\n",
      "\n",
      "Reward: 0\n",
      "Done: False\n",
      "Winner: None\n",
      "X|X|X\n",
      "-----\n",
      " |O| \n",
      "-----\n",
      " | |O\n",
      "-----\n",
      "\n",
      "Reward: 1\n",
      "Done: True\n",
      "Winner: 1\n"
     ]
    }
   ],
   "source": [
    "env = TicTacToeEnv()\n",
    "state = env.reset()\n",
    "\n",
    "env.render()\n",
    "\n",
    "state, reward, done = env.step(0)  # X\n",
    "env.render()\n",
    "print(\"Reward:\", reward)\n",
    "print(\"Done:\", done)\n",
    "print(\"Winner:\", env.winner)\n",
    "state, reward, done = env.step(4)  # O\n",
    "env.render()\n",
    "print(\"Reward:\", reward)\n",
    "print(\"Done:\", done)\n",
    "print(\"Winner:\", env.winner)\n",
    "state, reward, done = env.step(1)  # X\n",
    "env.render()\n",
    "print(\"Reward:\", reward)\n",
    "print(\"Done:\", done)\n",
    "print(\"Winner:\", env.winner)\n",
    "state, reward, done = env.step(8)  # O\n",
    "env.render()\n",
    "print(\"Reward:\", reward)\n",
    "print(\"Done:\", done)\n",
    "print(\"Winner:\", env.winner)\n",
    "state, reward, done = env.step(2)  # X wins\n",
    "\n",
    "env.render()\n",
    "print(\"Reward:\", reward)\n",
    "print(\"Done:\", done)\n",
    "print(\"Winner:\", env.winner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "49294b9c96c5ca5b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2026-06-10T09:05:25.875116Z",
     "start_time": "2026-06-10T09:05:25.846950Z"
    }
   },
   "outputs": [],
   "source": [
    "# ------------------------------\n",
    "# Q-learning Training\n",
    "# ------------------------------\n",
    "def train_q_learning(env, episodes=50000, alpha=0.5, gamma=0.9, epsilon=0.1):\n",
    "    q_table = {}  # key: (state, action) -> value: Q\n",
    "\n",
    "    return q_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ff802e500afa1db8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2026-06-10T09:11:22.062493Z",
     "start_time": "2026-06-10T09:05:28.827725Z"
    }
   },
   "outputs": [],
   "source": [
    "env = TicTacToeEnv()\n",
    "q_table = train_q_learning(env, episodes=500000, alpha=0.5, gamma=0.9, epsilon=0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fbe616a25981fe8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2026-06-10T10:48:03.703882Z",
     "start_time": "2026-06-10T10:48:03.444624Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8e4f1613338f9c0a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2026-06-10T10:48:05.580901Z",
     "start_time": "2026-06-10T10:48:05.210308Z"
    }
   },
   "outputs": [],
   "source": [
    "# Save Q-table for later use\n",
    "with open(\"tictactoe_qtable.pkl\", \"wb\") as f:\n",
    "    pickle.dump(q_table, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b406df62f24735be",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2026-06-10T10:48:06.591334Z",
     "start_time": "2026-06-10T10:48:06.568256Z"
    }
   },
   "outputs": [],
   "source": [
    "# ------------------------------\n",
    "# Human vs Agent\n",
    "# ------------------------------\n",
    "def play_human_vs_agent(env, q_table):\n",
    "    state = env.reset()\n",
    "    human_player = int(input(\"Do you want to play as X (1) or O (-1)? \"))\n",
    "    env.current_player = 1  # X always starts\n",
    "    done = False\n",
    "    env.render()\n",
    "\n",
    "    while not done:\n",
    "        if env.current_player == human_player:\n",
    "            actions = env.available_actions()\n",
    "            print(\"Available moves:\", actions)\n",
    "            move = int(input(\"Enter your move (0-8): \"))\n",
    "            while move not in actions:\n",
    "                move = int(input(\"Invalid move. Enter again (0-8): \"))\n",
    "            state, reward, done = env.step(move)\n",
    "        else:\n",
    "            actions = env.available_actions()\n",
    "            q_values = [q_table.get((state, a), 0) for a in actions]\n",
    "            print(f\"Q-values: {q_values}\")\n",
    "            max_q = max(q_values)\n",
    "            max_actions = [a for a, q in zip(actions, q_values) if q == max_q]\n",
    "            print(f\"Max - Actions: {max_actions}\")\n",
    "            move = np.random.choice(max_actions)\n",
    "            state, reward, done = env.step(move)\n",
    "            print(f\"Agent plays: {move}\")\n",
    "\n",
    "        env.render()\n",
    "\n",
    "    if (reward == 1 or reward==-1):\n",
    "        if reward == -human_player:\n",
    "            print(\"Agent wins!\")\n",
    "        else:\n",
    "            print(\"You win!\")\n",
    "\n",
    "    else:\n",
    "        print(\"It's a draw!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b71a94499e87e2a3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2026-06-10T10:48:44.325995Z",
     "start_time": "2026-06-10T10:48:08.221938Z"
    }
   },
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Do you want to play as X (1) or O (-1)?  1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " | | \n",
      "-----\n",
      " | | \n",
      "-----\n",
      " | | \n",
      "-----\n",
      "\n",
      "Available moves: [0, 1, 2, 3, 4, 5, 6, 7, 8]\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Enter your move (0-8):  1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " |X| \n",
      "-----\n",
      " | | \n",
      "-----\n",
      " | | \n",
      "-----\n",
      "\n",
      "Q-values: [0, 0, 0, 0, 0, 0, 0, 0]\n",
      "Max - Actions: [0, 2, 3, 4, 5, 6, 7, 8]\n",
      "Agent plays: 0\n",
      "O|X| \n",
      "-----\n",
      " | | \n",
      "-----\n",
      " | | \n",
      "-----\n",
      "\n",
      "Available moves: [2, 3, 4, 5, 6, 7, 8]\n"
     ]
    }
   ],
   "source": [
    "play_human_vs_agent(env, q_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3fedd188fd915f8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}