Q-Learning: Interactive Reinforcement Learning Foundation

import random
import numpy as np

class QLearningAgent:
    def __init__(self, grid_size=10, alpha=0.1, gamma=0.9, epsilon=0.1):
        self.grid_size = grid_size
        self.alpha = alpha      # Learning rate
        self.gamma = gamma      # Discount factor  
        self.epsilon = epsilon  # Exploration rate
        self.q_table = {}       # Q-values for each (state, action) pair
        self.actions = ['up', 'right', 'down', 'left']
        
        # Initialize Q-table
        for x in range(grid_size):
            for y in range(grid_size):
                for action in self.actions:
                    self.q_table[(x, y, action)] = 0.0
    
    def choose_action(self, x, y):
        """Choose action using epsilon-greedy policy"""
        if random.random() < self.epsilon:
            return random.choice(self.actions)  # Explore
        
        # Exploit: choose action with highest Q-value
        q_values = [self.q_table.get((x, y, action), 0) for action in self.actions]
        max_q = max(q_values)
        best_actions = [action for action, q in zip(self.actions, q_values) if q == max_q]
        return random.choice(best_actions)
    
    def update_q_value(self, state, action, reward, next_state):
        """Update Q-value using Q-learning formula"""
        x, y = state
        next_x, next_y = next_state
        
        current_q = self.q_table.get((x, y, action), 0)
        next_q_values = [self.q_table.get((next_x, next_y, a), 0) for a in self.actions]
        max_next_q = max(next_q_values) if next_q_values else 0
        
        # Q(s,a) = Q(s,a) + α[r + γ max Q(s',a') - Q(s,a)]
        new_q = current_q + self.alpha * (reward + self.gamma * max_next_q - current_q)
        self.q_table[(x, y, action)] = new_q
    
    def train(self, walls, goal, episodes=1000):
        """Train the agent for specified number of episodes"""
        for episode in range(episodes):
            x, y = 0, 0  # Start position
            
            for step in range(200):  # Max steps per episode
                action = self.choose_action(x, y)
                
                # Calculate new position and reward
                moves = {'up': (0, -1), 'right': (1, 0), 'down': (0, 1), 'left': (-1, 0)}
                dx, dy = moves[action]
                new_x, new_y = x + dx, y + dy
                
                if (new_x < 0 or new_x >= self.grid_size or 
                    new_y < 0 or new_y >= self.grid_size):
                    reward = -30  # Fell off grid
                    new_x, new_y = x, y
                elif (new_x, new_y) in walls:
                    reward = -30  # Hit wall
                    new_x, new_y = x, y
                elif (new_x, new_y) == goal:
                    reward = 30   # Reached goal
                else:
                    reward = -1   # Normal step
                
                # Update Q-value and move
                self.update_q_value((x, y), action, reward, (new_x, new_y))
                x, y = new_x, new_y
                
                if (x, y) == goal or reward == -30:
                    break

# Usage
walls = [(1,1), (2,1), (3,1)]
goal = (9, 8)
agent = QLearningAgent()
agent.train(walls, goal, episodes=500)