Q学習：インタラクティブ強化学習の基礎

import random
import numpy as np

class QLearningAgent:
    def __init__(self, grid_size=10, alpha=0.1, gamma=0.9, epsilon=0.1):
        self.grid_size = grid_size
        self.alpha = alpha      # 学習率
        self.gamma = gamma      # 割引係数
        self.epsilon = epsilon  # 探索率
        self.q_table = {}       # 各(状態, 行動)ペアのQ値
        self.actions = ['上', '右', '下', '左']
        
        # Q-テーブルを初期化
        for x in range(grid_size):
            for y in range(grid_size):
                for action in self.actions:
                    self.q_table[(x, y, action)] = 0.0
    
    def choose_action(self, x, y):
        """ε-greedyポリシーを使用して行動を選択"""
        if random.random() < self.epsilon:
            return random.choice(self.actions)  # 探索
        
        # 利用：最高のQ値を持つ行動を選択
        q_values = [self.q_table.get((x, y, action), 0) for action in self.actions]
        max_q = max(q_values)
        best_actions = [action for action, q in zip(self.actions, q_values) if q == max_q]
        return random.choice(best_actions)
    
    def update_q_value(self, state, action, reward, next_state):
        """Q学習の式を使用してQ値を更新"""
        x, y = state
        next_x, next_y = next_state
        
        current_q = self.q_table.get((x, y, action), 0)
        next_q_values = [self.q_table.get((next_x, next_y, a), 0) for a in self.actions]
        max_next_q = max(next_q_values) if next_q_values else 0
        
        # Q(s,a) = Q(s,a) + α[r + γ max Q(s',a') - Q(s,a)]
        new_q = current_q + self.alpha * (reward + self.gamma * max_next_q - current_q)
        self.q_table[(x, y, action)] = new_q
    
    def train(self, walls, goal, episodes=1000):
        """指定されたエピソード数でエージェントを訓練"""
        for episode in range(episodes):
            x, y = 0, 0  # 開始位置
            
            for step in range(200):  # エピソードあたりの最大ステップ数
                action = self.choose_action(x, y)
                
                # 新しい位置と報酬を計算
                moves = {'上': (0, -1), '右': (1, 0), '下': (0, 1), '左': (-1, 0)}
                dx, dy = moves[action]
                new_x, new_y = x + dx, y + dy
                
                if (new_x < 0 or new_x >= self.grid_size or 
                    new_y < 0 or new_y >= self.grid_size):
                    reward = -30  # グリッドから落下
                    new_x, new_y = x, y
                elif (new_x, new_y) in walls:
                    reward = -30  # 壁に衝突
                    new_x, new_y = x, y
                elif (new_x, new_y) == goal:
                    reward = 30   # ゴールに到達
                else:
                    reward = -1   # 通常のステップ
                
                # Q値を更新して移動
                self.update_q_value((x, y), action, reward, (new_x, new_y))
                x, y = new_x, new_y
                
                if (x, y) == goal or reward == -30:
                    break

# 使用例
walls = [(1,1), (2,1), (3,1)]
goal = (9, 8)
agent = QLearningAgent()
agent.train(walls, goal, episodes=500)