# Install dependencies:
# pip install (none; uses Python standard library)
#
# Hybrid agent: Q-learning learns a route in a 1-D environment.
# Reactive component checks whether an obstacle is detected.

import random


class HybridQLearningAgent:
    def __init__(self, goal=4):
        self.goal = goal
        self.q_table = {}
        self.actions = [-1, 1]
        self.alpha = 0.8
        self.gamma = 0.9

    def q_value(self, state, action):
        return self.q_table.get((state, action), 0.0)

    def choose_action(self, state, epsilon=0):
        if epsilon > 0 and random.random() < epsilon:
            return random.choice(self.actions)
        values = [self.q_value(state, action) for action in self.actions]
        return self.actions[values.index(max(values))]

    def train(self, episodes=100):
        for _ in range(episodes):
            state = 0
            while state != self.goal:
                action = self.choose_action(state, epsilon=0.2)
                next_state = max(0, min(self.goal, state + action))
                reward = 10 if next_state == self.goal else -1
                old_q = self.q_value(state, action)
                best_next = max(self.q_value(next_state, a)
                                for a in self.actions)
                self.q_table[(state, action)] = (
                    old_q + self.alpha *
                    (reward + self.gamma * best_next - old_q)
                )
                state = next_state

    def react(self, obstacle):
        if obstacle:
            return "Avoid obstacle"
        return "Continue with learned action"

    def deliberate(self):
        state = 0
        plan = [state]
        for _ in range(self.goal + 2):
            if state == self.goal:
                break
            action = self.choose_action(state)
            next_state = max(0, min(self.goal, state + action))
            # Avoid a repeated-state loop if Q-values are tied.
            if next_state == state:
                action = 1
                next_state = min(self.goal, state + action)
            state = next_state
            plan.append(state)
        return plan


if __name__ == "__main__":
    random.seed(7)
    agent = HybridQLearningAgent()
    agent.train()

    print("HYBRID AGENT USING Q-LEARNING")
    print("Learned Q-values:")
    for state in range(4):
        print("State", state,
              "Left:", round(agent.q_value(state, -1), 2),
              "Right:", round(agent.q_value(state, 1), 2))

    print("\nReactive component:")
    obstacle_detected = True
    print("Obstacle detected:", obstacle_detected)
    print("Action:", agent.react(obstacle_detected))

    print("\nDeliberative plan using Q-learning:")
    print(agent.deliberate())
    print("\nNote: Q-values can vary because training uses random exploration.")