diff --git a/src/human_voice_ai/policy/rl_policy.py b/src/human_voice_ai/policy/rl_policy.py index aebabb3..19a540c 100644 --- a/src/human_voice_ai/policy/rl_policy.py +++ b/src/human_voice_ai/policy/rl_policy.py @@ -130,14 +130,18 @@ def select_action(self, state: np.ndarray, training: bool = True) -> int: if training and random.random() < self.epsilon: return random.randrange(self.num_actions) - with torch.no_grad(): - # Convert state to tensor and add batch dimension if needed - if not isinstance(state, torch.Tensor): - state = torch.FloatTensor(state).unsqueeze(0).to(self.device) - - # Get Q-values and select action with highest Q-value - q_values = self.policy_net(state) - return q_values.argmax().item() + # Action scoring must not apply training-time dropout. Restore the + # caller's mode so subsequent optimizer updates still train normally. + was_training = self.policy_net.training + self.policy_net.eval() + try: + with torch.no_grad(): + if not isinstance(state, torch.Tensor): + state = torch.FloatTensor(state).unsqueeze(0).to(self.device) + q_values = self.policy_net(state) + return q_values.argmax().item() + finally: + self.policy_net.train(was_training) def store_transition(self, state, action, reward, next_state, done): """Store a transition in the replay buffer.""" diff --git a/src/human_voice_ai/rl/agent.py b/src/human_voice_ai/rl/agent.py index a18c0a8..a4040b6 100644 --- a/src/human_voice_ai/rl/agent.py +++ b/src/human_voice_ai/rl/agent.py @@ -229,7 +229,7 @@ def update(self, batch: List[Transition]) -> Dict[str, float]: # Calculate advantages and returns with torch.no_grad(): _, values = self.policy(states) - next_values = torch.cat([values[1:], torch.zeros(1)]) + next_values = torch.cat([values[1:], values.new_zeros(1)]) next_values[dones.bool()] = 0 deltas = rewards + self.gamma * next_values - values @@ -237,10 +237,10 @@ def update(self, batch: List[Transition]) -> Dict[str, float]: advantages[-1] = deltas[-1] for t in reversed(range(len(deltas) - 1)): - advantages[t] = deltas[t] + self.gamma * advantages[t + 1] + advantages[t] = deltas[t] + self.gamma * (1 - dones[t]) * advantages[t + 1] returns = advantages + values - advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8) + advantages = (advantages - advantages.mean()) / (advantages.std(unbiased=False) + 1e-8) # PPO update for _ in range(4): # Number of PPO epochs @@ -281,9 +281,11 @@ def update(self, batch: List[Transition]) -> Dict[str, float]: value_loss = F.mse_loss(values, returns) # Calculate explained variance - y_true = returns.numpy() - y_pred = values.numpy() - explained_var = 1 - np.var(y_true - y_pred) / np.var(y_true) + y_true = returns.cpu().numpy() + y_pred = values.cpu().numpy() + target_variance = np.var(y_true) + explained_var = (1 - np.var(y_true - y_pred) / target_variance + if target_variance > 0 else 0.0) return { "loss": loss.item(), diff --git a/tests/unit/policy/test_audio_qa_regressions.py b/tests/unit/policy/test_audio_qa_regressions.py new file mode 100644 index 0000000..8653fd6 --- /dev/null +++ b/tests/unit/policy/test_audio_qa_regressions.py @@ -0,0 +1,62 @@ +"""Numeric and mode regressions use real networks and optimizer updates.""" +import numpy as np +import pytest +import torch +from human_voice_ai.policy.rl_policy import EmotionPolicy +from human_voice_ai.rl.agent import PolicyNetwork, PPOTrainer, Transition + + +def test_greedy_actions_disable_dropout_and_restore_model_mode(): + policy = EmotionPolicy(4, 2, hidden_dim=8, epsilon_start=0, device="cpu") + layers = list(policy.policy_net.network) + with torch.no_grad(): + for layer in layers: + if isinstance(layer, torch.nn.Linear): + layer.weight.zero_() + layer.bias.zero_() + layers[0].weight[0, 0] = 1 + layers[3].weight[0, 0] = 1 + layers[6].weight[0, 0] = 1 + layers[6].bias[1] = .1 + state = np.ones(4, dtype=np.float32) + for training_mode in (True, False): + policy.policy_net.train(training_mode) + assert {policy.select_action(state, training=False) for _ in range(50)} == {0} + assert policy.policy_net.training is training_mode + + +def ppo_fixture(): + policy = PolicyNetwork(state_dim=4, hidden_dim=8, num_actions=5) + with torch.no_grad(): + for parameter in policy.parameters(): + parameter.zero_() + state = {"audio": torch.zeros(4), "text": torch.zeros(4), + "emotion_history": torch.zeros(5), "turn_count": torch.zeros(1)} + return policy, state + + +def test_singleton_ppo_update_keeps_metrics_parameters_and_gradients_finite(): + policy, state = ppo_fixture() + before = [value.detach().clone() for value in policy.parameters()] + metrics = PPOTrainer(policy).update([Transition(state, 0, 1., state, True)]) + assert all(np.isfinite(value) for value in metrics.values()) + assert all(torch.isfinite(value).all() for value in policy.parameters()) + assert all(value.grad is None or torch.isfinite(value.grad).all() for value in policy.parameters()) + assert any(not torch.equal(old, new) for old, new in zip(before, policy.parameters())) + + +def test_ppo_returns_do_not_cross_terminal_episode_boundaries(monkeypatch): + policy, state = ppo_fixture() + targets = [] + real_loss = torch.nn.functional.mse_loss + def capture(prediction, target, *args, **kwargs): + assert prediction.shape == target.shape == (2,) + targets.append(target.detach().clone()) + return real_loss(prediction, target, *args, **kwargs) + monkeypatch.setattr(torch.nn.functional, "mse_loss", capture) + metrics = PPOTrainer(policy).update([ + Transition(state, 0, 1., state, True), + Transition(state, 0, 100., state, True), + ]) + assert targets[0].tolist() == [1., 100.] + assert all(np.isfinite(value) for value in metrics.values())