Fix dropout inference and terminal/singleton PPO updates

src/human_voice_ai/policy/rl_policy.py

@@ -130,14 +130,18 @@ def select_action(self, state: np.ndarray, training: bool = True) -> int:
         if training and random.random() < self.epsilon:
             return random.randrange(self.num_actions)
 
-        with torch.no_grad():
-            # Convert state to tensor and add batch dimension if needed
-            if not isinstance(state, torch.Tensor):
-                state = torch.FloatTensor(state).unsqueeze(0).to(self.device)
-
-            # Get Q-values and select action with highest Q-value
-            q_values = self.policy_net(state)
-            return q_values.argmax().item()
+        # Action scoring must not apply training-time dropout. Restore the
+        # caller's mode so subsequent optimizer updates still train normally.
+        was_training = self.policy_net.training
+        self.policy_net.eval()
+        try:
+            with torch.no_grad():
+                if not isinstance(state, torch.Tensor):
+                    state = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+                q_values = self.policy_net(state)
+                return q_values.argmax().item()
+        finally:
+            self.policy_net.train(was_training)
 
     def store_transition(self, state, action, reward, next_state, done):
         """Store a transition in the replay buffer."""

src/human_voice_ai/rl/agent.py

@@ -229,18 +229,18 @@ def update(self, batch: List[Transition]) -> Dict[str, float]:
         # Calculate advantages and returns
         with torch.no_grad():
             _, values = self.policy(states)
-            next_values = torch.cat([values[1:], torch.zeros(1)])
+            next_values = torch.cat([values[1:], values.new_zeros(1)])
             next_values[dones.bool()] = 0
 
             deltas = rewards + self.gamma * next_values - values
             advantages = torch.zeros_like(deltas)
             advantages[-1] = deltas[-1]
 
             for t in reversed(range(len(deltas) - 1)):
-                advantages[t] = deltas[t] + self.gamma * advantages[t + 1]
+                advantages[t] = deltas[t] + self.gamma * (1 - dones[t]) * advantages[t + 1]
 
             returns = advantages + values
-            advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
+            advantages = (advantages - advantages.mean()) / (advantages.std(unbiased=False) + 1e-8)
 
         # PPO update
         for _ in range(4):  # Number of PPO epochs
@@ -281,9 +281,11 @@ def update(self, batch: List[Transition]) -> Dict[str, float]:
             value_loss = F.mse_loss(values, returns)
 
             # Calculate explained variance
-            y_true = returns.numpy()
-            y_pred = values.numpy()
-            explained_var = 1 - np.var(y_true - y_pred) / np.var(y_true)
+            y_true = returns.cpu().numpy()
+            y_pred = values.cpu().numpy()
+            target_variance = np.var(y_true)
+            explained_var = (1 - np.var(y_true - y_pred) / target_variance
+                             if target_variance > 0 else 0.0)
 
         return {
             "loss": loss.item(),

tests/unit/policy/test_audio_qa_regressions.py

@@ -0,0 +1,62 @@
+"""Numeric and mode regressions use real networks and optimizer updates."""
+import numpy as np
+import pytest
+import torch
+from human_voice_ai.policy.rl_policy import EmotionPolicy
+from human_voice_ai.rl.agent import PolicyNetwork, PPOTrainer, Transition
+
+
+def test_greedy_actions_disable_dropout_and_restore_model_mode():
+    policy = EmotionPolicy(4, 2, hidden_dim=8, epsilon_start=0, device="cpu")
+    layers = list(policy.policy_net.network)
+    with torch.no_grad():
+        for layer in layers:
+            if isinstance(layer, torch.nn.Linear):
+                layer.weight.zero_()
+                layer.bias.zero_()
+        layers[0].weight[0, 0] = 1
+        layers[3].weight[0, 0] = 1
+        layers[6].weight[0, 0] = 1
+        layers[6].bias[1] = .1
+    state = np.ones(4, dtype=np.float32)
+    for training_mode in (True, False):
+        policy.policy_net.train(training_mode)
+        assert {policy.select_action(state, training=False) for _ in range(50)} == {0}
+        assert policy.policy_net.training is training_mode
+
+
+def ppo_fixture():
+    policy = PolicyNetwork(state_dim=4, hidden_dim=8, num_actions=5)
+    with torch.no_grad():
+        for parameter in policy.parameters():
+            parameter.zero_()
+    state = {"audio": torch.zeros(4), "text": torch.zeros(4),
+             "emotion_history": torch.zeros(5), "turn_count": torch.zeros(1)}
+    return policy, state
+
+
+def test_singleton_ppo_update_keeps_metrics_parameters_and_gradients_finite():
+    policy, state = ppo_fixture()
+    before = [value.detach().clone() for value in policy.parameters()]
+    metrics = PPOTrainer(policy).update([Transition(state, 0, 1., state, True)])
+    assert all(np.isfinite(value) for value in metrics.values())
+    assert all(torch.isfinite(value).all() for value in policy.parameters())
+    assert all(value.grad is None or torch.isfinite(value.grad).all() for value in policy.parameters())
+    assert any(not torch.equal(old, new) for old, new in zip(before, policy.parameters()))
+
+
+def test_ppo_returns_do_not_cross_terminal_episode_boundaries(monkeypatch):
+    policy, state = ppo_fixture()
+    targets = []
+    real_loss = torch.nn.functional.mse_loss
+    def capture(prediction, target, *args, **kwargs):
+        assert prediction.shape == target.shape == (2,)
+        targets.append(target.detach().clone())
+        return real_loss(prediction, target, *args, **kwargs)
+    monkeypatch.setattr(torch.nn.functional, "mse_loss", capture)
+    metrics = PPOTrainer(policy).update([
+        Transition(state, 0, 1., state, True),
+        Transition(state, 0, 100., state, True),
+    ])
+    assert targets[0].tolist() == [1., 100.]
+    assert all(np.isfinite(value) for value in metrics.values())