Gaurav Ghosal the Effect of Modeling Human Rationality Level 2023

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

import torch
import torch.nn.functional as F

class PolicyNetwork(torch.nn.Module):
    def __init__(self, ...):
        super().__init__()
        # define network layers

    def forward(self, state):
        # compute logits
        return logits

def select_action(logits, tau):
    probabilities = F.softmax(logits / tau, dim=-1)
    action = torch.distributions.Categorical(probabilities).sample()
    return action

# During training, adjust tau and use it in loss calculation
tau = initial_tau  # This could be a fixed value or decay over episodes
logits = policy_network(state)
action = select_action(logits, tau)



# in real codebase 
def forward(self, x: th.Tensor, beta: th.Tensor=None) -> th.Tensor:
    x = self.linear(x)
    if beta is not None:
        x = x * beta # ! for Human Rationality Level Beta parameter, 0 meaning the reward signal is random, 1 meaning the reward signal is perfect
    logits = F.log_softmax(x, dim=-1)
    return logits