《用Python动手学习强化学习》【贝尔曼方程】【价值最大化】

糖逗

代码取自：《用Python动手学强化学习》第二章，基于价值最大化的贝尔曼方程

1. def R(s):
   
2.     if s == "happy_end":
   
3.         return 1
   
4.     elif s == "bad_end":
   
5.         return -1
   
6.     else:
   
7.         return 0
   
8. 
   
9. def transit_func(s, a):
   
10.     actions = s.split("_")[1:]
    
11.     LIMIT_GAME_COUNT = 5
    
12.     HAPPY_END_BORDER = 4
    
13.     MOVE_PROB = 0.9
    
14.    
    
15.     def next_state(state, action):
    
16.         return "_".join([state, action])
    
17.    
    
18.     if len(actions) == LIMIT_GAME_COUNT:
    
19.         up_count = sum([1 if a == "up" else 0 for a in actions])
    
20.         state = "happy_end" if up_count >= HAPPY_END_BORDER else "bad_end"
    
21.         prob = 1
    
22.         return {state : prob}
    
23.     else:#递归
    
24.         opposite = "up" if a == "down" else "down"
    
25.         return {next_state(s, a): MOVE_PROB,
    
26.                 next_state(s, opposite): 1 - MOVE_PROB}
    
27.         
    
28. def max_V_on_next_state(s):
    
29.     if s in ["happy_end", "bad_end"]:
    
30.         return 0
    
31.     actions = ["up", "down"]
    
32.     values = []
    
33.     for a in actions:
    
34.         transition_probs = transit_func(s, a)
    
35.         v = 0
    
36.         for next_state in transition_probs:
    
37.             prob = transition_probs[next_state]
    
38.             v += prob * V(next_state)
    
39.         values.append(v)
    
40.     return max(values)

复制代码

糖逗