this repo has no description
Continue rapport
+85
-1
+85
-1
rapport/proofs.typ
+85
-1
rapport/proofs.typ
···
117
117
118
118
== Simplification de l'expression de $L(s, a, cal(P), cal(P)', R)$ dans PPO-Clip <proof-ppo-clip-simplify>
119
119
120
+
#let clip = $op("clip")$
121
+
120
122
Soit $(s, a) in S times A$, et $cal(P)'$ une politique.
121
123
122
-
Posons $alpha := A_(cal(P), R) (s, a)$
124
+
Pour alléger les notations, posons $
125
+
alpha &:= A_(cal(P)', R) (s, a) \
126
+
q &:= Q_cal(P) (s, a) \
127
+
q' &:= Q_cal(P)' (s, a) \
128
+
$
129
+
130
+
#let why = explanation => $\ quad & #[car #explanation]$
131
+
132
+
#{
133
+
set math.equation(numbering: none)
134
+
grid(columns: (1fr, 1fr), row-gutter: 1em,
135
+
[
136
+
137
+
*Cas $alpha > 0$ et $q slash q' in [1-epsilon, 1+epsilon]$*
138
+
139
+
$
140
+
&L(s, a, cal(P), cal(P'), R) \
141
+
&= min(q/q' alpha, quad clip(q/q', thick 1-epsilon, thick 1+epsilon) alpha) \
142
+
&= min(q/q', quad clip(q/q', thick 1-epsilon, thick 1+epsilon) ) alpha why(alpha > 0) \
143
+
&= min(q/q', quad q/q') alpha \
144
+
&= min(q/q' , 1+epsilon) alpha \
145
+
$
146
+
], [
147
+
148
+
*Cas $alpha > 0$ et $q slash q' > 1+epsilon$*
149
+
150
+
$
151
+
&L(s, a, cal(P), cal(P'), R) \
152
+
&= min(q/q' alpha, quad clip(q/q', thick 1-epsilon, thick 1+epsilon) alpha) \
153
+
&= min(q/q', quad clip(q/q', thick 1-epsilon, thick 1+epsilon) ) alpha why(alpha > 0) \
154
+
&= min(q/q', quad 1+epsilon) alpha \
155
+
$
156
+
157
+
], [
158
+
159
+
*Cas $alpha > 0$ et $q slash q' < 1-epsilon$*
160
+
161
+
$
162
+
&L(s, a, cal(P), cal(P'), R) \
163
+
&= min(q/q' alpha, quad clip(q/q', thick 1-epsilon, thick 1+epsilon) alpha) \
164
+
&= min(q/q', quad clip(q/q', thick 1-epsilon, thick 1+epsilon) ) alpha why(alpha > 0) \
165
+
&= min(q/q', quad 1-epsilon) alpha \
166
+
&= min(q/q', quad 1+epsilon) alpha why(1+epsilon > 1-epsilon > q / q') \
167
+
$
168
+
169
+
], [
170
+
171
+
*Cas $alpha < 0$ et $q slash q' in [1-epsilon, 1+epsilon]$*
172
+
173
+
$
174
+
&L(s, a, cal(P), cal(P'), R) \
175
+
&= min(q/q' alpha, quad clip(q/q', thick 1-epsilon, thick 1+epsilon) alpha) \
176
+
&= max(q/q', quad clip(q/q', thick 1-epsilon, thick 1+epsilon) ) alpha why(alpha < 0) \
177
+
&= max(q/q', quad q/q') alpha \
178
+
&= max(q/q' , 1-epsilon) alpha \
179
+
$
180
+
181
+
], [
182
+
183
+
*Cas $alpha < 0$ et $q slash q' > 1+epsilon$*
184
+
185
+
$
186
+
&L(s, a, cal(P), cal(P'), R) \
187
+
&= min(q/q' alpha, quad clip(q/q', thick 1-epsilon, thick 1+epsilon) alpha) \
188
+
&= max(q/q', quad clip(q/q', thick 1-epsilon, thick 1+epsilon) ) alpha why(alpha < 0) \
189
+
&= max(q/q', quad 1+epsilon) alpha \
190
+
&= max(q/q', quad 1-epsilon) alpha why(1-epsilon < 1+epsilon < q / q') \
191
+
$
192
+
193
+
], [
194
+
195
+
*Cas $alpha < 0$ et $q slash q' < 1-epsilon$*
196
+
197
+
$
198
+
&L(s, a, cal(P), cal(P'), R) \
199
+
&= min(q/q' alpha, quad clip(q/q', thick 1-epsilon, thick 1+epsilon) alpha) \
200
+
&= max(q/q', quad clip(q/q', thick 1-epsilon, thick 1+epsilon) ) alpha why(alpha < 0) \
201
+
&= max(q/q', quad 1-epsilon) alpha
202
+
$
203
+
204
+
])
205
+
206
+
}