Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'kvm-x86-mmio-7.1' of https://github.com/kvm-x86/linux into HEAD
KVM x86 emulated MMIO changes for 7.1
Copy single-chunk MMIO write values into a persistent (per-fragment) field to
fix use-after-free stack bugs due to KVM dereferencing a stack pointer after an
exit to userspace.
Clean up and comment the emulated MMIO code to try to make it easier to
maintain (not necessarily "easy", but "easier").
+185
-197
-3
arch/x86/include/asm/kvm_host.h
-3
arch/x86/include/asm/kvm_host.h
+13
arch/x86/kvm/emulate.c
+13
arch/x86/kvm/emulate.c
···
1297
1297
int rc;
1298
1298
struct read_cache *mc = &ctxt->mem_read;
1299
1299
1300
+
/*
1301
+
* If the read gets a cache hit, simply copy the value from the cache.
1302
+
* A "hit" here means that there is unused data in the cache, i.e. when
1303
+
* re-emulating an instruction to complete a userspace exit, KVM relies
1304
+
* on "no decode" to ensure the instruction is re-emulated in the same
1305
+
* sequence, so that multiple reads are fulfilled in the correct order.
1306
+
*/
1300
1307
if (mc->pos < mc->end)
1301
1308
goto read_cached;
1302
1309
1303
1310
if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt))
1304
1311
return X86EMUL_UNHANDLEABLE;
1305
1312
1313
+
/*
1314
+
* Route all reads to the cache. This allows @dest to be an on-stack
1315
+
* variable without triggering use-after-free if KVM needs to exit to
1316
+
* userspace to handle an MMIO read (the MMIO fragment will point at
1317
+
* the current location in the cache).
1318
+
*/
1306
1319
rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1307
1320
&ctxt->exception);
1308
1321
if (rc != X86EMUL_CONTINUE)
+7
-15
arch/x86/kvm/svm/sev.c
+7
-15
arch/x86/kvm/svm/sev.c
···
4434
4434
4435
4435
switch (control->exit_code) {
4436
4436
case SVM_VMGEXIT_MMIO_READ:
4437
-
ret = setup_vmgexit_scratch(svm, true, control->exit_info_2);
4437
+
case SVM_VMGEXIT_MMIO_WRITE: {
4438
+
bool is_write = control->exit_code == SVM_VMGEXIT_MMIO_WRITE;
4439
+
4440
+
ret = setup_vmgexit_scratch(svm, !is_write, control->exit_info_2);
4438
4441
if (ret)
4439
4442
break;
4440
4443
4441
-
ret = kvm_sev_es_mmio_read(vcpu,
4442
-
control->exit_info_1,
4443
-
control->exit_info_2,
4444
-
svm->sev_es.ghcb_sa);
4444
+
ret = kvm_sev_es_mmio(vcpu, is_write, control->exit_info_1,
4445
+
control->exit_info_2, svm->sev_es.ghcb_sa);
4445
4446
break;
4446
-
case SVM_VMGEXIT_MMIO_WRITE:
4447
-
ret = setup_vmgexit_scratch(svm, false, control->exit_info_2);
4448
-
if (ret)
4449
-
break;
4450
-
4451
-
ret = kvm_sev_es_mmio_write(vcpu,
4452
-
control->exit_info_1,
4453
-
control->exit_info_2,
4454
-
svm->sev_es.ghcb_sa);
4455
-
break;
4447
+
}
4456
4448
case SVM_VMGEXIT_NMI_COMPLETE:
4457
4449
++vcpu->stat.nmi_window_exits;
4458
4450
svm->nmi_masked = false;
+4
-10
arch/x86/kvm/vmx/tdx.c
+4
-10
arch/x86/kvm/vmx/tdx.c
···
1467
1467
1468
1468
/* Request the device emulation to userspace device model. */
1469
1469
vcpu->mmio_is_write = write;
1470
-
if (!write)
1470
+
1471
+
__kvm_prepare_emulated_mmio_exit(vcpu, gpa, size, &val, write);
1472
+
1473
+
if (!write) {
1471
1474
vcpu->arch.complete_userspace_io = tdx_complete_mmio_read;
1472
-
1473
-
vcpu->run->mmio.phys_addr = gpa;
1474
-
vcpu->run->mmio.len = size;
1475
-
vcpu->run->mmio.is_write = write;
1476
-
vcpu->run->exit_reason = KVM_EXIT_MMIO;
1477
-
1478
-
if (write) {
1479
-
memcpy(vcpu->run->mmio.data, &val, size);
1480
-
} else {
1481
1475
vcpu->mmio_fragments[0].gpa = gpa;
1482
1476
vcpu->mmio_fragments[0].len = size;
1483
1477
trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, size, gpa, NULL);
+131
-164
arch/x86/kvm/x86.c
+131
-164
arch/x86/kvm/x86.c
···
7768
7768
}
7769
7769
7770
7770
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
7771
-
const void *v)
7771
+
void *__v)
7772
7772
{
7773
+
const void *v = __v;
7773
7774
int handled = 0;
7774
7775
int n;
7776
+
7777
+
trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, addr, __v);
7775
7778
7776
7779
do {
7777
7780
n = min(len, 8);
···
7809
7806
len -= n;
7810
7807
v += n;
7811
7808
} while (len);
7809
+
7810
+
if (len)
7811
+
trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len, addr, NULL);
7812
7812
7813
7813
return handled;
7814
7814
}
···
8101
8095
return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
8102
8096
}
8103
8097
8104
-
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
8105
-
const void *val, int bytes)
8098
+
struct read_write_emulator_ops {
8099
+
int (*read_write_guest)(struct kvm_vcpu *vcpu, gpa_t gpa,
8100
+
void *val, int bytes);
8101
+
int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
8102
+
int bytes, void *val);
8103
+
bool write;
8104
+
};
8105
+
8106
+
static int emulator_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa,
8107
+
void *val, int bytes)
8108
+
{
8109
+
return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
8110
+
}
8111
+
8112
+
static int emulator_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa,
8113
+
void *val, int bytes)
8106
8114
{
8107
8115
int ret;
8108
8116
···
8126
8106
kvm_page_track_write(vcpu, gpa, val, bytes);
8127
8107
return 1;
8128
8108
}
8129
-
8130
-
struct read_write_emulator_ops {
8131
-
int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
8132
-
int bytes);
8133
-
int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
8134
-
void *val, int bytes);
8135
-
int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
8136
-
int bytes, void *val);
8137
-
int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
8138
-
void *val, int bytes);
8139
-
bool write;
8140
-
};
8141
-
8142
-
static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
8143
-
{
8144
-
if (vcpu->mmio_read_completed) {
8145
-
trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
8146
-
vcpu->mmio_fragments[0].gpa, val);
8147
-
vcpu->mmio_read_completed = 0;
8148
-
return 1;
8149
-
}
8150
-
8151
-
return 0;
8152
-
}
8153
-
8154
-
static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
8155
-
void *val, int bytes)
8156
-
{
8157
-
return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
8158
-
}
8159
-
8160
-
static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
8161
-
void *val, int bytes)
8162
-
{
8163
-
return emulator_write_phys(vcpu, gpa, val, bytes);
8164
-
}
8165
-
8166
-
static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
8167
-
{
8168
-
trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
8169
-
return vcpu_mmio_write(vcpu, gpa, bytes, val);
8170
-
}
8171
-
8172
-
static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
8173
-
void *val, int bytes)
8174
-
{
8175
-
trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
8176
-
return X86EMUL_IO_NEEDED;
8177
-
}
8178
-
8179
-
static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
8180
-
void *val, int bytes)
8181
-
{
8182
-
struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
8183
-
8184
-
memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
8185
-
return X86EMUL_CONTINUE;
8186
-
}
8187
-
8188
-
static const struct read_write_emulator_ops read_emultor = {
8189
-
.read_write_prepare = read_prepare,
8190
-
.read_write_emulate = read_emulate,
8191
-
.read_write_mmio = vcpu_mmio_read,
8192
-
.read_write_exit_mmio = read_exit_mmio,
8193
-
};
8194
-
8195
-
static const struct read_write_emulator_ops write_emultor = {
8196
-
.read_write_emulate = write_emulate,
8197
-
.read_write_mmio = write_mmio,
8198
-
.read_write_exit_mmio = write_exit_mmio,
8199
-
.write = true,
8200
-
};
8201
8109
8202
8110
static int emulator_read_write_onepage(unsigned long addr, void *val,
8203
8111
unsigned int bytes,
···
8156
8208
return X86EMUL_PROPAGATE_FAULT;
8157
8209
}
8158
8210
8159
-
if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
8211
+
/*
8212
+
* If the memory is not _known_ to be emulated MMIO, attempt to access
8213
+
* guest memory. If accessing guest memory fails, e.g. because there's
8214
+
* no memslot, then handle the access as MMIO. Note, treating the
8215
+
* access as emulated MMIO is technically wrong if there is a memslot,
8216
+
* i.e. if accessing host user memory failed, but this has been KVM's
8217
+
* historical ABI for decades.
8218
+
*/
8219
+
if (!ret && ops->read_write_guest(vcpu, gpa, val, bytes))
8160
8220
return X86EMUL_CONTINUE;
8161
8221
8162
8222
/*
8163
-
* Is this MMIO handled locally?
8223
+
* Attempt to handle emulated MMIO within the kernel, e.g. for accesses
8224
+
* to an in-kernel local or I/O APIC, or to an ioeventfd range attached
8225
+
* to MMIO bus. If the access isn't fully resolved, insert an MMIO
8226
+
* fragment with the relevant details.
8164
8227
*/
8165
8228
handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
8166
8229
if (handled == bytes)
···
8184
8225
WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
8185
8226
frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
8186
8227
frag->gpa = gpa;
8187
-
frag->data = val;
8228
+
if (write && bytes <= 8u) {
8229
+
frag->val = 0;
8230
+
frag->data = &frag->val;
8231
+
memcpy(&frag->val, val, bytes);
8232
+
} else {
8233
+
frag->data = val;
8234
+
}
8188
8235
frag->len = bytes;
8236
+
8237
+
/*
8238
+
* Continue emulating, even though KVM needs to (eventually) do an MMIO
8239
+
* exit to userspace. If the access splits multiple pages, then KVM
8240
+
* needs to exit to userspace only after emulating both parts of the
8241
+
* access.
8242
+
*/
8189
8243
return X86EMUL_CONTINUE;
8190
8244
}
8191
8245
···
8209
8237
const struct read_write_emulator_ops *ops)
8210
8238
{
8211
8239
struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8212
-
gpa_t gpa;
8213
8240
int rc;
8214
8241
8215
-
if (ops->read_write_prepare &&
8216
-
ops->read_write_prepare(vcpu, val, bytes))
8242
+
if (WARN_ON_ONCE((bytes > 8u || !ops->write) && object_is_on_stack(val)))
8243
+
return X86EMUL_UNHANDLEABLE;
8244
+
8245
+
/*
8246
+
* If the read was already completed via a userspace MMIO exit, there's
8247
+
* nothing left to do except trace the MMIO read. When completing MMIO
8248
+
* reads, KVM re-emulates the instruction to propagate the value into
8249
+
* the correct destination, e.g. into the correct register, but the
8250
+
* value itself has already been copied to the read cache.
8251
+
*
8252
+
* Note! This is *tightly* coupled to read_emulated() satisfying reads
8253
+
* from the emulator's mem_read cache, so that the MMIO fragment data
8254
+
* is copied to the correct chunk of the correct operand.
8255
+
*/
8256
+
if (!ops->write && vcpu->mmio_read_completed) {
8257
+
/*
8258
+
* For simplicity, trace the entire MMIO read in one shot, even
8259
+
* though the GPA might be incorrect if there are two fragments
8260
+
* that aren't contiguous in the GPA space.
8261
+
*/
8262
+
trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
8263
+
vcpu->mmio_fragments[0].gpa, val);
8264
+
vcpu->mmio_read_completed = 0;
8217
8265
return X86EMUL_CONTINUE;
8266
+
}
8218
8267
8219
8268
vcpu->mmio_nr_fragments = 0;
8220
8269
···
8264
8271
if (!vcpu->mmio_nr_fragments)
8265
8272
return X86EMUL_CONTINUE;
8266
8273
8267
-
gpa = vcpu->mmio_fragments[0].gpa;
8268
-
8269
8274
vcpu->mmio_needed = 1;
8270
8275
vcpu->mmio_cur_fragment = 0;
8276
+
vcpu->mmio_is_write = ops->write;
8271
8277
8272
-
vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
8273
-
vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
8274
-
vcpu->run->exit_reason = KVM_EXIT_MMIO;
8275
-
vcpu->run->mmio.phys_addr = gpa;
8278
+
kvm_prepare_emulated_mmio_exit(vcpu, &vcpu->mmio_fragments[0]);
8276
8279
8277
-
return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
8280
+
/*
8281
+
* For MMIO reads, stop emulating and immediately exit to userspace, as
8282
+
* KVM needs the value to correctly emulate the instruction. For MMIO
8283
+
* writes, continue emulating as the write to MMIO is a side effect for
8284
+
* all intents and purposes. KVM will still exit to userspace, but
8285
+
* after completing emulation (see the check on vcpu->mmio_needed in
8286
+
* x86_emulate_instruction()).
8287
+
*/
8288
+
return ops->write ? X86EMUL_CONTINUE : X86EMUL_IO_NEEDED;
8278
8289
}
8279
8290
8280
8291
static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
···
8287
8290
unsigned int bytes,
8288
8291
struct x86_exception *exception)
8289
8292
{
8290
-
return emulator_read_write(ctxt, addr, val, bytes,
8291
-
exception, &read_emultor);
8293
+
static const struct read_write_emulator_ops ops = {
8294
+
.read_write_guest = emulator_read_guest,
8295
+
.read_write_mmio = vcpu_mmio_read,
8296
+
.write = false,
8297
+
};
8298
+
8299
+
return emulator_read_write(ctxt, addr, val, bytes, exception, &ops);
8292
8300
}
8293
8301
8294
8302
static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
···
8302
8300
unsigned int bytes,
8303
8301
struct x86_exception *exception)
8304
8302
{
8305
-
return emulator_read_write(ctxt, addr, (void *)val, bytes,
8306
-
exception, &write_emultor);
8303
+
static const struct read_write_emulator_ops ops = {
8304
+
.read_write_guest = emulator_write_guest,
8305
+
.read_write_mmio = vcpu_mmio_write,
8306
+
.write = true,
8307
+
};
8308
+
8309
+
return emulator_read_write(ctxt, addr, (void *)val, bytes, exception, &ops);
8307
8310
}
8308
8311
8309
8312
#define emulator_try_cmpxchg_user(t, ptr, old, new) \
···
9701
9694
unsigned long val;
9702
9695
9703
9696
/* We should only ever be called with arch.pio.count equal to 1 */
9704
-
BUG_ON(vcpu->arch.pio.count != 1);
9697
+
if (KVM_BUG_ON(vcpu->arch.pio.count != 1, vcpu->kvm))
9698
+
return -EIO;
9705
9699
9706
9700
if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip))) {
9707
9701
vcpu->arch.pio.count = 0;
···
11824
11816
11825
11817
static int complete_emulated_pio(struct kvm_vcpu *vcpu)
11826
11818
{
11827
-
BUG_ON(!vcpu->arch.pio.count);
11819
+
if (KVM_BUG_ON(!vcpu->arch.pio.count, vcpu->kvm))
11820
+
return -EIO;
11828
11821
11829
11822
return complete_emulated_io(vcpu);
11830
11823
}
···
11854
11845
struct kvm_mmio_fragment *frag;
11855
11846
unsigned len;
11856
11847
11857
-
BUG_ON(!vcpu->mmio_needed);
11848
+
if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm))
11849
+
return -EIO;
11858
11850
11859
11851
/* Complete previous fragment */
11860
11852
frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
···
11868
11858
frag++;
11869
11859
vcpu->mmio_cur_fragment++;
11870
11860
} else {
11861
+
if (WARN_ON_ONCE(frag->data == &frag->val))
11862
+
return -EIO;
11863
+
11871
11864
/* Go forward to the next mmio piece. */
11872
11865
frag->data += len;
11873
11866
frag->gpa += len;
···
11887
11874
return complete_emulated_io(vcpu);
11888
11875
}
11889
11876
11890
-
run->exit_reason = KVM_EXIT_MMIO;
11891
-
run->mmio.phys_addr = frag->gpa;
11892
-
if (vcpu->mmio_is_write)
11893
-
memcpy(run->mmio.data, frag->data, min(8u, frag->len));
11894
-
run->mmio.len = min(8u, frag->len);
11895
-
run->mmio.is_write = vcpu->mmio_is_write;
11877
+
kvm_prepare_emulated_mmio_exit(vcpu, frag);
11896
11878
vcpu->arch.complete_userspace_io = complete_emulated_mmio;
11897
11879
return 0;
11898
11880
}
···
14263
14255
struct kvm_mmio_fragment *frag;
14264
14256
unsigned int len;
14265
14257
14266
-
BUG_ON(!vcpu->mmio_needed);
14258
+
if (KVM_BUG_ON(!vcpu->mmio_needed, vcpu->kvm))
14259
+
return -EIO;
14267
14260
14268
14261
/* Complete previous fragment */
14269
14262
frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
···
14286
14277
if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
14287
14278
vcpu->mmio_needed = 0;
14288
14279
14289
-
// VMG change, at this point, we're always done
14290
-
// RIP has already been advanced
14280
+
/*
14281
+
* All done, as frag->data always points at the GHCB scratch
14282
+
* area and VMGEXIT is trap-like (RIP is advanced by hardware).
14283
+
*/
14291
14284
return 1;
14292
14285
}
14293
14286
14294
14287
// More MMIO is needed
14295
-
run->mmio.phys_addr = frag->gpa;
14296
-
run->mmio.len = min(8u, frag->len);
14297
-
run->mmio.is_write = vcpu->mmio_is_write;
14298
-
if (run->mmio.is_write)
14299
-
memcpy(run->mmio.data, frag->data, min(8u, frag->len));
14300
-
run->exit_reason = KVM_EXIT_MMIO;
14301
-
14288
+
kvm_prepare_emulated_mmio_exit(vcpu, frag);
14302
14289
vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
14303
-
14304
14290
return 0;
14305
14291
}
14306
14292
14307
-
int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
14308
-
void *data)
14293
+
int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa,
14294
+
unsigned int bytes, void *data)
14309
14295
{
14310
-
int handled;
14311
14296
struct kvm_mmio_fragment *frag;
14297
+
int handled;
14312
14298
14313
-
if (!data)
14299
+
if (!data || WARN_ON_ONCE(object_is_on_stack(data)))
14314
14300
return -EINVAL;
14315
14301
14316
-
handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);
14302
+
if (is_write)
14303
+
handled = vcpu_mmio_write(vcpu, gpa, bytes, data);
14304
+
else
14305
+
handled = vcpu_mmio_read(vcpu, gpa, bytes, data);
14317
14306
if (handled == bytes)
14318
14307
return 1;
14319
14308
···
14319
14312
gpa += handled;
14320
14313
data += handled;
14321
14314
14322
-
/*TODO: Check if need to increment number of frags */
14315
+
/*
14316
+
* TODO: Determine whether or not userspace plays nice with MMIO
14317
+
* requests that split a page boundary.
14318
+
*/
14323
14319
frag = vcpu->mmio_fragments;
14324
-
vcpu->mmio_nr_fragments = 1;
14325
14320
frag->len = bytes;
14326
14321
frag->gpa = gpa;
14327
14322
frag->data = data;
14328
14323
14329
14324
vcpu->mmio_needed = 1;
14330
14325
vcpu->mmio_cur_fragment = 0;
14331
-
14332
-
vcpu->run->mmio.phys_addr = gpa;
14333
-
vcpu->run->mmio.len = min(8u, frag->len);
14334
-
vcpu->run->mmio.is_write = 1;
14335
-
memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
14336
-
vcpu->run->exit_reason = KVM_EXIT_MMIO;
14337
-
14338
-
vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
14339
-
14340
-
return 0;
14341
-
}
14342
-
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio_write);
14343
-
14344
-
int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
14345
-
void *data)
14346
-
{
14347
-
int handled;
14348
-
struct kvm_mmio_fragment *frag;
14349
-
14350
-
if (!data)
14351
-
return -EINVAL;
14352
-
14353
-
handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);
14354
-
if (handled == bytes)
14355
-
return 1;
14356
-
14357
-
bytes -= handled;
14358
-
gpa += handled;
14359
-
data += handled;
14360
-
14361
-
/*TODO: Check if need to increment number of frags */
14362
-
frag = vcpu->mmio_fragments;
14363
14326
vcpu->mmio_nr_fragments = 1;
14364
-
frag->len = bytes;
14365
-
frag->gpa = gpa;
14366
-
frag->data = data;
14327
+
vcpu->mmio_is_write = is_write;
14367
14328
14368
-
vcpu->mmio_needed = 1;
14369
-
vcpu->mmio_cur_fragment = 0;
14370
-
14371
-
vcpu->run->mmio.phys_addr = gpa;
14372
-
vcpu->run->mmio.len = min(8u, frag->len);
14373
-
vcpu->run->mmio.is_write = 0;
14374
-
vcpu->run->exit_reason = KVM_EXIT_MMIO;
14375
-
14329
+
kvm_prepare_emulated_mmio_exit(vcpu, frag);
14376
14330
vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
14377
-
14378
14331
return 0;
14379
14332
}
14380
-
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio_read);
14333
+
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_sev_es_mmio);
14381
14334
14382
14335
static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)
14383
14336
{
+28
-4
arch/x86/kvm/x86.h
+28
-4
arch/x86/kvm/x86.h
···
712
712
__reserved_bits; \
713
713
})
714
714
715
-
int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t src, unsigned int bytes,
716
-
void *dst);
717
-
int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t src, unsigned int bytes,
718
-
void *dst);
715
+
int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa,
716
+
unsigned int bytes, void *data);
719
717
int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
720
718
unsigned int port, void *data, unsigned int count,
721
719
int in);
720
+
721
+
static inline void __kvm_prepare_emulated_mmio_exit(struct kvm_vcpu *vcpu,
722
+
gpa_t gpa, unsigned int len,
723
+
const void *data,
724
+
bool is_write)
725
+
{
726
+
struct kvm_run *run = vcpu->run;
727
+
728
+
KVM_BUG_ON(len > 8, vcpu->kvm);
729
+
730
+
run->mmio.len = len;
731
+
run->mmio.is_write = is_write;
732
+
run->exit_reason = KVM_EXIT_MMIO;
733
+
run->mmio.phys_addr = gpa;
734
+
if (is_write)
735
+
memcpy(run->mmio.data, data, len);
736
+
}
737
+
738
+
static inline void kvm_prepare_emulated_mmio_exit(struct kvm_vcpu *vcpu,
739
+
struct kvm_mmio_fragment *frag)
740
+
{
741
+
WARN_ON_ONCE(!vcpu->mmio_needed || !vcpu->mmio_nr_fragments);
742
+
743
+
__kvm_prepare_emulated_mmio_exit(vcpu, frag->gpa, min(8u, frag->len),
744
+
frag->data, vcpu->mmio_is_write);
745
+
}
722
746
723
747
static inline bool user_exit_on_hypercall(struct kvm *kvm, unsigned long hc_nr)
724
748
{