Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: qat - add qat_6xxx driver
Add a new driver, qat_6xxx, to support QAT GEN6 devices.
QAT GEN6 devices are a follow-on generation of GEN4 devices and
differently from the previous generation, they can support all three
services (symmetric, asymmetric, and data compression) concurrently.
In order to have the qat_6xxx driver to reuse some of the GEN4 logic,
a new abstraction layer has been introduced to bridge the two
implementations. This allows to avoid code duplication and to keep the
qat_6xxx driver isolated from the GEN4 logic. This approach has been
used for the PF to VF logic and the HW CSR access logic.
Signed-off-by: Laurent M Coquerel <laurent.m.coquerel@intel.com>
Co-developed-by: George Abraham P <george.abraham.p@intel.com>
Signed-off-by: George Abraham P <george.abraham.p@intel.com>
Co-developed-by: Karthikeyan Gopal <karthikeyan.gopal@intel.com>
Signed-off-by: Karthikeyan Gopal <karthikeyan.gopal@intel.com>
Co-developed-by: Suman Kumar Chakraborty <suman.kumar.chakraborty@intel.com>
Signed-off-by: Suman Kumar Chakraborty <suman.kumar.chakraborty@intel.com>
Reviewed-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Laurent M Coquerel
and committed by
Herbert Xu
17fd7514
942028bc
+1328
+12
drivers/crypto/intel/qat/Kconfig
+12
drivers/crypto/intel/qat/Kconfig
···
70
70
To compile this as a module, choose M here: the module
71
71
will be called qat_420xx.
72
72
73
+
config CRYPTO_DEV_QAT_6XXX
74
+
tristate "Support for Intel(R) QuickAssist Technology QAT_6XXX"
75
+
depends on (X86 || COMPILE_TEST)
76
+
depends on PCI
77
+
select CRYPTO_DEV_QAT
78
+
help
79
+
Support for Intel(R) QuickAssist Technology QAT_6xxx
80
+
for accelerating crypto and compression workloads.
81
+
82
+
To compile this as a module, choose M here: the module
83
+
will be called qat_6xxx.
84
+
73
85
config CRYPTO_DEV_QAT_DH895xCCVF
74
86
tristate "Support for Intel(R) DH895xCC Virtual Function"
75
87
depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST)
+1
drivers/crypto/intel/qat/Makefile
+1
drivers/crypto/intel/qat/Makefile
···
6
6
obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/
7
7
obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/
8
8
obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx/
9
+
obj-$(CONFIG_CRYPTO_DEV_QAT_6XXX) += qat_6xxx/
9
10
obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/
10
11
obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/
11
12
obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/
+3
drivers/crypto/intel/qat/qat_6xxx/Makefile
+3
drivers/crypto/intel/qat/qat_6xxx/Makefile
+843
drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c
+843
drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
#include <linux/array_size.h>
4
+
#include <linux/bitfield.h>
5
+
#include <linux/bitops.h>
6
+
#include <linux/bits.h>
7
+
#include <linux/iopoll.h>
8
+
#include <linux/pci.h>
9
+
#include <linux/types.h>
10
+
11
+
#include <adf_accel_devices.h>
12
+
#include <adf_admin.h>
13
+
#include <adf_cfg.h>
14
+
#include <adf_cfg_services.h>
15
+
#include <adf_clock.h>
16
+
#include <adf_common_drv.h>
17
+
#include <adf_fw_config.h>
18
+
#include <adf_gen6_pm.h>
19
+
#include <adf_gen6_shared.h>
20
+
#include <adf_timer.h>
21
+
#include "adf_6xxx_hw_data.h"
22
+
#include "icp_qat_fw_comp.h"
23
+
#include "icp_qat_hw_51_comp.h"
24
+
25
+
#define RP_GROUP_0_MASK (BIT(0) | BIT(2))
26
+
#define RP_GROUP_1_MASK (BIT(1) | BIT(3))
27
+
#define RP_GROUP_ALL_MASK (RP_GROUP_0_MASK | RP_GROUP_1_MASK)
28
+
29
+
#define ADF_AE_GROUP_0 GENMASK(3, 0)
30
+
#define ADF_AE_GROUP_1 GENMASK(7, 4)
31
+
#define ADF_AE_GROUP_2 BIT(8)
32
+
33
+
struct adf_ring_config {
34
+
u32 ring_mask;
35
+
enum adf_cfg_service_type ring_type;
36
+
const unsigned long *thrd_mask;
37
+
};
38
+
39
+
static u32 rmask_two_services[] = {
40
+
RP_GROUP_0_MASK,
41
+
RP_GROUP_1_MASK,
42
+
};
43
+
44
+
enum adf_gen6_rps {
45
+
RP0 = 0,
46
+
RP1 = 1,
47
+
RP2 = 2,
48
+
RP3 = 3,
49
+
RP_MAX = RP3
50
+
};
51
+
52
+
/*
53
+
* thrd_mask_[sym|asym|cpr|dcc]: these static arrays define the thread
54
+
* configuration for handling requests of specific services across the
55
+
* accelerator engines. Each element in an array corresponds to an
56
+
* accelerator engine, with the value being a bitmask that specifies which
57
+
* threads within that engine are capable of processing the particular service.
58
+
*
59
+
* For example, a value of 0x0C means that threads 2 and 3 are enabled for the
60
+
* service in the respective accelerator engine.
61
+
*/
62
+
static const unsigned long thrd_mask_sym[ADF_6XXX_MAX_ACCELENGINES] = {
63
+
0x0C, 0x0C, 0x0C, 0x0C, 0x1C, 0x1C, 0x1C, 0x1C, 0x00
64
+
};
65
+
66
+
static const unsigned long thrd_mask_asym[ADF_6XXX_MAX_ACCELENGINES] = {
67
+
0x70, 0x70, 0x70, 0x70, 0x60, 0x60, 0x60, 0x60, 0x00
68
+
};
69
+
70
+
static const unsigned long thrd_mask_cpr[ADF_6XXX_MAX_ACCELENGINES] = {
71
+
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00
72
+
};
73
+
74
+
static const unsigned long thrd_mask_dcc[ADF_6XXX_MAX_ACCELENGINES] = {
75
+
0x00, 0x00, 0x00, 0x00, 0x07, 0x07, 0x03, 0x03, 0x00
76
+
};
77
+
78
+
static const char *const adf_6xxx_fw_objs[] = {
79
+
[ADF_FW_CY_OBJ] = ADF_6XXX_CY_OBJ,
80
+
[ADF_FW_DC_OBJ] = ADF_6XXX_DC_OBJ,
81
+
[ADF_FW_ADMIN_OBJ] = ADF_6XXX_ADMIN_OBJ,
82
+
};
83
+
84
+
static const struct adf_fw_config adf_default_fw_config[] = {
85
+
{ ADF_AE_GROUP_1, ADF_FW_DC_OBJ },
86
+
{ ADF_AE_GROUP_0, ADF_FW_CY_OBJ },
87
+
{ ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ },
88
+
};
89
+
90
+
static struct adf_hw_device_class adf_6xxx_class = {
91
+
.name = ADF_6XXX_DEVICE_NAME,
92
+
.type = DEV_6XXX,
93
+
};
94
+
95
+
static bool services_supported(unsigned long mask)
96
+
{
97
+
int num_svc;
98
+
99
+
if (mask >= BIT(SVC_BASE_COUNT))
100
+
return false;
101
+
102
+
num_svc = hweight_long(mask);
103
+
switch (num_svc) {
104
+
case ADF_ONE_SERVICE:
105
+
return true;
106
+
case ADF_TWO_SERVICES:
107
+
case ADF_THREE_SERVICES:
108
+
return !test_bit(SVC_DCC, &mask);
109
+
default:
110
+
return false;
111
+
}
112
+
}
113
+
114
+
static int get_service(unsigned long *mask)
115
+
{
116
+
if (test_and_clear_bit(SVC_ASYM, mask))
117
+
return SVC_ASYM;
118
+
119
+
if (test_and_clear_bit(SVC_SYM, mask))
120
+
return SVC_SYM;
121
+
122
+
if (test_and_clear_bit(SVC_DC, mask))
123
+
return SVC_DC;
124
+
125
+
if (test_and_clear_bit(SVC_DCC, mask))
126
+
return SVC_DCC;
127
+
128
+
return -EINVAL;
129
+
}
130
+
131
+
static enum adf_cfg_service_type get_ring_type(enum adf_services service)
132
+
{
133
+
switch (service) {
134
+
case SVC_SYM:
135
+
return SYM;
136
+
case SVC_ASYM:
137
+
return ASYM;
138
+
case SVC_DC:
139
+
case SVC_DCC:
140
+
return COMP;
141
+
default:
142
+
return UNUSED;
143
+
}
144
+
}
145
+
146
+
static const unsigned long *get_thrd_mask(enum adf_services service)
147
+
{
148
+
switch (service) {
149
+
case SVC_SYM:
150
+
return thrd_mask_sym;
151
+
case SVC_ASYM:
152
+
return thrd_mask_asym;
153
+
case SVC_DC:
154
+
return thrd_mask_cpr;
155
+
case SVC_DCC:
156
+
return thrd_mask_dcc;
157
+
default:
158
+
return NULL;
159
+
}
160
+
}
161
+
162
+
static int get_rp_config(struct adf_accel_dev *accel_dev, struct adf_ring_config *rp_config,
163
+
unsigned int *num_services)
164
+
{
165
+
unsigned int i, nservices;
166
+
unsigned long mask;
167
+
int ret, service;
168
+
169
+
ret = adf_get_service_mask(accel_dev, &mask);
170
+
if (ret)
171
+
return ret;
172
+
173
+
nservices = hweight_long(mask);
174
+
if (nservices > MAX_NUM_CONCURR_SVC)
175
+
return -EINVAL;
176
+
177
+
for (i = 0; i < nservices; i++) {
178
+
service = get_service(&mask);
179
+
if (service < 0)
180
+
return service;
181
+
182
+
rp_config[i].ring_type = get_ring_type(service);
183
+
rp_config[i].thrd_mask = get_thrd_mask(service);
184
+
185
+
/*
186
+
* If there is only one service enabled, use all ring pairs for
187
+
* that service.
188
+
* If there are two services enabled, use ring pairs 0 and 2 for
189
+
* one service and ring pairs 1 and 3 for the other service.
190
+
*/
191
+
switch (nservices) {
192
+
case ADF_ONE_SERVICE:
193
+
rp_config[i].ring_mask = RP_GROUP_ALL_MASK;
194
+
break;
195
+
case ADF_TWO_SERVICES:
196
+
rp_config[i].ring_mask = rmask_two_services[i];
197
+
break;
198
+
case ADF_THREE_SERVICES:
199
+
rp_config[i].ring_mask = BIT(i);
200
+
201
+
/* If ASYM is enabled, use additional ring pair */
202
+
if (service == SVC_ASYM)
203
+
rp_config[i].ring_mask |= BIT(RP3);
204
+
205
+
break;
206
+
default:
207
+
return -EINVAL;
208
+
}
209
+
}
210
+
211
+
*num_services = nservices;
212
+
213
+
return 0;
214
+
}
215
+
216
+
static u32 adf_gen6_get_arb_mask(struct adf_accel_dev *accel_dev, unsigned int ae)
217
+
{
218
+
struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC];
219
+
unsigned int num_services, i, thrd;
220
+
u32 ring_mask, thd2arb_mask = 0;
221
+
const unsigned long *p_mask;
222
+
223
+
if (get_rp_config(accel_dev, rp_config, &num_services))
224
+
return 0;
225
+
226
+
/*
227
+
* The thd2arb_mask maps ring pairs to threads within an accelerator engine.
228
+
* It ensures that jobs submitted to ring pairs are scheduled on threads capable
229
+
* of handling the specified service type.
230
+
*
231
+
* Each group of 4 bits in the mask corresponds to a thread, with each bit
232
+
* indicating whether a job from a ring pair can be scheduled on that thread.
233
+
* The use of 4 bits is due to the organization of ring pairs into groups of
234
+
* four, where each group shares the same configuration.
235
+
*/
236
+
for (i = 0; i < num_services; i++) {
237
+
p_mask = &rp_config[i].thrd_mask[ae];
238
+
ring_mask = rp_config[i].ring_mask;
239
+
240
+
for_each_set_bit(thrd, p_mask, ADF_NUM_THREADS_PER_AE)
241
+
thd2arb_mask |= ring_mask << (thrd * 4);
242
+
}
243
+
244
+
return thd2arb_mask;
245
+
}
246
+
247
+
static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev)
248
+
{
249
+
enum adf_cfg_service_type rps[ADF_GEN6_NUM_BANKS_PER_VF] = { };
250
+
struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC];
251
+
unsigned int num_services, rp_num, i;
252
+
unsigned long cfg_mask;
253
+
u16 ring_to_svc_map;
254
+
255
+
if (get_rp_config(accel_dev, rp_config, &num_services))
256
+
return 0;
257
+
258
+
/*
259
+
* Loop through the configured services and populate the `rps` array that
260
+
* contains what service that particular ring pair can handle (i.e. symmetric
261
+
* crypto, asymmetric crypto, data compression or compression chaining).
262
+
*/
263
+
for (i = 0; i < num_services; i++) {
264
+
cfg_mask = rp_config[i].ring_mask;
265
+
for_each_set_bit(rp_num, &cfg_mask, ADF_GEN6_NUM_BANKS_PER_VF)
266
+
rps[rp_num] = rp_config[i].ring_type;
267
+
}
268
+
269
+
/*
270
+
* The ring_mask is structured into segments of 3 bits, with each
271
+
* segment representing the service configuration for a specific ring pair.
272
+
* Since ring pairs are organized into groups of 4, the ring_mask contains 4
273
+
* such 3-bit segments, each corresponding to one ring pair.
274
+
*
275
+
* The device has 64 ring pairs, which are organized in groups of 4, namely
276
+
* 16 groups. Each group has the same configuration, represented here by
277
+
* `ring_to_svc_map`.
278
+
*/
279
+
ring_to_svc_map = rps[RP0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT |
280
+
rps[RP1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT |
281
+
rps[RP2] << ADF_CFG_SERV_RING_PAIR_2_SHIFT |
282
+
rps[RP3] << ADF_CFG_SERV_RING_PAIR_3_SHIFT;
283
+
284
+
return ring_to_svc_map;
285
+
}
286
+
287
+
static u32 get_accel_mask(struct adf_hw_device_data *self)
288
+
{
289
+
return ADF_GEN6_ACCELERATORS_MASK;
290
+
}
291
+
292
+
static u32 get_num_accels(struct adf_hw_device_data *self)
293
+
{
294
+
return ADF_GEN6_MAX_ACCELERATORS;
295
+
}
296
+
297
+
static u32 get_num_aes(struct adf_hw_device_data *self)
298
+
{
299
+
return self ? hweight32(self->ae_mask) : 0;
300
+
}
301
+
302
+
static u32 get_misc_bar_id(struct adf_hw_device_data *self)
303
+
{
304
+
return ADF_GEN6_PMISC_BAR;
305
+
}
306
+
307
+
static u32 get_etr_bar_id(struct adf_hw_device_data *self)
308
+
{
309
+
return ADF_GEN6_ETR_BAR;
310
+
}
311
+
312
+
static u32 get_sram_bar_id(struct adf_hw_device_data *self)
313
+
{
314
+
return ADF_GEN6_SRAM_BAR;
315
+
}
316
+
317
+
static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
318
+
{
319
+
return DEV_SKU_1;
320
+
}
321
+
322
+
static void get_arb_info(struct arb_info *arb_info)
323
+
{
324
+
arb_info->arb_cfg = ADF_GEN6_ARB_CONFIG;
325
+
arb_info->arb_offset = ADF_GEN6_ARB_OFFSET;
326
+
arb_info->wt2sam_offset = ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET;
327
+
}
328
+
329
+
static void get_admin_info(struct admin_info *admin_csrs_info)
330
+
{
331
+
admin_csrs_info->mailbox_offset = ADF_GEN6_MAILBOX_BASE_OFFSET;
332
+
admin_csrs_info->admin_msg_ur = ADF_GEN6_ADMINMSGUR_OFFSET;
333
+
admin_csrs_info->admin_msg_lr = ADF_GEN6_ADMINMSGLR_OFFSET;
334
+
}
335
+
336
+
static u32 get_heartbeat_clock(struct adf_hw_device_data *self)
337
+
{
338
+
return ADF_GEN6_COUNTER_FREQ;
339
+
}
340
+
341
+
static void enable_error_correction(struct adf_accel_dev *accel_dev)
342
+
{
343
+
void __iomem *csr = adf_get_pmisc_base(accel_dev);
344
+
345
+
/*
346
+
* Enable all error notification bits in errsou3 except VFLR
347
+
* notification on host.
348
+
*/
349
+
ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_VFLNOTIFY);
350
+
}
351
+
352
+
static void enable_ints(struct adf_accel_dev *accel_dev)
353
+
{
354
+
void __iomem *addr = adf_get_pmisc_base(accel_dev);
355
+
356
+
/* Enable bundle interrupts */
357
+
ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET, 0);
358
+
ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET, 0);
359
+
360
+
/* Enable misc interrupts */
361
+
ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_MASK_OFFSET, 0);
362
+
}
363
+
364
+
static void set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
365
+
{
366
+
void __iomem *addr = adf_get_pmisc_base(accel_dev);
367
+
u64 val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
368
+
u64 val = ADF_SSM_WDT_DEFAULT_VALUE;
369
+
370
+
/* Enable watchdog timer for sym and dc */
371
+
ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTATHL_OFFSET, ADF_SSMWDTATHH_OFFSET, val);
372
+
ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTCNVL_OFFSET, ADF_SSMWDTCNVH_OFFSET, val);
373
+
ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTUCSL_OFFSET, ADF_SSMWDTUCSH_OFFSET, val);
374
+
ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTDCPRL_OFFSET, ADF_SSMWDTDCPRH_OFFSET, val);
375
+
376
+
/* Enable watchdog timer for pke */
377
+
ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, val_pke);
378
+
}
379
+
380
+
/*
381
+
* The vector routing table is used to select the MSI-X entry to use for each
382
+
* interrupt source.
383
+
* The first ADF_GEN6_ETR_MAX_BANKS entries correspond to ring interrupts.
384
+
* The final entry corresponds to VF2PF or error interrupts.
385
+
* This vector table could be used to configure one MSI-X entry to be shared
386
+
* between multiple interrupt sources.
387
+
*
388
+
* The default routing is set to have a one to one correspondence between the
389
+
* interrupt source and the MSI-X entry used.
390
+
*/
391
+
static void set_msix_default_rttable(struct adf_accel_dev *accel_dev)
392
+
{
393
+
void __iomem *csr = adf_get_pmisc_base(accel_dev);
394
+
unsigned int i;
395
+
396
+
for (i = 0; i <= ADF_GEN6_ETR_MAX_BANKS; i++)
397
+
ADF_CSR_WR(csr, ADF_GEN6_MSIX_RTTABLE_OFFSET(i), i);
398
+
}
399
+
400
+
static int reset_ring_pair(void __iomem *csr, u32 bank_number)
401
+
{
402
+
u32 status;
403
+
int ret;
404
+
405
+
/*
406
+
* Write rpresetctl register BIT(0) as 1.
407
+
* Since rpresetctl registers have no RW fields, no need to preserve
408
+
* values for other bits. Just write directly.
409
+
*/
410
+
ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
411
+
ADF_WQM_CSR_RPRESETCTL_RESET);
412
+
413
+
/* Read rpresetsts register and wait for rp reset to complete */
414
+
ret = read_poll_timeout(ADF_CSR_RD, status,
415
+
status & ADF_WQM_CSR_RPRESETSTS_STATUS,
416
+
ADF_RPRESET_POLL_DELAY_US,
417
+
ADF_RPRESET_POLL_TIMEOUT_US, true,
418
+
csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
419
+
if (ret)
420
+
return ret;
421
+
422
+
/* When ring pair reset is done, clear rpresetsts */
423
+
ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), ADF_WQM_CSR_RPRESETSTS_STATUS);
424
+
425
+
return 0;
426
+
}
427
+
428
+
static int ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number)
429
+
{
430
+
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
431
+
void __iomem *csr = adf_get_etr_base(accel_dev);
432
+
int ret;
433
+
434
+
if (bank_number >= hw_data->num_banks)
435
+
return -EINVAL;
436
+
437
+
dev_dbg(&GET_DEV(accel_dev), "ring pair reset for bank:%d\n", bank_number);
438
+
439
+
ret = reset_ring_pair(csr, bank_number);
440
+
if (ret)
441
+
dev_err(&GET_DEV(accel_dev), "ring pair reset failed (timeout)\n");
442
+
else
443
+
dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n");
444
+
445
+
return ret;
446
+
}
447
+
448
+
static int build_comp_block(void *ctx, enum adf_dc_algo algo)
449
+
{
450
+
struct icp_qat_fw_comp_req *req_tmpl = ctx;
451
+
struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
452
+
struct icp_qat_hw_comp_51_config_csr_lower hw_comp_lower_csr = { };
453
+
struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
454
+
u32 lower_val;
455
+
456
+
switch (algo) {
457
+
case QAT_DEFLATE:
458
+
header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC;
459
+
break;
460
+
default:
461
+
return -EINVAL;
462
+
}
463
+
464
+
hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED;
465
+
hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1;
466
+
lower_val = ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(hw_comp_lower_csr);
467
+
cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val;
468
+
cd_pars->u.sl.comp_slice_cfg_word[1] = 0;
469
+
470
+
return 0;
471
+
}
472
+
473
+
static int build_decomp_block(void *ctx, enum adf_dc_algo algo)
474
+
{
475
+
struct icp_qat_fw_comp_req *req_tmpl = ctx;
476
+
struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
477
+
struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
478
+
479
+
switch (algo) {
480
+
case QAT_DEFLATE:
481
+
header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS;
482
+
break;
483
+
default:
484
+
return -EINVAL;
485
+
}
486
+
487
+
cd_pars->u.sl.comp_slice_cfg_word[0] = 0;
488
+
cd_pars->u.sl.comp_slice_cfg_word[1] = 0;
489
+
490
+
return 0;
491
+
}
492
+
493
+
static void adf_gen6_init_dc_ops(struct adf_dc_ops *dc_ops)
494
+
{
495
+
dc_ops->build_comp_block = build_comp_block;
496
+
dc_ops->build_decomp_block = build_decomp_block;
497
+
}
498
+
499
+
static int adf_gen6_init_thd2arb_map(struct adf_accel_dev *accel_dev)
500
+
{
501
+
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
502
+
u32 *thd2arb_map = hw_data->thd_to_arb_map;
503
+
unsigned int i;
504
+
505
+
for (i = 0; i < hw_data->num_engines; i++) {
506
+
thd2arb_map[i] = adf_gen6_get_arb_mask(accel_dev, i);
507
+
dev_dbg(&GET_DEV(accel_dev), "ME:%d arb_mask:%#x\n", i, thd2arb_map[i]);
508
+
}
509
+
510
+
return 0;
511
+
}
512
+
513
+
static void set_vc_csr_for_bank(void __iomem *csr, u32 bank_number)
514
+
{
515
+
u32 value;
516
+
517
+
/*
518
+
* After each PF FLR, for each of the 64 ring pairs in the PF, the
519
+
* driver must program the ringmodectl CSRs.
520
+
*/
521
+
value = ADF_CSR_RD(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number));
522
+
value |= FIELD_PREP(ADF_GEN6_RINGMODECTL_TC_MASK, ADF_GEN6_RINGMODECTL_TC_DEFAULT);
523
+
value |= FIELD_PREP(ADF_GEN6_RINGMODECTL_TC_EN_MASK, ADF_GEN6_RINGMODECTL_TC_EN_OP1);
524
+
ADF_CSR_WR(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number), value);
525
+
}
526
+
527
+
static int set_vc_config(struct adf_accel_dev *accel_dev)
528
+
{
529
+
struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
530
+
u32 value;
531
+
int err;
532
+
533
+
/*
534
+
* After each PF FLR, the driver must program the Port Virtual Channel (VC)
535
+
* Control Registers.
536
+
* Read PVC0CTL then write the masked values.
537
+
*/
538
+
pci_read_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, &value);
539
+
value |= FIELD_PREP(ADF_GEN6_PVC0CTL_TCVCMAP_MASK, ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT);
540
+
err = pci_write_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, value);
541
+
if (err) {
542
+
dev_err(&GET_DEV(accel_dev), "pci write to PVC0CTL failed\n");
543
+
return pcibios_err_to_errno(err);
544
+
}
545
+
546
+
/* Read PVC1CTL then write masked values */
547
+
pci_read_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, &value);
548
+
value |= FIELD_PREP(ADF_GEN6_PVC1CTL_TCVCMAP_MASK, ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT);
549
+
value |= FIELD_PREP(ADF_GEN6_PVC1CTL_VCEN_MASK, ADF_GEN6_PVC1CTL_VCEN_ON);
550
+
err = pci_write_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, value);
551
+
if (err)
552
+
dev_err(&GET_DEV(accel_dev), "pci write to PVC1CTL failed\n");
553
+
554
+
return pcibios_err_to_errno(err);
555
+
}
556
+
557
+
static int adf_gen6_set_vc(struct adf_accel_dev *accel_dev)
558
+
{
559
+
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
560
+
void __iomem *csr = adf_get_etr_base(accel_dev);
561
+
u32 i;
562
+
563
+
for (i = 0; i < hw_data->num_banks; i++) {
564
+
dev_dbg(&GET_DEV(accel_dev), "set virtual channels for bank:%d\n", i);
565
+
set_vc_csr_for_bank(csr, i);
566
+
}
567
+
568
+
return set_vc_config(accel_dev);
569
+
}
570
+
571
+
static u32 get_ae_mask(struct adf_hw_device_data *self)
572
+
{
573
+
unsigned long fuses = self->fuses[ADF_FUSECTL4];
574
+
u32 mask = ADF_6XXX_ACCELENGINES_MASK;
575
+
576
+
/*
577
+
* If bit 0 is set in the fuses, the first 4 engines are disabled.
578
+
* If bit 4 is set, the second group of 4 engines are disabled.
579
+
* If bit 8 is set, the admin engine (bit 8) is disabled.
580
+
*/
581
+
if (test_bit(0, &fuses))
582
+
mask &= ~ADF_AE_GROUP_0;
583
+
584
+
if (test_bit(4, &fuses))
585
+
mask &= ~ADF_AE_GROUP_1;
586
+
587
+
if (test_bit(8, &fuses))
588
+
mask &= ~ADF_AE_GROUP_2;
589
+
590
+
return mask;
591
+
}
592
+
593
+
static u32 get_accel_cap(struct adf_accel_dev *accel_dev)
594
+
{
595
+
u32 capabilities_sym, capabilities_asym;
596
+
u32 capabilities_dc;
597
+
unsigned long mask;
598
+
u32 caps = 0;
599
+
u32 fusectl1;
600
+
601
+
fusectl1 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL1];
602
+
603
+
/* Read accelerator capabilities mask */
604
+
capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
605
+
ICP_ACCEL_CAPABILITIES_CIPHER |
606
+
ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
607
+
ICP_ACCEL_CAPABILITIES_SHA3 |
608
+
ICP_ACCEL_CAPABILITIES_SHA3_EXT |
609
+
ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
610
+
ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
611
+
ICP_ACCEL_CAPABILITIES_AES_V2;
612
+
613
+
/* A set bit in fusectl1 means the corresponding feature is OFF in this SKU */
614
+
if (fusectl1 & ICP_ACCEL_GEN6_MASK_UCS_SLICE) {
615
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
616
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
617
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY;
618
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC;
619
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2;
620
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
621
+
}
622
+
if (fusectl1 & ICP_ACCEL_GEN6_MASK_AUTH_SLICE) {
623
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
624
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3;
625
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT;
626
+
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
627
+
}
628
+
629
+
capabilities_asym = 0;
630
+
631
+
capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION |
632
+
ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
633
+
ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
634
+
ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
635
+
636
+
if (fusectl1 & ICP_ACCEL_GEN6_MASK_CPR_SLICE) {
637
+
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
638
+
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION;
639
+
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION;
640
+
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
641
+
}
642
+
643
+
if (adf_get_service_mask(accel_dev, &mask))
644
+
return 0;
645
+
646
+
if (test_bit(SVC_ASYM, &mask))
647
+
caps |= capabilities_asym;
648
+
if (test_bit(SVC_SYM, &mask))
649
+
caps |= capabilities_sym;
650
+
if (test_bit(SVC_DC, &mask))
651
+
caps |= capabilities_dc;
652
+
if (test_bit(SVC_DCC, &mask)) {
653
+
/*
654
+
* Sym capabilities are available for chaining operations,
655
+
* but sym crypto instances cannot be supported
656
+
*/
657
+
caps = capabilities_dc | capabilities_sym;
658
+
caps &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
659
+
}
660
+
661
+
return caps;
662
+
}
663
+
664
+
static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev)
665
+
{
666
+
return ARRAY_SIZE(adf_default_fw_config);
667
+
}
668
+
669
+
static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num)
670
+
{
671
+
int num_fw_objs = ARRAY_SIZE(adf_6xxx_fw_objs);
672
+
int id;
673
+
674
+
id = adf_default_fw_config[obj_num].obj;
675
+
if (id >= num_fw_objs)
676
+
return NULL;
677
+
678
+
return adf_6xxx_fw_objs[id];
679
+
}
680
+
681
+
static const char *uof_get_name_6xxx(struct adf_accel_dev *accel_dev, u32 obj_num)
682
+
{
683
+
return uof_get_name(accel_dev, obj_num);
684
+
}
685
+
686
+
static int uof_get_obj_type(struct adf_accel_dev *accel_dev, u32 obj_num)
687
+
{
688
+
if (obj_num >= uof_get_num_objs(accel_dev))
689
+
return -EINVAL;
690
+
691
+
return adf_default_fw_config[obj_num].obj;
692
+
}
693
+
694
+
static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
695
+
{
696
+
return adf_default_fw_config[obj_num].ae_mask;
697
+
}
698
+
699
+
static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev)
700
+
{
701
+
if (adf_gen6_init_thd2arb_map(accel_dev))
702
+
dev_warn(&GET_DEV(accel_dev),
703
+
"Failed to generate thread to arbiter mapping");
704
+
705
+
return GET_HW_DATA(accel_dev)->thd_to_arb_map;
706
+
}
707
+
708
+
static int adf_init_device(struct adf_accel_dev *accel_dev)
709
+
{
710
+
void __iomem *addr = adf_get_pmisc_base(accel_dev);
711
+
u32 status;
712
+
u32 csr;
713
+
int ret;
714
+
715
+
/* Temporarily mask PM interrupt */
716
+
csr = ADF_CSR_RD(addr, ADF_GEN6_ERRMSK2);
717
+
csr |= ADF_GEN6_PM_SOU;
718
+
ADF_CSR_WR(addr, ADF_GEN6_ERRMSK2, csr);
719
+
720
+
/* Set DRV_ACTIVE bit to power up the device */
721
+
ADF_CSR_WR(addr, ADF_GEN6_PM_INTERRUPT, ADF_GEN6_PM_DRV_ACTIVE);
722
+
723
+
/* Poll status register to make sure the device is powered up */
724
+
ret = read_poll_timeout(ADF_CSR_RD, status,
725
+
status & ADF_GEN6_PM_INIT_STATE,
726
+
ADF_GEN6_PM_POLL_DELAY_US,
727
+
ADF_GEN6_PM_POLL_TIMEOUT_US, true, addr,
728
+
ADF_GEN6_PM_STATUS);
729
+
if (ret) {
730
+
dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n");
731
+
return ret;
732
+
}
733
+
734
+
dev_dbg(&GET_DEV(accel_dev), "Setting virtual channels for device qat_dev%d\n",
735
+
accel_dev->accel_id);
736
+
737
+
ret = adf_gen6_set_vc(accel_dev);
738
+
if (ret)
739
+
dev_err(&GET_DEV(accel_dev), "Failed to set virtual channels\n");
740
+
741
+
return ret;
742
+
}
743
+
744
+
static int enable_pm(struct adf_accel_dev *accel_dev)
745
+
{
746
+
return adf_init_admin_pm(accel_dev, ADF_GEN6_PM_DEFAULT_IDLE_FILTER);
747
+
}
748
+
749
+
static int dev_config(struct adf_accel_dev *accel_dev)
750
+
{
751
+
int ret;
752
+
753
+
ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC);
754
+
if (ret)
755
+
return ret;
756
+
757
+
ret = adf_cfg_section_add(accel_dev, "Accelerator0");
758
+
if (ret)
759
+
return ret;
760
+
761
+
switch (adf_get_service_enabled(accel_dev)) {
762
+
case SVC_DC:
763
+
case SVC_DCC:
764
+
ret = adf_gen6_comp_dev_config(accel_dev);
765
+
break;
766
+
default:
767
+
ret = adf_gen6_no_dev_config(accel_dev);
768
+
break;
769
+
}
770
+
if (ret)
771
+
return ret;
772
+
773
+
__set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
774
+
775
+
return ret;
776
+
}
777
+
778
+
void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data)
779
+
{
780
+
hw_data->dev_class = &adf_6xxx_class;
781
+
hw_data->instance_id = adf_6xxx_class.instances++;
782
+
hw_data->num_banks = ADF_GEN6_ETR_MAX_BANKS;
783
+
hw_data->num_banks_per_vf = ADF_GEN6_NUM_BANKS_PER_VF;
784
+
hw_data->num_rings_per_bank = ADF_GEN6_NUM_RINGS_PER_BANK;
785
+
hw_data->num_accel = ADF_GEN6_MAX_ACCELERATORS;
786
+
hw_data->num_engines = ADF_6XXX_MAX_ACCELENGINES;
787
+
hw_data->num_logical_accel = 1;
788
+
hw_data->tx_rx_gap = ADF_GEN6_RX_RINGS_OFFSET;
789
+
hw_data->tx_rings_mask = ADF_GEN6_TX_RINGS_MASK;
790
+
hw_data->ring_to_svc_map = 0;
791
+
hw_data->alloc_irq = adf_isr_resource_alloc;
792
+
hw_data->free_irq = adf_isr_resource_free;
793
+
hw_data->enable_error_correction = enable_error_correction;
794
+
hw_data->get_accel_mask = get_accel_mask;
795
+
hw_data->get_ae_mask = get_ae_mask;
796
+
hw_data->get_num_accels = get_num_accels;
797
+
hw_data->get_num_aes = get_num_aes;
798
+
hw_data->get_sram_bar_id = get_sram_bar_id;
799
+
hw_data->get_etr_bar_id = get_etr_bar_id;
800
+
hw_data->get_misc_bar_id = get_misc_bar_id;
801
+
hw_data->get_arb_info = get_arb_info;
802
+
hw_data->get_admin_info = get_admin_info;
803
+
hw_data->get_accel_cap = get_accel_cap;
804
+
hw_data->get_sku = get_sku;
805
+
hw_data->init_admin_comms = adf_init_admin_comms;
806
+
hw_data->exit_admin_comms = adf_exit_admin_comms;
807
+
hw_data->send_admin_init = adf_send_admin_init;
808
+
hw_data->init_arb = adf_init_arb;
809
+
hw_data->exit_arb = adf_exit_arb;
810
+
hw_data->get_arb_mapping = adf_get_arbiter_mapping;
811
+
hw_data->enable_ints = enable_ints;
812
+
hw_data->reset_device = adf_reset_flr;
813
+
hw_data->admin_ae_mask = ADF_6XXX_ADMIN_AE_MASK;
814
+
hw_data->fw_name = ADF_6XXX_FW;
815
+
hw_data->fw_mmp_name = ADF_6XXX_MMP;
816
+
hw_data->uof_get_name = uof_get_name_6xxx;
817
+
hw_data->uof_get_num_objs = uof_get_num_objs;
818
+
hw_data->uof_get_obj_type = uof_get_obj_type;
819
+
hw_data->uof_get_ae_mask = uof_get_ae_mask;
820
+
hw_data->set_msix_rttable = set_msix_default_rttable;
821
+
hw_data->set_ssm_wdtimer = set_ssm_wdtimer;
822
+
hw_data->get_ring_to_svc_map = get_ring_to_svc_map;
823
+
hw_data->disable_iov = adf_disable_sriov;
824
+
hw_data->ring_pair_reset = ring_pair_reset;
825
+
hw_data->dev_config = dev_config;
826
+
hw_data->get_hb_clock = get_heartbeat_clock;
827
+
hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE;
828
+
hw_data->start_timer = adf_timer_start;
829
+
hw_data->stop_timer = adf_timer_stop;
830
+
hw_data->init_device = adf_init_device;
831
+
hw_data->enable_pm = enable_pm;
832
+
hw_data->services_supported = services_supported;
833
+
834
+
adf_gen6_init_hw_csr_ops(&hw_data->csr_ops);
835
+
adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops);
836
+
adf_gen6_init_dc_ops(&hw_data->dc_ops);
837
+
}
838
+
839
+
void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data)
840
+
{
841
+
if (hw_data->dev_class->instances)
842
+
hw_data->dev_class->instances--;
843
+
}
+148
drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h
+148
drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
#ifndef ADF_6XXX_HW_DATA_H_
4
+
#define ADF_6XXX_HW_DATA_H_
5
+
6
+
#include <linux/bits.h>
7
+
#include <linux/time.h>
8
+
#include <linux/units.h>
9
+
10
+
#include "adf_accel_devices.h"
11
+
#include "adf_cfg_common.h"
12
+
#include "adf_dc.h"
13
+
14
+
/* PCIe configuration space */
15
+
#define ADF_GEN6_BAR_MASK (BIT(0) | BIT(2) | BIT(4))
16
+
#define ADF_GEN6_SRAM_BAR 0
17
+
#define ADF_GEN6_PMISC_BAR 1
18
+
#define ADF_GEN6_ETR_BAR 2
19
+
#define ADF_6XXX_MAX_ACCELENGINES 9
20
+
21
+
/* Clocks frequency */
22
+
#define ADF_GEN6_COUNTER_FREQ (100 * HZ_PER_MHZ)
23
+
24
+
/* Physical function fuses */
25
+
#define ADF_GEN6_FUSECTL0_OFFSET 0x2C8
26
+
#define ADF_GEN6_FUSECTL1_OFFSET 0x2CC
27
+
#define ADF_GEN6_FUSECTL4_OFFSET 0x2D8
28
+
29
+
/* Accelerators */
30
+
#define ADF_GEN6_ACCELERATORS_MASK 0x1
31
+
#define ADF_GEN6_MAX_ACCELERATORS 1
32
+
33
+
/* MSI-X interrupt */
34
+
#define ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET 0x41A040
35
+
#define ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET 0x41A044
36
+
#define ADF_GEN6_SMIAPF_MASK_OFFSET 0x41A084
37
+
#define ADF_GEN6_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 4))
38
+
39
+
/* Bank and ring configuration */
40
+
#define ADF_GEN6_NUM_RINGS_PER_BANK 2
41
+
#define ADF_GEN6_NUM_BANKS_PER_VF 4
42
+
#define ADF_GEN6_ETR_MAX_BANKS 64
43
+
#define ADF_GEN6_RX_RINGS_OFFSET 1
44
+
#define ADF_GEN6_TX_RINGS_MASK 0x1
45
+
46
+
/* Arbiter configuration */
47
+
#define ADF_GEN6_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0))
48
+
#define ADF_GEN6_ARB_OFFSET 0x000
49
+
#define ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET 0x400
50
+
51
+
/* Admin interface configuration */
52
+
#define ADF_GEN6_ADMINMSGUR_OFFSET 0x500574
53
+
#define ADF_GEN6_ADMINMSGLR_OFFSET 0x500578
54
+
#define ADF_GEN6_MAILBOX_BASE_OFFSET 0x600970
55
+
56
+
/*
57
+
* Watchdog timers
58
+
* Timeout is in cycles. Clock speed may vary across products but this
59
+
* value should be a few milli-seconds.
60
+
*/
61
+
#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL
62
+
#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000ULL
63
+
#define ADF_SSMWDTATHL_OFFSET 0x5208
64
+
#define ADF_SSMWDTATHH_OFFSET 0x520C
65
+
#define ADF_SSMWDTCNVL_OFFSET 0x5408
66
+
#define ADF_SSMWDTCNVH_OFFSET 0x540C
67
+
#define ADF_SSMWDTUCSL_OFFSET 0x5808
68
+
#define ADF_SSMWDTUCSH_OFFSET 0x580C
69
+
#define ADF_SSMWDTDCPRL_OFFSET 0x5A08
70
+
#define ADF_SSMWDTDCPRH_OFFSET 0x5A0C
71
+
#define ADF_SSMWDTPKEL_OFFSET 0x5E08
72
+
#define ADF_SSMWDTPKEH_OFFSET 0x5E0C
73
+
74
+
/* Ring reset */
75
+
#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC)
76
+
#define ADF_RPRESET_POLL_DELAY_US 20
77
+
#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0)
78
+
#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + (bank) * 8)
79
+
#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0)
80
+
#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4)
81
+
82
+
/* Controls and sets up the corresponding ring mode of operation */
83
+
#define ADF_GEN6_CSR_RINGMODECTL(bank) (0x9000 + (bank) * 4)
84
+
85
+
/* Specifies the traffic class to use for the transactions to/from the ring */
86
+
#define ADF_GEN6_RINGMODECTL_TC_MASK GENMASK(18, 16)
87
+
#define ADF_GEN6_RINGMODECTL_TC_DEFAULT 0x7
88
+
89
+
/* Specifies usage of tc for the transactions to/from this ring */
90
+
#define ADF_GEN6_RINGMODECTL_TC_EN_MASK GENMASK(20, 19)
91
+
92
+
/*
93
+
* Use the value programmed in the tc field for request descriptor
94
+
* and metadata read transactions
95
+
*/
96
+
#define ADF_GEN6_RINGMODECTL_TC_EN_OP1 0x1
97
+
98
+
/* VC0 Resource Control Register */
99
+
#define ADF_GEN6_PVC0CTL_OFFSET 0x204
100
+
#define ADF_GEN6_PVC0CTL_TCVCMAP_OFFSET 1
101
+
#define ADF_GEN6_PVC0CTL_TCVCMAP_MASK GENMASK(7, 1)
102
+
#define ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT 0x7F
103
+
104
+
/* VC1 Resource Control Register */
105
+
#define ADF_GEN6_PVC1CTL_OFFSET 0x210
106
+
#define ADF_GEN6_PVC1CTL_TCVCMAP_OFFSET 1
107
+
#define ADF_GEN6_PVC1CTL_TCVCMAP_MASK GENMASK(7, 1)
108
+
#define ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT 0x40
109
+
#define ADF_GEN6_PVC1CTL_VCEN_OFFSET 31
110
+
#define ADF_GEN6_PVC1CTL_VCEN_MASK BIT(31)
111
+
/* RW bit: 0x1 - enables a Virtual Channel, 0x0 - disables */
112
+
#define ADF_GEN6_PVC1CTL_VCEN_ON 0x1
113
+
114
+
/* Error source mask registers */
115
+
#define ADF_GEN6_ERRMSK0 0x41A210
116
+
#define ADF_GEN6_ERRMSK1 0x41A214
117
+
#define ADF_GEN6_ERRMSK2 0x41A218
118
+
#define ADF_GEN6_ERRMSK3 0x41A21C
119
+
120
+
#define ADF_GEN6_VFLNOTIFY BIT(7)
121
+
122
+
/* Number of heartbeat counter pairs */
123
+
#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE
124
+
125
+
/* Physical function fuses */
126
+
#define ADF_6XXX_ACCELENGINES_MASK GENMASK(8, 0)
127
+
#define ADF_6XXX_ADMIN_AE_MASK GENMASK(8, 8)
128
+
129
+
/* Firmware binaries */
130
+
#define ADF_6XXX_FW "qat_6xxx.bin"
131
+
#define ADF_6XXX_MMP "qat_6xxx_mmp.bin"
132
+
#define ADF_6XXX_CY_OBJ "qat_6xxx_cy.bin"
133
+
#define ADF_6XXX_DC_OBJ "qat_6xxx_dc.bin"
134
+
#define ADF_6XXX_ADMIN_OBJ "qat_6xxx_admin.bin"
135
+
136
+
enum icp_qat_gen6_slice_mask {
137
+
ICP_ACCEL_GEN6_MASK_UCS_SLICE = BIT(0),
138
+
ICP_ACCEL_GEN6_MASK_AUTH_SLICE = BIT(1),
139
+
ICP_ACCEL_GEN6_MASK_PKE_SLICE = BIT(2),
140
+
ICP_ACCEL_GEN6_MASK_CPR_SLICE = BIT(3),
141
+
ICP_ACCEL_GEN6_MASK_DCPRZ_SLICE = BIT(4),
142
+
ICP_ACCEL_GEN6_MASK_WCP_WAT_SLICE = BIT(6),
143
+
};
144
+
145
+
void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data);
146
+
void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data);
147
+
148
+
#endif /* ADF_6XXX_HW_DATA_H_ */
+224
drivers/crypto/intel/qat/qat_6xxx/adf_drv.c
+224
drivers/crypto/intel/qat/qat_6xxx/adf_drv.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
#include <linux/array_size.h>
4
+
#include <linux/device.h>
5
+
#include <linux/dma-mapping.h>
6
+
#include <linux/errno.h>
7
+
#include <linux/list.h>
8
+
#include <linux/module.h>
9
+
#include <linux/pci.h>
10
+
#include <linux/types.h>
11
+
12
+
#include <adf_accel_devices.h>
13
+
#include <adf_cfg.h>
14
+
#include <adf_common_drv.h>
15
+
#include <adf_dbgfs.h>
16
+
17
+
#include "adf_gen6_shared.h"
18
+
#include "adf_6xxx_hw_data.h"
19
+
20
+
static int bar_map[] = {
21
+
0, /* SRAM */
22
+
2, /* PMISC */
23
+
4, /* ETR */
24
+
};
25
+
26
+
static void adf_device_down(void *accel_dev)
27
+
{
28
+
adf_dev_down(accel_dev);
29
+
}
30
+
31
+
static void adf_dbgfs_cleanup(void *accel_dev)
32
+
{
33
+
adf_dbgfs_exit(accel_dev);
34
+
}
35
+
36
+
static void adf_cfg_device_remove(void *accel_dev)
37
+
{
38
+
adf_cfg_dev_remove(accel_dev);
39
+
}
40
+
41
+
static void adf_cleanup_hw_data(void *accel_dev)
42
+
{
43
+
struct adf_accel_dev *accel_device = accel_dev;
44
+
45
+
if (accel_device->hw_device) {
46
+
adf_clean_hw_data_6xxx(accel_device->hw_device);
47
+
accel_device->hw_device = NULL;
48
+
}
49
+
}
50
+
51
+
static void adf_devmgr_remove(void *accel_dev)
52
+
{
53
+
adf_devmgr_rm_dev(accel_dev, NULL);
54
+
}
55
+
56
+
static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
57
+
{
58
+
struct adf_accel_pci *accel_pci_dev;
59
+
struct adf_hw_device_data *hw_data;
60
+
struct device *dev = &pdev->dev;
61
+
struct adf_accel_dev *accel_dev;
62
+
struct adf_bar *bar;
63
+
unsigned int i;
64
+
int ret;
65
+
66
+
if (num_possible_nodes() > 1 && dev_to_node(dev) < 0) {
67
+
/*
68
+
* If the accelerator is connected to a node with no memory
69
+
* there is no point in using the accelerator since the remote
70
+
* memory transaction will be very slow.
71
+
*/
72
+
return dev_err_probe(dev, -EINVAL, "Invalid NUMA configuration.\n");
73
+
}
74
+
75
+
accel_dev = devm_kzalloc(dev, sizeof(*accel_dev), GFP_KERNEL);
76
+
if (!accel_dev)
77
+
return -ENOMEM;
78
+
79
+
INIT_LIST_HEAD(&accel_dev->crypto_list);
80
+
INIT_LIST_HEAD(&accel_dev->list);
81
+
accel_pci_dev = &accel_dev->accel_pci_dev;
82
+
accel_pci_dev->pci_dev = pdev;
83
+
accel_dev->owner = THIS_MODULE;
84
+
85
+
hw_data = devm_kzalloc(dev, sizeof(*hw_data), GFP_KERNEL);
86
+
if (!hw_data)
87
+
return -ENOMEM;
88
+
89
+
pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
90
+
pci_read_config_dword(pdev, ADF_GEN6_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]);
91
+
pci_read_config_dword(pdev, ADF_GEN6_FUSECTL0_OFFSET, &hw_data->fuses[ADF_FUSECTL0]);
92
+
pci_read_config_dword(pdev, ADF_GEN6_FUSECTL1_OFFSET, &hw_data->fuses[ADF_FUSECTL1]);
93
+
94
+
if (!(hw_data->fuses[ADF_FUSECTL1] & ICP_ACCEL_GEN6_MASK_WCP_WAT_SLICE))
95
+
return dev_err_probe(dev, -EFAULT, "Wireless mode is not supported.\n");
96
+
97
+
/* Enable PCI device */
98
+
ret = pcim_enable_device(pdev);
99
+
if (ret)
100
+
return dev_err_probe(dev, ret, "Cannot enable PCI device.\n");
101
+
102
+
ret = adf_devmgr_add_dev(accel_dev, NULL);
103
+
if (ret)
104
+
return dev_err_probe(dev, ret, "Failed to add new accelerator device.\n");
105
+
106
+
ret = devm_add_action_or_reset(dev, adf_devmgr_remove, accel_dev);
107
+
if (ret)
108
+
return ret;
109
+
110
+
accel_dev->hw_device = hw_data;
111
+
adf_init_hw_data_6xxx(accel_dev->hw_device);
112
+
113
+
ret = devm_add_action_or_reset(dev, adf_cleanup_hw_data, accel_dev);
114
+
if (ret)
115
+
return ret;
116
+
117
+
/* Get Accelerators and Accelerator Engine masks */
118
+
hw_data->accel_mask = hw_data->get_accel_mask(hw_data);
119
+
hw_data->ae_mask = hw_data->get_ae_mask(hw_data);
120
+
accel_pci_dev->sku = hw_data->get_sku(hw_data);
121
+
122
+
/* If the device has no acceleration engines then ignore it */
123
+
if (!hw_data->accel_mask || !hw_data->ae_mask ||
124
+
(~hw_data->ae_mask & ADF_GEN6_ACCELERATORS_MASK)) {
125
+
ret = -EFAULT;
126
+
return dev_err_probe(dev, ret, "No acceleration units were found.\n");
127
+
}
128
+
129
+
/* Create device configuration table */
130
+
ret = adf_cfg_dev_add(accel_dev);
131
+
if (ret)
132
+
return ret;
133
+
134
+
ret = devm_add_action_or_reset(dev, adf_cfg_device_remove, accel_dev);
135
+
if (ret)
136
+
return ret;
137
+
138
+
/* Set DMA identifier */
139
+
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
140
+
if (ret)
141
+
return dev_err_probe(dev, ret, "No usable DMA configuration.\n");
142
+
143
+
ret = adf_gen6_cfg_dev_init(accel_dev);
144
+
if (ret)
145
+
return dev_err_probe(dev, ret, "Failed to initialize configuration.\n");
146
+
147
+
/* Get accelerator capability mask */
148
+
hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev);
149
+
if (!hw_data->accel_capabilities_mask) {
150
+
ret = -EINVAL;
151
+
return dev_err_probe(dev, ret, "Failed to get capabilities mask.\n");
152
+
}
153
+
154
+
for (i = 0; i < ARRAY_SIZE(bar_map); i++) {
155
+
bar = &accel_pci_dev->pci_bars[i];
156
+
157
+
/* Map 64-bit PCIe BAR */
158
+
bar->virt_addr = pcim_iomap_region(pdev, bar_map[i], pci_name(pdev));
159
+
if (!bar->virt_addr) {
160
+
ret = -ENOMEM;
161
+
return dev_err_probe(dev, ret, "Failed to ioremap PCI region.\n");
162
+
}
163
+
}
164
+
165
+
pci_set_master(pdev);
166
+
167
+
/*
168
+
* The PCI config space is saved at this point and will be restored
169
+
* after a Function Level Reset (FLR) as the FLR does not completely
170
+
* restore it.
171
+
*/
172
+
ret = pci_save_state(pdev);
173
+
if (ret)
174
+
return dev_err_probe(dev, ret, "Failed to save pci state.\n");
175
+
176
+
adf_dbgfs_init(accel_dev);
177
+
178
+
ret = devm_add_action_or_reset(dev, adf_dbgfs_cleanup, accel_dev);
179
+
if (ret)
180
+
return ret;
181
+
182
+
ret = adf_dev_up(accel_dev, true);
183
+
if (ret)
184
+
return ret;
185
+
186
+
ret = devm_add_action_or_reset(dev, adf_device_down, accel_dev);
187
+
if (ret)
188
+
return ret;
189
+
190
+
ret = adf_sysfs_init(accel_dev);
191
+
192
+
return ret;
193
+
}
194
+
195
+
static void adf_shutdown(struct pci_dev *pdev)
196
+
{
197
+
struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
198
+
199
+
adf_dev_down(accel_dev);
200
+
}
201
+
202
+
static const struct pci_device_id adf_pci_tbl[] = {
203
+
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_6XXX) },
204
+
{ }
205
+
};
206
+
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
207
+
208
+
static struct pci_driver adf_driver = {
209
+
.id_table = adf_pci_tbl,
210
+
.name = ADF_6XXX_DEVICE_NAME,
211
+
.probe = adf_probe,
212
+
.shutdown = adf_shutdown,
213
+
.sriov_configure = adf_sriov_configure,
214
+
.err_handler = &adf_err_handler,
215
+
};
216
+
module_pci_driver(adf_driver);
217
+
218
+
MODULE_LICENSE("GPL");
219
+
MODULE_AUTHOR("Intel");
220
+
MODULE_FIRMWARE(ADF_6XXX_FW);
221
+
MODULE_FIRMWARE(ADF_6XXX_MMP);
222
+
MODULE_DESCRIPTION("Intel(R) QuickAssist Technology for GEN6 Devices");
223
+
MODULE_SOFTDEP("pre: crypto-intel_qat");
224
+
MODULE_IMPORT_NS("CRYPTO_QAT");
+1
drivers/crypto/intel/qat/qat_common/Makefile
+1
drivers/crypto/intel/qat/qat_common/Makefile
+2
drivers/crypto/intel/qat/qat_common/adf_accel_devices.h
+2
drivers/crypto/intel/qat/qat_common/adf_accel_devices.h
···
26
26
#define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf"
27
27
#define ADF_4XXX_DEVICE_NAME "4xxx"
28
28
#define ADF_420XX_DEVICE_NAME "420xx"
29
+
#define ADF_6XXX_DEVICE_NAME "6xxx"
29
30
#define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940
30
31
#define PCI_DEVICE_ID_INTEL_QAT_4XXXIOV 0x4941
31
32
#define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942
···
36
35
#define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946
37
36
#define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947
38
37
#define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948
38
+
#define PCI_DEVICE_ID_INTEL_QAT_6XXX_IOV 0x4949
39
39
40
40
#define ADF_DEVICE_FUSECTL_OFFSET 0x40
41
41
#define ADF_DEVICE_LEGFUSE_OFFSET 0x4C
+1
drivers/crypto/intel/qat/qat_common/adf_cfg_common.h
+1
drivers/crypto/intel/qat/qat_common/adf_cfg_common.h
+1
drivers/crypto/intel/qat/qat_common/adf_fw_config.h
+1
drivers/crypto/intel/qat/qat_common/adf_fw_config.h
+28
drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h
+28
drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
#ifndef ADF_GEN6_PM_H
4
+
#define ADF_GEN6_PM_H
5
+
6
+
#include <linux/bits.h>
7
+
#include <linux/time.h>
8
+
9
+
struct adf_accel_dev;
10
+
11
+
/* Power management */
12
+
#define ADF_GEN6_PM_POLL_DELAY_US 20
13
+
#define ADF_GEN6_PM_POLL_TIMEOUT_US USEC_PER_SEC
14
+
#define ADF_GEN6_PM_STATUS 0x50A00C
15
+
#define ADF_GEN6_PM_INTERRUPT 0x50A028
16
+
17
+
/* Power management source in ERRSOU2 and ERRMSK2 */
18
+
#define ADF_GEN6_PM_SOU BIT(18)
19
+
20
+
/* cpm_pm_interrupt bitfields */
21
+
#define ADF_GEN6_PM_DRV_ACTIVE BIT(20)
22
+
23
+
#define ADF_GEN6_PM_DEFAULT_IDLE_FILTER 0x6
24
+
25
+
/* cpm_pm_status bitfields */
26
+
#define ADF_GEN6_PM_INIT_STATE BIT(21)
27
+
28
+
#endif /* ADF_GEN6_PM_H */